Git-Mirrors/constellation
1
0
Fork 0
mirror of https://github.com/edgelesssys/constellation.git synced 2024-10-01 01:36:09 -04:00
Code Issues Packages Projects Releases Wiki Activity

terraform-provider: cleanup and improve docs (#2685)

Browse Source 
Co-authored-by: Moritz Sanft <58110325+msanft@users.noreply.github.com>
This commit is contained in:
Adrian Stobbe 2023-12-14 15:47:55 +01:00 committed by GitHub
parent 9a4e96905f
commit 37580009fe
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
25 changed files with 262 additions and 592 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
.github/actions/upload_terraform_module/action.yml vendored
View File

@ -1,9 +1,5 @@
name: Upload Terraform module
description: "Upload the Terraform module as an artifact."
inputs:
version:
description: "The Constellation CLI version to use in the Terraform Module. Leave empty when a built CLI is already present when applying the module."
name: Upload Terraform infrastructure module
description: "Upload the Terraform infrastructure module as an artifact."
runs:
using: "composite"
@ -11,18 +7,7 @@ runs:
- name: Copy Terraform module
shell: bash
run: |
cp -r terraform terraform-module
find terraform-module -name "*.go" -type f -delete
find terraform-module -name "*.bazel" -type f -delete
- name: Stamp version
shell: bash
run: |
sed -i "s/@@CONSTELLATION_VERSION@@/${{ inputs.version }}/g" \
terraform-module/legacy-module/constellation-cluster/variables.tf \
terraform-module/legacy-module/aws-constellation/variables.tf \
terraform-module/legacy-module/azure-constellation/variables.tf \
terraform-module/legacy-module/gcp-constellation/variables.tf
cp -r terraform/infrastructure terraform-module
- name: Zip terraform dir
shell: bash

4
.github/workflows/draft-release.yml vendored
View File

@ -173,10 +173,8 @@ jobs:
with:
ref: ${{ inputs.ref || github.head_ref }}
- name: Upload Terraform module
- name: Upload Terraform infrastructure module
uses: ./.github/actions/upload_terraform_module
with:
version: ${{ inputs.versionName }}
push-containers:
runs-on: ubuntu-22.04

287
.github/workflows/e2e-test-tf-module.yml vendored
View File

@ -1,287 +0,0 @@
name: e2e test Terraform module
on:
workflow_dispatch:
inputs:
ref:
type: string
description: "Git ref to checkout"
cloudProvider:
description: "Which cloud provider to use."
type: choice
options:
- "aws"
- "azure"
- "gcp"
required: true
regionZone:
description: "Region or zone to create the cluster in. Leave empty for default region/zone."
type: string
image:
description: "OS Image version used in the cluster's VMs, as specified in the Constellation config. If not set, the latest nightly image from main is used."
type: string
cliVersion:
description: "Constellation CLI version to use. Empty value means build from source."
type: string
workflow_call:
inputs:
ref:
type: string
description: "Git ref to checkout"
cloudProvider:
description: "Which cloud provider to use."
type: string
required: true
regionZone:
description: "Which zone to use."
type: string
image:
description: "OS Image version used in the cluster's VMs, as specified in the Constellation config. If not set, the latest nightly image from main is used."
type: string
cliVersion:
description: "Constellation CLI version to use. Empty value means build from source."
type: string
jobs:
tf-module-test:
runs-on: ubuntu-22.04
permissions:
id-token: write
contents: read
packages: write
steps:
- name: Checkout
id: checkout
uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0
with:
ref: ${{ inputs.ref || github.head_ref }}
- name: Get Latest Image
id: find-latest-image
uses: ./.github/actions/find_latest_image
with:
git-ref: ${{ inputs.ref }}
imageVersion: ${{ inputs.image }}
ref: main
stream: nightly
- name: Upload Terraform module
uses: ./.github/actions/upload_terraform_module
with:
version: ${{ inputs.cliVersion }}
- name: Download Terraform module
uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2
with:
name: terraform-module
- name: Unzip Terraform module
shell: bash
run: |
unzip terraform-module.zip -d ${{ github.workspace }}
rm terraform-module.zip
- name: Create resource prefix
id: create-prefix
shell: bash
run: |
run_id=${{ github.run_id }}
last_three="${run_id: -3}"
echo "prefix=e2e-${last_three}-${{ github.run_attempt }}" | tee -a "$GITHUB_OUTPUT"
- name: Create AWS Terraform variable input file
if: inputs.cloudProvider == 'aws'
working-directory: ${{ github.workspace }}/terraform-module/legacy-module/aws-constellation
shell: bash
run: |
cat > terraform.tfvars <<EOF
name = "${{ steps.create-prefix.outputs.prefix }}"
image = "${{ steps.find-latest-image.outputs.image }}"
zone = "${{ inputs.regionZone || 'us-east-2c' }}"
name_prefix = "${{ steps.create-prefix.outputs.prefix }}"
debug = ${{ steps.find-latest-image.outputs.isDebugImage }}
node_groups = {
control_plane_default = {
role = "control-plane"
instance_type = "m6a.xlarge"
disk_size = 30
disk_type = "gp3"
initial_count = 2
zone = "${{ inputs.regionZone || 'us-east-2c' }}"
},
worker_default = {
role = "worker"
instance_type = "m6a.xlarge"
disk_size = 30
disk_type = "gp3"
initial_count = 2
zone = "${{ inputs.regionZone || 'us-east-2c' }}"
}
}
EOF
cat terraform.tfvars
- name: Create Azure Terraform variable input file
if: inputs.cloudProvider == 'azure'
working-directory: ${{ github.workspace }}/terraform-module/legacy-module/azure-constellation
shell: bash
run: |
cat > terraform.tfvars <<EOF
name = "${{ steps.create-prefix.outputs.prefix }}"
image = "${{ steps.find-latest-image.outputs.image }}"
location = "${{ inputs.regionZone || 'northeurope' }}"
service_principal_name = "${{ steps.create-prefix.outputs.prefix }}-sp"
resource_group_name = "${{ steps.create-prefix.outputs.prefix }}-rg"
debug = ${{ steps.find-latest-image.outputs.isDebugImage }}
node_groups = {
control_plane_default = {
role = "control-plane"
instance_type = "Standard_DC4as_v5"
disk_size = 30
disk_type = "Premium_LRS"
initial_count = 2
},
worker_default = {
role = "worker"
instance_type = "Standard_DC4as_v5"
disk_size = 30
disk_type = "Premium_LRS"
initial_count = 2
}
}
EOF
cat terraform.tfvars
- name: Create GCP Terraform variable input file
if: inputs.cloudProvider == 'gcp'
working-directory: ${{ github.workspace }}/terraform-module/legacy-module/gcp-constellation
shell: bash
run: |
cat > terraform.tfvars <<EOF
name = "${{ steps.create-prefix.outputs.prefix }}"
project = "constellation-e2e"
service_account_id = "${{ steps.create-prefix.outputs.prefix }}-sa"
image = "${{ steps.find-latest-image.outputs.image }}"
zone = "${{ inputs.regionZone || 'europe-west3-b' }}"
debug = ${{ steps.find-latest-image.outputs.isDebugImage }}
node_groups = {
control_plane_default = {
role = "control-plane"
instance_type = "n2d-standard-4"
disk_size = 30
disk_type = "pd-ssd"
initial_count = 2
zone = "${{ inputs.regionZone || 'europe-west3-b' }}"
},
worker_default = {
role = "worker"
instance_type = "n2d-standard-4"
disk_size = 30
disk_type = "pd-ssd"
initial_count = 2
zone = "${{ inputs.regionZone || 'europe-west3-b' }}"
}
}
EOF
cat terraform.tfvars
- name: Install dependencies (Terraform)
shell: bash
run: |
sudo apt update && sudo apt install gpg
wget -O- https://apt.releases.hashicorp.com/gpg | sudo gpg --dearmor -o /usr/share/keyrings/hashicorp-archive-keyring.gpg
gpg --no-default-keyring --keyring /usr/share/keyrings/hashicorp-archive-keyring.gpg --fingerprint
echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/hashicorp-archive-keyring.gpg] https://apt.releases.hashicorp.com $(lsb_release -cs) main" | sudo tee /etc/apt/sources.list.d/hashicorp.list
sudo apt update
sudo apt install terraform=1.4.4-*
- name: Log in to the Container registry
uses: ./.github/actions/container_registry_login
with:
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Setup bazel
if: inputs.cliVersion == ''
uses: ./.github/actions/setup_bazel_nix
with:
useCache: "true"
buildBuddyApiKey: ${{ secrets.BUILDBUDDY_ORG_API_KEY }}
- name: Build CLI
if: inputs.cliVersion == ''
uses: ./.github/actions/build_cli
with:
outputPath: "constellation"
enterpriseCLI: true
push: true
- name: Download CLI
if: inputs.cliVersion != ''
shell: bash
run: |
curl -fsSL -o constellation https://github.com/edgelesssys/constellation/releases/download/${{ inputs.cliVersion }}/constellation-linux-amd64
chmod u+x constellation
./constellation version
sudo sh -c 'echo "127.0.0.1 license.confidential.cloud" >> /etc/hosts'
- name: Login to AWS (IAM + Cluster role)
if: inputs.cloudProvider == 'aws'
uses: aws-actions/configure-aws-credentials@5fd3084fc36e372ff1fff382a39b10d03659f355 # v2.2.0
with:
role-to-assume: arn:aws:iam::795746500882:role/GithubActionsE2ETerraform
aws-region: eu-central-1
# extend token expiry to 6 hours to ensure constellation can terminate
role-duration-seconds: 21600
- name: Login to Azure (IAM + Cluster service principal)
if: inputs.cloudProvider == 'azure'
uses: ./.github/actions/login_azure
with:
azure_credentials: ${{ secrets.AZURE_E2E_TF_CREDENTIALS }}
- name: Login to GCP (IAM + Cluster service account)
if: inputs.cloudProvider == 'gcp'
uses: ./.github/actions/login_gcp
with:
service_account: "terraform-e2e@constellation-e2e.iam.gserviceaccount.com"
- name: Apply Terraform Cluster
id: apply_terraform
working-directory: ${{ github.workspace }}/terraform-module/${{ inputs.cloudProvider }}-constellation
shell: bash
run: |
cp ../../constellation .
terraform init
terraform apply -var-file=terraform.tfvars -auto-approve
- name: Destroy Terraform Cluster
# outcome is part of the steps context (https://docs.github.com/en/actions/learn-github-actions/contexts#steps-context)
if: always() && steps.apply_terraform.outcome != 'skipped'
working-directory: ${{ github.workspace }}/terraform-module/${{ inputs.cloudProvider }}-constellation
shell: bash
run: |
terraform init
terraform destroy -var-file=terraform.tfvars -auto-approve
- name: Verify cleanup
working-directory: ${{ github.workspace }}/terraform-module/${{ inputs.cloudProvider }}-constellation
shell: bash
run: |
if [ -f constellation-mastersecret.json ] || [ -f constellation-conf.yaml ]; then
echo "Files constellation-mastersecret.json or constellation-conf.yaml still exist"
exit 1
fi
- name: Notify about failure
if: |
failure() &&
github.ref == 'refs/heads/main' &&
github.event_name == 'schedule'
continue-on-error: true
uses: ./.github/actions/notify_e2e_failure
with:
projectWriteToken: ${{ secrets.PROJECT_WRITE_TOKEN }}
test: "terraform-module"
provider: ${{ inputs.cloudProvider }}

19
.github/workflows/e2e-test-weekly.yml vendored
View File

@ -368,22 +368,3 @@ jobs:
uses: ./.github/workflows/e2e-windows.yml
with:
scheduled: ${{ github.event_name == 'schedule' }}
e2e-tf-module:
name: Test Terraform Module
strategy:
fail-fast: false
max-parallel: 5
matrix:
include:
- provider: "gcp"
- provider: "aws"
- provider: "azure"
permissions:
id-token: write
contents: read
packages: write
secrets: inherit
uses: ./.github/workflows/e2e-test-tf-module.yml
with:
cloudProvider: "${{ matrix.provider }}"

2
docs/docs/workflows/create.md
View File

@ -45,7 +45,7 @@ It's recommended to use Terraform for infrastructure management, but you can use
:::info
When using Terraform, you can use the [Terraform module](./terraform-module.md) for ease of use to manage the entire Constellation cluster.
When using Terraform, you can use the [Terraform provider](./terraform-provider.md) to manage the entire Constellation cluster lifecycle.
:::

151
docs/docs/workflows/terraform-module.md
View File

@ -1,151 +0,0 @@
# Use the Terraform module
You can manage a Constellation cluster through Terraform.
The module package is available as part of the [GitHub release](https://github.com/edgelesssys/constellation/releases/). It consists of a convenience module for each cloud service provider (`{csp}-constellation`) that combines the IAM (`infrastructure/{csp}/iam`), infrastructure (`infrastructure/{csp}`), and constellation (`constellation-cluster`) modules.
## Prerequisites
- a Linux / Mac operating system
- a Terraform installation of version `v1.4.4` or above
## Quick setup
The convenience module allows setting up a Constellation cluster with a single module. It's easiest to consume the module through a remote source, as shown below.
This allows to upgrade the cluster to a newer Constellation version by simply updating the module source.
:::caution
In the current release of the module, `terraform apply` creates files such as `constellation-conf.yaml`, `constellation-state.yaml` , `constellation-admin.conf`, `constellation-mastersecret.json`, and a directory `constellation-terraform"` containing backups. Make sure to check in these files in your version control when using GitOps.
The files are deleted on `terraform destroy`.
:::
1. Create a directory (workspace) for your Constellation cluster.
```bash
mkdir constellation-workspace
cd constellation-workspace
```
1. Create a `main.tf` file to call the CSP specific Constellation module.
<tabs groupId="csp">
<tabItem value="azure" label="Azure">
```
module "azure-constellation" {
source = "https://github.com/edgelesssys/constellation/releases/download/<version>/terraform-module.zip//terraform-module/legacy-module/azure-constellation" // replace <version> with a Constellation version, e.g., v2.13.0
name = "constell"
location = "northeurope"
service_principal_name = "az-sp"
resource_group_name = "constell-rg"
node_groups = {
control_plane_default = {
role = "control-plane"
instance_type = "Standard_DC4as_v5"
disk_size = 30
disk_type = "Premium_LRS"
initial_count = 3
},
worker_default = {
role = "worker"
instance_type = "Standard_DC4as_v5"
disk_size = 30
disk_type = "Premium_LRS"
initial_count = 2
}
}
}
```
</tabItem>
<tabItem value="aws" label="AWS">
```
module "aws-constellation" {
source = "https://github.com/edgelesssys/constellation/releases/download/<version>/terraform-module.zip//terraform-module/legacy-module/aws-constellation" // replace <version> with a Constellation version, e.g., v2.13.0
name = "constell"
zone = "us-east-2c"
name_prefix = "example"
node_groups = {
control_plane_default = {
role = "control-plane"
zone = "us-east-2c"
instance_type = "m6a.xlarge"
disk_size = 30
disk_type = "gp3"
initial_count = 3
},
worker_default = {
role = "worker"
zone = "us-east-2c"
instance_type = "m6a.xlarge"
disk_size = 30
disk_type = "gp3"
initial_count = 2
}
}
}
```
</tabItem>
<tabItem value="gcp" label="GCP">
```
module "gcp-constellation" {
source = "https://github.com/edgelesssys/constellation/releases/download/<version>/terraform-module.zip//terraform-module/legacy-module/gcp-constellation" // replace <version> with a Constellation version, e.g., v2.13.0
name = "constell"
project = "constell-proj" // replace with your project id
service_account_id = "constid"
zone = "europe-west2-a"
node_groups = {
control_plane_default = {
role = "control-plane"
zone = "europe-west2-a"
instance_type = "n2d-standard-4"
disk_size = 30
disk_type = "pd-ssd"
initial_count = 3
},
worker_default = {
role = "worker"
zone = "europe-west2-a"
instance_type = "n2d-standard-4"
disk_size = 30
disk_type = "pd-ssd"
initial_count = 2
}
}
}
```
</tabItem>
</tabs>
3. Initialize and apply the module.
```bash
terraform init
terraform apply
```
## Custom setup
If you need to separate IAM and cluster management or need custom infrastructure, you can also call the submodules individually.
Look at the respective convenience module (`{csp}-constellation`) for how you can structure the module calls.
The submodules are:
- `constellation-cluster`: manages the Constellation cluster
- `fetch-image`: translates the Constellation image version to the image ID of the cloud service provider
- `infrastructure/{csp}`: contains the cluster infrastructure resources
- `infrastructure/iam/{csp}`: contains the IAM resources used within the cluster
## Cluster upgrades
:::tip
For general information on cluster upgrades, see [Upgrade your cluster](./upgrade.md).
:::
Using a [remote address as module source](https://developer.hashicorp.com/terraform/language/modules/sources#fetching-archives-over-http) as shown in [Quick setup](#quick-setup) is recommended because it simplifies the upgrade process. For [local paths as module source](https://developer.hashicorp.com/terraform/language/modules/sources#local-paths), you would need to manually overwrite the Terraform files in the Terraform workspace. The steps for the remote source setup are as follows:
1. Update the `<version>` variable inside the `source` field of the module.
2. Upgrade the Terraform module and provider dependencies and apply the Constellation upgrade.
```bash
terraform init -upgrade
terraform apply
```

72
docs/docs/workflows/terraform-provider.md Normal file
View File

@ -0,0 +1,72 @@
# Use the Terraform provider
The Constellation Terraform provider allows to manage the full lifecycle of a Constellation cluster (namely creation, upgrades, and deletion) via Terraform.
<!-- TODO(elchead): check link during release -->
The provider is available through the [Terraform registry](https://registry.terraform.io/providers/edgelesssys/constellation/latest) and is released in lock-step with Constellation releases.
## Prerequisites
- a Linux / Mac operating system (ARM64/AMD64)
- a Terraform installation of version `v1.4.4` or above
## Quick setup
This example shows how to set up a Constellation cluster with the reference IAM and infrastructure setup. This setup is also used when creating a Constellation cluster through the Constellation CLI. You can either consume the IAM / infrastructure modules through a remote source (recommended) or local files. The latter requires downloading the infrastructure and IAM modules for the corresponding CSP from `terraform-modules.zip` on the [Constellation release page](https://github.com/edgelesssys/constellation/releases/latest) and placing them in the Terraform workspace directory.
1. Create a directory (workspace) for your Constellation cluster.
```bash
mkdir constellation-workspace
cd constellation-workspace
```
1. Create a `main.tf` file.
<!--TODO(elchead): AB#3607 put correct examples, with follow up PR with #2713 examples
<tabs groupId="csp">
<tabItem value="azure" label="Azure">
</tabItem>
<tabItem value="aws" label="AWS">
</tabItem>
<tabItem value="gcp" label="GCP">
</tabItem>
</tabs>-->
1. Initialize and apply the file.
```bash
terraform init
terraform apply
```
## Bringing your own infrastructure
If you need a custom infrastructure setup, you can download the infrastructure / IAM Terraform modules for the respective CSP from the Constellation [GitHub releases](https://github.com/edgelesssys/constellation/releases). You can modify / extend the modules, per your requirements, while keeping the basic functionality intact.
The module contains:
- `{csp}`: cloud resources the cluster runs on
- `iam/{csp}`: IAM resources used within the cluster
When upgrading your cluster, make sure to check the Constellation release notes for potential breaking changes in the reference infrastructure / IAM modules that need to be considered.
## Cluster upgrades
:::tip
For general information on cluster upgrades, see the [dedicated upgrade page](./upgrade.md).
:::
The steps for applying the upgrade are as follows:
1. Update the version constraint of the Constellation Terraform provider in the `required_providers` block in your Terraform configuration.
2. If you explicitly set any of the version attributes of the provider's resources and data sources (e.g. `image_version` or `constellation_microservice_version`), make sure to update them too. Refer to the [version support policy](https://github.com/edgelesssys/constellation/blob/main/dev-docs/workflows/versions-support.md) for more information on how each Constellation version and its dependencies are supported.
3. Update the IAM / infrastructure modules.
- For [remote address as module source](https://developer.hashicorp.com/terraform/language/modules/sources#fetching-archives-over-http), update the version number inside the address of the `source` field of the infra / IAM module to the target version.
- For [local paths as module source](https://developer.hashicorp.com/terraform/language/modules/sources#local-paths), see the changes made in the reference modules since the upgrade's origin version and adjust your infrastructure configuration accordingly.
4. Upgrade the Terraform module and provider dependencies and apply the targeted configuration.
```bash
terraform init -upgrade
terraform apply
```

4
docs/sidebars.js
View File

@ -197,8 +197,8 @@ const sidebars = {
},
{
type: 'doc',
label: 'Use the Terraform module',
id: 'workflows/terraform-module',
label: 'Use the Terraform provider',
id: 'workflows/terraform-provider',
},
// {
// type: 'doc',

3
terraform-provider-constellation/BUILD.bazel
View File

@ -31,6 +31,9 @@ go_library(
srcs = ["main.go"],
importpath = "github.com/edgelesssys/constellation/v2/terraform-provider-constellation",
visibility = ["//visibility:private"],
x_defs = {
"version": "{STABLE_STAMP_VERSION}",
},
deps = [
"//terraform-provider-constellation/internal/provider",
"@com_github_hashicorp_terraform_plugin_framework//providerserver",

2
terraform-provider-constellation/docs/data-sources/attestation.md
View File

@ -32,10 +32,10 @@ data "constellation_attestation" "test" {
* `gcp-sev-es`
- `csp` (String) CSP (Cloud Service Provider) to use. (e.g. `azure`)
See the [full list of CSPs](https://docs.edgeless.systems/constellation/overview/clouds) that Constellation supports.
- `image_version` (String) The image version to use
### Optional
- `image_version` (String) The image version to use. If not set, the provider version value is used.
- `maa_url` (String) For Azure only, the URL of the Microsoft Azure Attestation service
### Read-Only

6
terraform-provider-constellation/docs/data-sources/image.md
View File

@ -3,12 +3,12 @@
page_title: "constellation_image Data Source - constellation"
subcategory: ""
description: |-
Data source to retrieve the Constellation OS image reference for a given CSP and Attestation Variant.
Data source to resolve the CSP-specific OS image reference for a given version and attestation variant.
---
# constellation_image (Data Source)
Data source to retrieve the Constellation OS image reference for a given CSP and Attestation Variant.
Data source to resolve the CSP-specific OS image reference for a given version and attestation variant.
## Example Usage
@ -33,10 +33,10 @@ data "constellation_image" "example" {
* `gcp-sev-es`
- `csp` (String) CSP (Cloud Service Provider) to use. (e.g. `azure`)
See the [full list of CSPs](https://docs.edgeless.systems/constellation/overview/clouds) that Constellation supports.
- `image_version` (String) Version of the Constellation OS image to use. (e.g. `v2.13.0`)
### Optional
- `image_version` (String) Version of the Constellation OS image to use. (e.g. `v2.13.0`). If not set, the provider version value is used.
- `marketplace_image` (Boolean) Whether a marketplace image should be used. Currently only supported for Azure.
- `region` (String) Region to retrieve the image for. Only required for AWS.
The Constellation OS image must be [replicated to the region](https://docs.edgeless.systems/constellation/workflows/config),and the region must [support AMD SEV-SNP](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/snp-requirements.html), if it is used for Attestation.

7
terraform-provider-constellation/docs/index.md
View File

@ -4,12 +4,19 @@ page_title: "constellation Provider"
subcategory: ""
description: |-
The Constellation provider manages Constellation clusters.
Given user-defined infrastructure in Terraform, the provider with its main 'constellation_cluster' resource manages the entire lifecycle of a cluster.
The provider allows easy usage of custom infrastructure setups and GitOps workflows.
It is released as part of Constellation releases, such that each provider version is compatible with the corresponding Constellation version.
---
# constellation Provider
The Constellation provider manages Constellation clusters.
Given user-defined infrastructure in Terraform, the provider with its main 'constellation_cluster' resource manages the entire lifecycle of a cluster.
The provider allows easy usage of custom infrastructure setups and GitOps workflows.
It is released as part of Constellation releases, such that each provider version is compatible with the corresponding Constellation version.
## Example Usage
```terraform

2
terraform-provider-constellation/docs/resources/cluster.md
View File

@ -42,7 +42,7 @@ resource "constellation_cluster" "aws_example" {
### Required
- `attestation` (Attributes) Attestation comprises the measurements and SEV-SNP specific parameters. (see [below for nested schema](#nestedatt--attestation))
- `attestation` (Attributes) Attestation comprises the measurements and SEV-SNP specific parameters. The output of the [constellation_attestation](../data-sources/attestation.md) data source provides sensible defaults. (see [below for nested schema](#nestedatt--attestation))
- `csp` (String) The Cloud Service Provider (CSP) the cluster should run on.
- `image_reference` (String) Constellation OS image reference to use in the CSP specific reference format. Use the [`constellation_image`](../data-sources/image.md) data source to find the correct image reference for your CSP.
- `image_version` (String) Constellation OS image version to use in the CSP specific reference format. Use the [`constellation_image`](../data-sources/image.md) data source to find the correct image version for your CSP.

4
terraform-provider-constellation/internal/data/providerdata.go
View File

@ -8,4 +8,6 @@ package data
// ProviderData is the data that get's passed down from the provider
// configuration to the resources and data sources.
type ProviderData struct{}
type ProviderData struct {
Version string
}

5
terraform-provider-constellation/internal/provider/BUILD.bazel
View File

@ -78,12 +78,15 @@ go_test(
"requires-network",
],
# keep
x_defs = {"runsUnder": "bazel"},
x_defs = {
"runsUnder": "bazel",
},
deps = [
"//internal/attestation/idkeydigest",
"//internal/attestation/measurements",
"//internal/attestation/variant",
"//internal/config",
"//internal/constants",
"@com_github_hashicorp_terraform_plugin_framework//providerserver",
"@com_github_hashicorp_terraform_plugin_go//tfprotov6",
"@com_github_hashicorp_terraform_plugin_testing//helper/resource",

53
terraform-provider-constellation/internal/provider/attestation_data_source.go
View File

@ -17,6 +17,7 @@ import (
"github.com/edgelesssys/constellation/v2/internal/attestation/variant"
"github.com/edgelesssys/constellation/v2/internal/cloud/cloudprovider"
"github.com/edgelesssys/constellation/v2/internal/sigstore"
"github.com/edgelesssys/constellation/v2/terraform-provider-constellation/internal/data"
"github.com/hashicorp/terraform-plugin-framework/datasource"
"github.com/hashicorp/terraform-plugin-framework/datasource/schema"
"github.com/hashicorp/terraform-plugin-framework/path"
@ -37,6 +38,7 @@ type AttestationDataSource struct {
client *http.Client
fetcher attestationconfigapi.Fetcher
rekor *sigstore.Rekor
version string
}
// AttestationDataSourceModel describes the data source data model.
@ -49,7 +51,21 @@ type AttestationDataSourceModel struct {
}
// Configure configures the data source.
func (d *AttestationDataSource) Configure(_ context.Context, _ datasource.ConfigureRequest, resp *datasource.ConfigureResponse) {
func (d *AttestationDataSource) Configure(_ context.Context, req datasource.ConfigureRequest, resp *datasource.ConfigureResponse) {
// Prevent panic if the provider has not been configured. is necessary!
if req.ProviderData == nil {
return
}
providerData, ok := req.ProviderData.(data.ProviderData)
if !ok {
resp.Diagnostics.AddError(
"Unexpected Data Source Configure Type",
fmt.Sprintf("Expected data.ProviderData, got: %T. Please report this issue to the provider developers.", req.ProviderData),
)
return
}
d.version = providerData.Version
d.client = http.DefaultClient
d.fetcher = attestationconfigapi.NewFetcher()
rekor, err := sigstore.NewRekor()
@ -70,13 +86,12 @@ func (d *AttestationDataSource) Schema(_ context.Context, _ datasource.SchemaReq
resp.Schema = schema.Schema{
// This description is used by the documentation generator and the language server.
MarkdownDescription: "The data source to fetch measurements from a configured cloud provider and image.",
Attributes: map[string]schema.Attribute{
"csp": newCSPAttribute(),
"attestation_variant": newAttestationVariantAttribute(attributeInput),
"image_version": schema.StringAttribute{
MarkdownDescription: "The image version to use",
Required: true,
MarkdownDescription: "The image version to use. If not set, the provider version value is used.",
Optional: true,
},
"maa_url": schema.StringAttribute{
MarkdownDescription: "For Azure only, the URL of the Microsoft Azure Attestation service",
@ -87,6 +102,28 @@ func (d *AttestationDataSource) Schema(_ context.Context, _ datasource.SchemaReq
}
}
// ValidateConfig validates the configuration for the image data source.
func (d *AttestationDataSource) ValidateConfig(ctx context.Context, req datasource.ValidateConfigRequest, resp *datasource.ValidateConfigResponse) {
var data AttestationDataSourceModel
resp.Diagnostics.Append(req.Config.Get(ctx, &data)...)
if resp.Diagnostics.HasError() {
return
}
if !data.AttestationVariant.Equal(types.StringValue("azure-sev-snp")) && !data.MaaURL.IsNull() {
resp.Diagnostics.AddAttributeWarning(
path.Root("maa_url"),
"MAA URL should only be set for Azure SEV-SNP", "Only when attestation_variant is set to 'azure-sev-snp', 'maa_url' should be specified.",
)
return
}
if data.AttestationVariant.Equal(types.StringValue("azure-sev-snp")) && data.MaaURL.IsNull() {
tflog.Info(ctx, "MAA URL not set, MAA fallback will be unavaiable")
}
}
// Read reads from the data source.
func (d *AttestationDataSource) Read(ctx context.Context, req datasource.ReadRequest, resp *datasource.ReadResponse) {
var data AttestationDataSourceModel
@ -130,7 +167,13 @@ func (d *AttestationDataSource) Read(ctx context.Context, req datasource.ReadReq
resp.Diagnostics.AddError("Converting SNP attestation", err.Error())
}
verifyFetcher := measurements.NewVerifyFetcher(sigstore.NewCosignVerifier, d.rekor, d.client)
fetchedMeasurements, err := verifyFetcher.FetchAndVerifyMeasurements(ctx, data.ImageVersion.ValueString(),
imageVersion := data.ImageVersion.ValueString()
if imageVersion == "" {
tflog.Info(ctx, fmt.Sprintf("No image version specified, using provider version %s", d.version))
imageVersion = d.version // Use provider version as default.
}
fetchedMeasurements, err := verifyFetcher.FetchAndVerifyMeasurements(ctx, imageVersion,
csp, attestationVariant, false)
if err != nil {
var rekErr *measurements.RekorError

6
terraform-provider-constellation/internal/provider/attestation_data_source_test.go
View File

@ -17,8 +17,8 @@ func TestAccAttestationSource(t *testing.T) {
bazelPreCheck := func() { bazelSetTerraformBinaryPath(t) }
testCases := map[string]resource.TestCase{
"aws sev-snp succcess": {
ProtoV6ProviderFactories: testAccProtoV6ProviderFactories,
"aws sev-snp succcess without explicit image_version": {
ProtoV6ProviderFactories: testAccProtoV6ProviderFactoriesWithVersion("v2.13.0"), // do this to test if a valid default version is set
PreCheck: bazelPreCheck,
Steps: []resource.TestStep{
{
@ -26,7 +26,6 @@ func TestAccAttestationSource(t *testing.T) {
data "constellation_attestation" "test" {
csp = "aws"
attestation_variant = "aws-sev-snp"
image_version = "v2.13.0"
}
`,
Check: resource.ComposeAggregateTestCheckFunc(
@ -54,6 +53,7 @@ func TestAccAttestationSource(t *testing.T) {
csp = "azure"
attestation_variant = "azure-sev-snp"
image_version = "v2.13.0"
maa_url = "https://www.example.com"
}
`,
Check: resource.ComposeAggregateTestCheckFunc(

60
terraform-provider-constellation/internal/provider/image_data_source.go
View File

@ -14,10 +14,12 @@ import (
"github.com/edgelesssys/constellation/v2/internal/attestation/variant"
"github.com/edgelesssys/constellation/v2/internal/cloud/cloudprovider"
"github.com/edgelesssys/constellation/v2/internal/imagefetcher"
"github.com/edgelesssys/constellation/v2/terraform-provider-constellation/internal/data"
"github.com/hashicorp/terraform-plugin-framework/datasource"
"github.com/hashicorp/terraform-plugin-framework/datasource/schema"
"github.com/hashicorp/terraform-plugin-framework/path"
"github.com/hashicorp/terraform-plugin-framework/types"
"github.com/hashicorp/terraform-plugin-log/tflog"
)
var (
@ -38,6 +40,7 @@ func NewImageDataSource() datasource.DataSource {
// It is used to retrieve the Constellation OS image reference for a given CSP and Attestation Variant.
type ImageDataSource struct {
imageFetcher imageFetcher
version string
}
// imageFetcher gets an image reference from the versionsapi.
@ -66,14 +69,14 @@ func (d *ImageDataSource) Metadata(_ context.Context, req datasource.MetadataReq
// Schema returns the schema for the image data source.
func (d *ImageDataSource) Schema(_ context.Context, _ datasource.SchemaRequest, resp *datasource.SchemaResponse) {
resp.Schema = schema.Schema{
Description: "Data source to retrieve the Constellation OS image reference for a given CSP and Attestation Variant.",
MarkdownDescription: "Data source to retrieve the Constellation OS image reference for a given CSP and Attestation Variant.",
Description: "The data source to resolve the CSP-specific OS image reference for a given version and attestation variant.",
MarkdownDescription: "Data source to resolve the CSP-specific OS image reference for a given version and attestation variant.",
Attributes: map[string]schema.Attribute{
"attestation_variant": newAttestationVariantAttribute(attributeInput),
"image_version": schema.StringAttribute{
Description: "Version of the Constellation OS image to use. (e.g. `v2.13.0`)",
MarkdownDescription: "Version of the Constellation OS image to use. (e.g. `v2.13.0`)",
Required: true, // TODO(msanft): Make this optional to support "lockstep" mode.
Description: "Version of the Constellation OS image to use. (e.g. `v2.13.0`). If not set, the provider version is used.",
MarkdownDescription: "Version of the Constellation OS image to use. (e.g. `v2.13.0`). If not set, the provider version value is used.",
Optional: true,
},
"csp": newCSPAttribute(),
"marketplace_image": schema.BoolAttribute{
@ -97,13 +100,43 @@ func (d *ImageDataSource) Schema(_ context.Context, _ datasource.SchemaRequest,
}
}
// TODO(msanft): Possibly implement more complex validation for inter-dependencies between attributes.
// E.g., region should be required if, and only if, AWS is used.
// ValidateConfig validates the configuration for the image data source.
func (d *ImageDataSource) ValidateConfig(ctx context.Context, req datasource.ValidateConfigRequest, resp *datasource.ValidateConfigResponse) {
var data ImageDataSourceModel
resp.Diagnostics.Append(req.Config.Get(ctx, &data)...)
if resp.Diagnostics.HasError() {
return
}
if data.CSP.Equal(types.StringValue("aws")) && data.Region.IsNull() {
resp.Diagnostics.AddAttributeError(
path.Root("region"),
"Region must be set for AWS", "When csp is set to 'aws', 'region' must be specified.",
)
return
}
}
// Configure configures the data source.
func (d *ImageDataSource) Configure(_ context.Context, _ datasource.ConfigureRequest, _ *datasource.ConfigureResponse) {
// Create the image-fetcher client.
func (d *ImageDataSource) Configure(_ context.Context, req datasource.ConfigureRequest, resp *datasource.ConfigureResponse) {
d.imageFetcher = imagefetcher.New()
// Prevent panic if the provider has not been configured. is necessary!
if req.ProviderData == nil {
return
}
providerData, ok := req.ProviderData.(data.ProviderData)
if !ok {
resp.Diagnostics.AddError(
"Unexpected Data Source Configure Type",
fmt.Sprintf("Expected data.ProviderData, got: %T. Please report this issue to the provider developers.", req.ProviderData),
)
return
}
d.version = providerData.Version
}
// Read reads from the data source.
@ -111,7 +144,6 @@ func (d *ImageDataSource) Read(ctx context.Context, req datasource.ReadRequest,
// Retrieve the configuration values for this data source instance.
var data ImageDataSourceModel
resp.Diagnostics.Append(req.Config.Get(ctx, &data)...)
// Check configuration for errors.
csp := cloudprovider.FromString(data.CSP.ValueString())
if csp == cloudprovider.Unknown {
@ -135,9 +167,15 @@ func (d *ImageDataSource) Read(ctx context.Context, req datasource.ReadRequest,
return
}
imageVersion := data.ImageVersion.ValueString()
if imageVersion == "" {
tflog.Info(ctx, fmt.Sprintf("No image version specified, using provider version %s", d.version))
imageVersion = d.version // Use provider version as default.
}
// Retrieve Image Reference
imageRef, err := d.imageFetcher.FetchReference(ctx, csp, attestationVariant,
data.ImageVersion.ValueString(), data.Region.ValueString(), data.MarketplaceImage.ValueBool())
imageVersion, data.Region.ValueString(), data.MarketplaceImage.ValueBool())
if err != nil {
resp.Diagnostics.AddError(
"Error fetching Image Reference",

33
terraform-provider-constellation/internal/provider/image_data_source_test.go
View File

@ -18,6 +18,22 @@ func TestAccImageDataSource(t *testing.T) {
bazelPreCheck := func() { bazelSetTerraformBinaryPath(t) }
testCases := map[string]resource.TestCase{
"no image_version succeeds": {
ProtoV6ProviderFactories: testAccProtoV6ProviderFactoriesWithVersion("v2.13.0"),
PreCheck: bazelPreCheck,
Steps: []resource.TestStep{
{
Config: testingConfig + `
data "constellation_image" "test" {
attestation_variant = "aws-sev-snp"
csp = "aws"
region = "eu-west-1"
}
`,
Check: resource.TestCheckResourceAttrSet("data.constellation_image.test", "reference"),
},
},
},
"aws succcess": {
ProtoV6ProviderFactories: testAccProtoV6ProviderFactories,
PreCheck: bazelPreCheck,
@ -37,6 +53,23 @@ func TestAccImageDataSource(t *testing.T) {
},
},
},
"aws without region fails": {
ProtoV6ProviderFactories: testAccProtoV6ProviderFactories,
PreCheck: bazelPreCheck,
Steps: []resource.TestStep{
// Read testing
{
Config: testingConfig + `
data "constellation_image" "test" {
image_version = "v2.13.0"
attestation_variant = "aws-sev-snp"
csp = "aws"
}
`,
ExpectError: regexp.MustCompile(".*Region must be set for AWS.*"),
},
},
},
"azure success": {
ProtoV6ProviderFactories: testAccProtoV6ProviderFactories,
PreCheck: bazelPreCheck,

17
terraform-provider-constellation/internal/provider/provider.go
View File

@ -27,9 +27,7 @@ type ConstellationProviderModel struct{}
// ConstellationProvider is the provider implementation.
type ConstellationProvider struct {
// version is set to the provider version on release, "dev" when the
// provider is built and ran locally, and "test" when running acceptance
// testing.
// version is set to the provider version on release, and the pseudo version on local builds. The pseudo version is not a valid default for the image_version attribute.
version string
}
@ -51,8 +49,12 @@ func (p *ConstellationProvider) Metadata(_ context.Context, _ provider.MetadataR
// Schema defines the HCL schema of the provider, i.e. what attributes it has and what they are used for.
func (p *ConstellationProvider) Schema(_ context.Context, _ provider.SchemaRequest, resp *provider.SchemaResponse) {
resp.Schema = schema.Schema{
Description: "The Constellation provider manages Constellation clusters.",
MarkdownDescription: `The Constellation provider manages Constellation clusters.`, // TODO(msanft): Provide a more sophisticated description.
Description: "The Constellation provider manages Constellation clusters.",
MarkdownDescription: `The Constellation provider manages Constellation clusters.
Given user-defined infrastructure in Terraform, the provider with its main 'constellation_cluster' resource manages the entire lifecycle of a cluster.
The provider allows easy usage of custom infrastructure setups and GitOps workflows.
It is released as part of Constellation releases, such that each provider version is compatible with the corresponding Constellation version.`,
}
}
@ -67,8 +69,9 @@ func (p *ConstellationProvider) Configure(ctx context.Context, req provider.Conf
return
}
// TODO(msanft): Initialize persistent clients here.
config := datastruct.ProviderData{}
config := datastruct.ProviderData{
Version: p.version,
}
// Make the clients available during data source and resource "Configure" methods.
resp.DataSourceData = config

13
terraform-provider-constellation/internal/provider/provider_test.go
View File

@ -12,6 +12,7 @@ import (
"testing"
"github.com/bazelbuild/rules_go/go/runfiles"
"github.com/edgelesssys/constellation/v2/internal/constants"
"github.com/hashicorp/terraform-plugin-framework/providerserver"
"github.com/hashicorp/terraform-plugin-go/tfprotov6"
)
@ -26,9 +27,17 @@ const (
// testAccProtoV6ProviderFactories are used to instantiate a provider during
// acceptance testing. The factory function will be invoked for every Terraform
// CLI command executed to create a provider server to which the CLI can
// reattach. It sets a pseudo version for the provider version.
var testAccProtoV6ProviderFactories = testAccProtoV6ProviderFactoriesWithVersion(constants.BinaryVersion().String())
// testAccProtoV6ProviderFactoriesWithVersion are used to instantiate a provider during
// acceptance testing. The factory function will be invoked for every Terraform
// CLI command executed to create a provider server to which the CLI can
// reattach.
var testAccProtoV6ProviderFactories = map[string]func() (tfprotov6.ProviderServer, error){
"constellation": providerserver.NewProtocol6WithError(New("test")()),
var testAccProtoV6ProviderFactoriesWithVersion = func(version string) map[string]func() (tfprotov6.ProviderServer, error) {
return map[string]func() (tfprotov6.ProviderServer, error){
"constellation": providerserver.NewProtocol6WithError(New(version)()),
}
}
// bazelSetTerraformBinaryPath sets the path to the Terraform binary for

8
terraform-provider-constellation/internal/provider/shared_attributes.go
View File

@ -73,11 +73,15 @@ func newMeasurementsAttribute(t attributeType) schema.Attribute {
func newAttestationConfigAttribute(t attributeType) schema.Attribute {
isInput := bool(t)
var additionalDescription string
if isInput {
additionalDescription = " The output of the [constellation_attestation](../data-sources/attestation.md) data source provides sensible defaults. "
}
return schema.SingleNestedAttribute{
Computed: !isInput,
Required: isInput,
MarkdownDescription: "Attestation comprises the measurements and SEV-SNP specific parameters.",
Description: "The values provide sensible defaults. See the docs for advanced usage.", // TODO(elchead): AB#3568
MarkdownDescription: "Attestation comprises the measurements and SEV-SNP specific parameters." + additionalDescription,
Description: "Attestation comprises the measurements and SEV-SNP specific parameters." + additionalDescription,
Attributes: map[string]schema.Attribute{
"variant": newAttestationVariantAttribute(t), // duplicated for convenience in cluster resource
"bootloader_version": schema.Int64Attribute{

2
terraform-provider-constellation/main.go
View File

@ -15,7 +15,7 @@ import (
"github.com/hashicorp/terraform-plugin-framework/providerserver"
)
// TODO(msanft): Set this accordingly in the release CI.
// version is the version of Constellation to use. Left as a separate variable to allow override during build.
var version = "dev"
func main() {

48
terraform/legacy-module/common/fetch-image/.terraform.lock.hcl
View File

@ -1,48 +0,0 @@
# This file is maintained automatically by "terraform init".
# Manual edits may be lost in future updates.
provider "registry.terraform.io/hashicorp/local" {
version = "2.4.0"
hashes = [
"h1:7RnIbO3CFakblTJs7o0mUiY44dc9xGYsLhSNFSNS1Ds=",
"h1:Bs7LAkV/iQTLv72j+cTMrvx2U3KyXrcVHaGbdns1NcE=",
"h1:OMqURhlP2IgLEgUCzSlaKXyb/IbnKForgDSjZF/NY4Y=",
"h1:R97FTYETo88sT2VHfMgkPU3lzCsZLunPftjSI5vfKe8=",
"h1:ZUEYUmm2t4vxwzxy1BvN1wL6SDWrDxfH7pxtzX8c6d0=",
"zh:53604cd29cb92538668fe09565c739358dc53ca56f9f11312b9d7de81e48fab9",
"zh:66a46e9c508716a1c98efbf793092f03d50049fa4a83cd6b2251e9a06aca2acf",
"zh:70a6f6a852dd83768d0778ce9817d81d4b3f073fab8fa570bff92dcb0824f732",
"zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3",
"zh:82a803f2f484c8b766e2e9c32343e9c89b91997b9f8d2697f9f3837f62926b35",
"zh:9708a4e40d6cc4b8afd1352e5186e6e1502f6ae599867c120967aebe9d90ed04",
"zh:973f65ce0d67c585f4ec250c1e634c9b22d9c4288b484ee2a871d7fa1e317406",
"zh:c8fa0f98f9316e4cfef082aa9b785ba16e36ff754d6aba8b456dab9500e671c6",
"zh:cfa5342a5f5188b20db246c73ac823918c189468e1382cb3c48a9c0c08fc5bf7",
"zh:e0e2b477c7e899c63b06b38cd8684a893d834d6d0b5e9b033cedc06dd7ffe9e2",
"zh:f62d7d05ea1ee566f732505200ab38d94315a4add27947a60afa29860822d3fc",
"zh:fa7ce69dde358e172bd719014ad637634bbdabc49363104f4fca759b4b73f2ce",
]
}
provider "registry.terraform.io/hashicorp/null" {
version = "3.2.1"
hashes = [
"h1:FbGfc+muBsC17Ohy5g806iuI1hQc4SIexpYCrQHQd8w=",
"h1:tSj1mL6OQ8ILGqR2mDu7OYYYWf+hoir0pf9KAQ8IzO8=",
"h1:vUW21lLLsKlxtBf0QF7LKJreKxs0CM7YXGzqW1N/ODY=",
"h1:wqgRvlyVIbkCeCQs+5jj6zVuQL0KDxZZtNofGqqlSdI=",
"h1:ydA0/SNRVB1o95btfshvYsmxA+jZFRZcvKzZSB+4S1M=",
"zh:58ed64389620cc7b82f01332e27723856422820cfd302e304b5f6c3436fb9840",
"zh:62a5cc82c3b2ddef7ef3a6f2fedb7b9b3deff4ab7b414938b08e51d6e8be87cb",
"zh:63cff4de03af983175a7e37e52d4bd89d990be256b16b5c7f919aff5ad485aa5",
"zh:74cb22c6700e48486b7cabefa10b33b801dfcab56f1a6ac9b6624531f3d36ea3",
"zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3",
"zh:79e553aff77f1cfa9012a2218b8238dd672ea5e1b2924775ac9ac24d2a75c238",
"zh:a1e06ddda0b5ac48f7e7c7d59e1ab5a4073bbcf876c73c0299e4610ed53859dc",
"zh:c37a97090f1a82222925d45d84483b2aa702ef7ab66532af6cbcfb567818b970",
"zh:e4453fbebf90c53ca3323a92e7ca0f9961427d2f0ce0d2b65523cc04d5d999c2",
"zh:e80a746921946d8b6761e77305b752ad188da60688cfd2059322875d363be5f5",
"zh:fbdb892d9822ed0e4cb60f2fedbdbb556e4da0d88d3b942ae963ed6ff091e48f",
"zh:fca01a623d90d0cad0843102f9b8b9fe0d3ff8244593bd817f126582b52dd694",
]
}

25
terraform/legacy-module/constellation-cluster/.terraform.lock.hcl
View File

@ -1,25 +0,0 @@
# This file is maintained automatically by "terraform init".
# Manual edits may be lost in future updates.
provider "registry.terraform.io/hashicorp/null" {
version = "3.2.1"
hashes = [
"h1:FbGfc+muBsC17Ohy5g806iuI1hQc4SIexpYCrQHQd8w=",
"h1:tSj1mL6OQ8ILGqR2mDu7OYYYWf+hoir0pf9KAQ8IzO8=",
"h1:vUW21lLLsKlxtBf0QF7LKJreKxs0CM7YXGzqW1N/ODY=",
"h1:wqgRvlyVIbkCeCQs+5jj6zVuQL0KDxZZtNofGqqlSdI=",
"h1:ydA0/SNRVB1o95btfshvYsmxA+jZFRZcvKzZSB+4S1M=",
"zh:58ed64389620cc7b82f01332e27723856422820cfd302e304b5f6c3436fb9840",
"zh:62a5cc82c3b2ddef7ef3a6f2fedb7b9b3deff4ab7b414938b08e51d6e8be87cb",
"zh:63cff4de03af983175a7e37e52d4bd89d990be256b16b5c7f919aff5ad485aa5",
"zh:74cb22c6700e48486b7cabefa10b33b801dfcab56f1a6ac9b6624531f3d36ea3",
"zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3",
"zh:79e553aff77f1cfa9012a2218b8238dd672ea5e1b2924775ac9ac24d2a75c238",
"zh:a1e06ddda0b5ac48f7e7c7d59e1ab5a4073bbcf876c73c0299e4610ed53859dc",
"zh:c37a97090f1a82222925d45d84483b2aa702ef7ab66532af6cbcfb567818b970",
"zh:e4453fbebf90c53ca3323a92e7ca0f9961427d2f0ce0d2b65523cc04d5d999c2",
"zh:e80a746921946d8b6761e77305b752ad188da60688cfd2059322875d363be5f5",
"zh:fbdb892d9822ed0e4cb60f2fedbdbb556e4da0d88d3b942ae963ed6ff091e48f",
"zh:fca01a623d90d0cad0843102f9b8b9fe0d3ff8244593bd817f126582b52dd694",
]
}