mirror of
https://github.com/edgelesssys/constellation.git
synced 2024-12-24 06:59:40 -05:00
ci: add e2e test for self-managed infrastructure (#2472)
* add self-managed infra e2e test * self-managed terminatio Signed-off-by: Moritz Sanft <58110325+msanft@users.noreply.github.com> * fix upgrade test Signed-off-by: Moritz Sanft <58110325+msanft@users.noreply.github.com> * fix indentation Signed-off-by: Moritz Sanft <58110325+msanft@users.noreply.github.com> * use -r when copying dir Signed-off-by: Moritz Sanft <58110325+msanft@users.noreply.github.com> * add terraform variable parsing * copy constellation conf Signed-off-by: Moritz Sanft <58110325+msanft@users.noreply.github.com> * remove unnecessary line breaks * add missing value Signed-off-by: Moritz Sanft <58110325+msanft@users.noreply.github.com> * add image fetching for CSP Signed-off-by: Moritz Sanft <58110325+msanft@users.noreply.github.com> * fix quoting Signed-off-by: Moritz Sanft <58110325+msanft@users.noreply.github.com> * add missing input to internal lb test * normalize Azure URLs.. Of course * tidy Signed-off-by: Moritz Sanft <58110325+msanft@users.noreply.github.com> * fix expressions * initsecret to hex * update hexdump cmd * add build test Signed-off-by: Moritz Sanft <58110325+msanft@users.noreply.github.com> * add node / pod cidr outputs Signed-off-by: Moritz Sanft <58110325+msanft@users.noreply.github.com> * explicitly delete the state file Signed-off-by: Moritz Sanft <58110325+msanft@users.noreply.github.com> * add missing license header Signed-off-by: Moritz Sanft <58110325+msanft@users.noreply.github.com> * always write all outputs Signed-off-by: Moritz Sanft <58110325+msanft@users.noreply.github.com> * fix list output Signed-off-by: Moritz Sanft <58110325+msanft@users.noreply.github.com> * remove state-file and admin-conf on destroy * dont use test payload Signed-off-by: Moritz Sanft <58110325+msanft@users.noreply.github.com> * [remove] use self managed infra in manual e2e for testing Signed-off-by: Moritz Sanft <58110325+msanft@users.noreply.github.com> * init: always skip infrastructure phase * patch maa in workflow Signed-off-by: Moritz Sanft <58110325+msanft@users.noreply.github.com> * default to Constellation-created infra in e2e test --------- Signed-off-by: Moritz Sanft <58110325+msanft@users.noreply.github.com>
This commit is contained in:
parent
f4bfbe3564
commit
402a8834ca
18
.github/actions/constellation_create/action.yml
vendored
18
.github/actions/constellation_create/action.yml
vendored
@ -1,5 +1,5 @@
|
|||||||
name: Constellation create
|
name: Constellation create
|
||||||
description: Create a new Constellation cluster using latest OS image.
|
description: Create a new Constellation cluster using the latest OS image.
|
||||||
|
|
||||||
inputs:
|
inputs:
|
||||||
workerNodesCount:
|
workerNodesCount:
|
||||||
@ -50,6 +50,9 @@ inputs:
|
|||||||
internalLoadBalancer:
|
internalLoadBalancer:
|
||||||
description: "Whether to use an internal load balancer for the control plane"
|
description: "Whether to use an internal load balancer for the control plane"
|
||||||
required: false
|
required: false
|
||||||
|
selfManagedInfra:
|
||||||
|
description: "Use self-managed infrastructure instead of infrastructure created by the Constellation CLI."
|
||||||
|
required: true
|
||||||
|
|
||||||
outputs:
|
outputs:
|
||||||
kubeconfig:
|
kubeconfig:
|
||||||
@ -124,14 +127,25 @@ runs:
|
|||||||
run: |
|
run: |
|
||||||
yq eval -i '(.internalLoadBalancer) = true' constellation-conf.yaml
|
yq eval -i '(.internalLoadBalancer) = true' constellation-conf.yaml
|
||||||
|
|
||||||
- name: Constellation create
|
- name: Show Cluster Configuration
|
||||||
shell: bash
|
shell: bash
|
||||||
run: |
|
run: |
|
||||||
echo "Creating cluster using config:"
|
echo "Creating cluster using config:"
|
||||||
cat constellation-conf.yaml
|
cat constellation-conf.yaml
|
||||||
sudo sh -c 'echo "127.0.0.1 license.confidential.cloud" >> /etc/hosts' || true
|
sudo sh -c 'echo "127.0.0.1 license.confidential.cloud" >> /etc/hosts' || true
|
||||||
|
|
||||||
|
- name: Constellation create (CLI)
|
||||||
|
if : inputs.selfManagedInfra != 'true'
|
||||||
|
shell: bash
|
||||||
|
run: |
|
||||||
constellation create -y --debug --tf-log=DEBUG
|
constellation create -y --debug --tf-log=DEBUG
|
||||||
|
|
||||||
|
- name: Constellation create (self-managed)
|
||||||
|
if : inputs.selfManagedInfra == 'true'
|
||||||
|
uses: ./.github/actions/self_managed_create
|
||||||
|
with:
|
||||||
|
cloudProvider: ${{ inputs.cloudProvider }}
|
||||||
|
|
||||||
- name: Cdbg deploy
|
- name: Cdbg deploy
|
||||||
if: inputs.isDebugImage == 'true'
|
if: inputs.isDebugImage == 'true'
|
||||||
uses: ./.github/actions/cdbg_deploy
|
uses: ./.github/actions/cdbg_deploy
|
||||||
|
15
.github/actions/constellation_destroy/action.yml
vendored
15
.github/actions/constellation_destroy/action.yml
vendored
@ -5,6 +5,9 @@ inputs:
|
|||||||
kubeconfig:
|
kubeconfig:
|
||||||
description: "The kubeconfig for the cluster."
|
description: "The kubeconfig for the cluster."
|
||||||
required: true
|
required: true
|
||||||
|
selfManagedInfra:
|
||||||
|
description: "Use self-managed infrastructure instead of infrastructure created by the Constellation CLI."
|
||||||
|
required: true
|
||||||
|
|
||||||
runs:
|
runs:
|
||||||
using: "composite"
|
using: "composite"
|
||||||
@ -39,6 +42,18 @@ runs:
|
|||||||
echo "::endgroup::"
|
echo "::endgroup::"
|
||||||
|
|
||||||
- name: Constellation terminate
|
- name: Constellation terminate
|
||||||
|
if: inputs.selfManagedInfra != 'true'
|
||||||
shell: bash
|
shell: bash
|
||||||
run: |
|
run: |
|
||||||
constellation terminate --yes --tf-log=DEBUG
|
constellation terminate --yes --tf-log=DEBUG
|
||||||
|
|
||||||
|
- name: Constellation terminate (self-managed)
|
||||||
|
if: inputs.selfManagedInfra == 'true'
|
||||||
|
shell: bash
|
||||||
|
working-directory: ${{ github.workspace }}/e2e-infra
|
||||||
|
run: |
|
||||||
|
terraform init
|
||||||
|
terraform destroy -auto-approve
|
||||||
|
# Explicitly delete the state file
|
||||||
|
rm ${{ github.workspace }}/constellation-state.yaml
|
||||||
|
rm ${{ github.workspace }}/constellation-admin.conf
|
||||||
|
5
.github/actions/e2e_test/action.yml
vendored
5
.github/actions/e2e_test/action.yml
vendored
@ -76,6 +76,9 @@ inputs:
|
|||||||
description: "Enable security policy for the cluster."
|
description: "Enable security policy for the cluster."
|
||||||
internalLoadBalancer:
|
internalLoadBalancer:
|
||||||
description: "Enable internal load balancer for the cluster."
|
description: "Enable internal load balancer for the cluster."
|
||||||
|
selfManagedInfra:
|
||||||
|
description: "Use self-managed infrastructure instead of infrastructure created by the Constellation CLI."
|
||||||
|
default: "false"
|
||||||
|
|
||||||
outputs:
|
outputs:
|
||||||
kubeconfig:
|
kubeconfig:
|
||||||
@ -260,6 +263,8 @@ runs:
|
|||||||
kubernetesVersion: ${{ inputs.kubernetesVersion }}
|
kubernetesVersion: ${{ inputs.kubernetesVersion }}
|
||||||
refStream: ${{ inputs.refStream }}
|
refStream: ${{ inputs.refStream }}
|
||||||
internalLoadBalancer: ${{ inputs.internalLoadBalancer }}
|
internalLoadBalancer: ${{ inputs.internalLoadBalancer }}
|
||||||
|
test: ${{ inputs.test }}
|
||||||
|
selfManagedInfra: ${{ inputs.selfManagedInfra }}
|
||||||
|
|
||||||
- name: Deploy log- and metrics-collection (Kubernetes)
|
- name: Deploy log- and metrics-collection (Kubernetes)
|
||||||
id: deploy-logcollection
|
id: deploy-logcollection
|
||||||
|
111
.github/actions/self_managed_create/action.yml
vendored
Normal file
111
.github/actions/self_managed_create/action.yml
vendored
Normal file
@ -0,0 +1,111 @@
|
|||||||
|
name: Self-managed infrastructure creation
|
||||||
|
description: "Create the required infrastructure for a Constellation cluster manually."
|
||||||
|
|
||||||
|
inputs:
|
||||||
|
cloudProvider:
|
||||||
|
description: "The cloud provider the test runs on."
|
||||||
|
required: true
|
||||||
|
|
||||||
|
runs:
|
||||||
|
using: "composite"
|
||||||
|
steps:
|
||||||
|
- name: Copy Terraform configuration and Constellation config
|
||||||
|
shell: bash
|
||||||
|
working-directory:
|
||||||
|
run: |
|
||||||
|
cp -r ${{ github.workspace }}/cli/internal/terraform/terraform/${{ inputs.cloudProvider }} ${{ github.workspace }}/e2e-infra
|
||||||
|
cp ${{ github.workspace }}/constellation-conf.yaml ${{ github.workspace }}/e2e-infra
|
||||||
|
|
||||||
|
- name: Get CSP image reference
|
||||||
|
id: get_image
|
||||||
|
shell: bash
|
||||||
|
working-directory: ${{ github.workspace }}/e2e-infra
|
||||||
|
run: |
|
||||||
|
echo "image_ref=$(bazel run //hack/image-fetch:image-fetch)" >> $GITHUB_OUTPUT
|
||||||
|
|
||||||
|
- name: Write Terraform variables
|
||||||
|
shell: bash
|
||||||
|
working-directory: ${{ github.workspace }}/e2e-infra
|
||||||
|
run: |
|
||||||
|
echo "name = \"$(yq '.name' constellation-conf.yaml)\"" >> terraform.tfvars
|
||||||
|
echo "debug = $(yq '.debugCluster' constellation-conf.yaml)" >> terraform.tfvars
|
||||||
|
echo "custom_endpoint = \"$(yq '.customEndpoint' constellation-conf.yaml)\"" >> terraform.tfvars
|
||||||
|
echo "image_id = \"${{ steps.get_image.outputs.image_ref }}\"" >> terraform.tfvars
|
||||||
|
echo "node_groups = {
|
||||||
|
control_plane_default = {
|
||||||
|
role = \"$(yq '.nodeGroups.control_plane_default.role' constellation-conf.yaml)\"
|
||||||
|
zone = \"$(yq '.nodeGroups.control_plane_default.zone' constellation-conf.yaml)\"
|
||||||
|
instance_type = \"$(yq '.nodeGroups.control_plane_default.instanceType' constellation-conf.yaml)\"
|
||||||
|
disk_size = \"$(yq '.nodeGroups.control_plane_default.stateDiskSizeGB' constellation-conf.yaml)\"
|
||||||
|
disk_type = \"$(yq '.nodeGroups.control_plane_default.stateDiskType' constellation-conf.yaml)\"
|
||||||
|
initial_count = \"$(yq '.nodeGroups.control_plane_default.initialCount' constellation-conf.yaml)\"
|
||||||
|
}
|
||||||
|
worker_default = {
|
||||||
|
role = \"$(yq '.nodeGroups.worker_default.role' constellation-conf.yaml)\"
|
||||||
|
zone = \"$(yq '.nodeGroups.worker_default.zone' constellation-conf.yaml)\"
|
||||||
|
instance_type = \"$(yq '.nodeGroups.worker_default.instanceType' constellation-conf.yaml)\"
|
||||||
|
disk_size = \"$(yq '.nodeGroups.worker_default.stateDiskSizeGB' constellation-conf.yaml)\"
|
||||||
|
disk_type = \"$(yq '.nodeGroups.worker_default.stateDiskType' constellation-conf.yaml)\"
|
||||||
|
initial_count = \"$(yq '.nodeGroups.worker_default.initialCount' constellation-conf.yaml)\"
|
||||||
|
}
|
||||||
|
}" >> terraform.tfvars
|
||||||
|
if [[ "${{ inputs.cloudProvider }}" == 'aws' ]]; then
|
||||||
|
echo "iam_instance_profile_control_plane = \"$(yq '.provider.aws.iamProfileControlPlane' constellation-conf.yaml)\"" >> terraform.tfvars
|
||||||
|
echo "iam_instance_profile_worker_nodes = \"$(yq '.provider.aws.iamProfileWorkerNodes' constellation-conf.yaml)\"" >> terraform.tfvars
|
||||||
|
echo "region = \"$(yq '.provider.aws.region' constellation-conf.yaml)\"" >> terraform.tfvars
|
||||||
|
echo "zone = \"$(yq '.provider.aws.zone' constellation-conf.yaml)\"" >> terraform.tfvars
|
||||||
|
echo "ami = \"${{ steps.get_image.outputs.image_ref }}\"" >> terraform.tfvars
|
||||||
|
echo "enable_snp = $(yq '.attestation | has("awsSEVSNP")' constellation-conf.yaml)" >> terraform.tfvars
|
||||||
|
elif [[ "${{ inputs.cloudProvider }}" == 'azure' ]]; then
|
||||||
|
echo "location = \"$(yq '.provider.azure.location' constellation-conf.yaml)\"" >> terraform.tfvars
|
||||||
|
echo "create_maa = $(yq '.attestation | has("azureSEVSNP")' constellation-conf.yaml)" >> terraform.tfvars
|
||||||
|
echo "confidential_vm = $(yq '.attestation | has("azureSEVSNP")' constellation-conf.yaml)" >> terraform.tfvars
|
||||||
|
echo "secure_boot = $(yq '.provider.azure.secureBoot' constellation-conf.yaml)" >> terraform.tfvars
|
||||||
|
echo "resource_group = \"$(yq '.provider.azure.resourceGroup' constellation-conf.yaml)\"" >> terraform.tfvars
|
||||||
|
echo "user_assigned_identity = \"$(yq '.provider.azure.userAssignedIdentity' constellation-conf.yaml)\"" >> terraform.tfvars
|
||||||
|
elif [[ "${{ inputs.cloudProvider }}" == 'gcp' ]]; then
|
||||||
|
echo "project = \"$(yq '.provider.gcp.project' constellation-conf.yaml)\"" >> terraform.tfvars
|
||||||
|
echo "region = \"$(yq '.provider.gcp.region' constellation-conf.yaml)\"" >> terraform.tfvars
|
||||||
|
echo "zone = \"$(yq '.provider.gcp.zone' constellation-conf.yaml)\"" >> terraform.tfvars
|
||||||
|
fi
|
||||||
|
terraform fmt terraform.tfvars
|
||||||
|
echo "Using Terraform variables:"
|
||||||
|
cat terraform.tfvars
|
||||||
|
|
||||||
|
- name: Apply Terraform configuration
|
||||||
|
shell: bash
|
||||||
|
working-directory: ${{ github.workspace }}/e2e-infra
|
||||||
|
run: |
|
||||||
|
terraform init
|
||||||
|
terraform apply -auto-approve
|
||||||
|
|
||||||
|
- name: Patch MAA Policy
|
||||||
|
shell: bash
|
||||||
|
working-directory: ${{ github.workspace }}/e2e-infra
|
||||||
|
if: ${{ inputs.cloudProvider }} == 'azure'
|
||||||
|
run: |
|
||||||
|
bazel run //hack/maa-patch:maa-patch $(terraform output attestationURL | jq -r)
|
||||||
|
|
||||||
|
- name: Write outputs to state file
|
||||||
|
shell: bash
|
||||||
|
working-directory: ${{ github.workspace }}/e2e-infra
|
||||||
|
run: |
|
||||||
|
yq eval '.version ="v1"' --inplace ${{ github.workspace }}/constellation-state.yaml
|
||||||
|
yq eval ".infrastructure.initSecret =\"$(terraform output initSecret | jq -r | tr -d '\n' | hexdump -ve '/1 "%02x"' && echo '')\"" --inplace ${{ github.workspace }}/constellation-state.yaml
|
||||||
|
yq eval ".infrastructure.clusterEndpoint =\"$(terraform output out_of_cluster_endpoint | jq -r)\"" --inplace ${{ github.workspace }}/constellation-state.yaml
|
||||||
|
yq eval ".infrastructure.inClusterEndpoint =\"$(terraform output in_cluster_endpoint | jq -r)\"" --inplace ${{ github.workspace }}/constellation-state.yaml
|
||||||
|
yq eval ".infrastructure.ipCidrNode =\"$(terraform output ip_cidr_nodes | jq -r)\"" --inplace ${{ github.workspace }}/constellation-state.yaml
|
||||||
|
yq eval ".infrastructure.uid =\"$(terraform output uid | jq -r)\"" --inplace ${{ github.workspace }}/constellation-state.yaml
|
||||||
|
yq eval ".infrastructure.name =\"$(terraform output name | jq -r)\"" --inplace ${{ github.workspace }}/constellation-state.yaml
|
||||||
|
yq eval ".infrastructure.apiServerCertSANs =$(terraform output -json api_server_cert_sans)" --inplace ${{ github.workspace }}/constellation-state.yaml
|
||||||
|
if [[ "${{ inputs.cloudProvider }}" == 'azure' ]]; then
|
||||||
|
yq eval ".infrastructure.azure.resourceGroup =\"$(terraform output resource_group | jq -r)\"" --inplace ${{ github.workspace }}/constellation-state.yaml
|
||||||
|
yq eval ".infrastructure.azure.subscriptionID =\"$(terraform output subscription_id | jq -r)\"" --inplace ${{ github.workspace }}/constellation-state.yaml
|
||||||
|
yq eval ".infrastructure.azure.networkSecurityGroupName =\"$(terraform output network_security_group_name | jq -r)\"" --inplace ${{ github.workspace }}/constellation-state.yaml
|
||||||
|
yq eval ".infrastructure.azure.loadBalancerName =\"$(terraform output loadbalancer_name | jq -r)\"" --inplace ${{ github.workspace }}/constellation-state.yaml
|
||||||
|
yq eval ".infrastructure.azure.userAssignedIdentity =\"$(terraform output user_assigned_identity_client_id | jq -r)\"" --inplace ${{ github.workspace }}/constellation-state.yaml
|
||||||
|
yq eval ".infrastructure.azure.attestationURL =\"$(terraform output attestationURL | jq -r)\"" --inplace ${{ github.workspace }}/constellation-state.yaml
|
||||||
|
elif [[ "${{ inputs.cloudProvider }}" == 'gcp' ]]; then
|
||||||
|
yq eval ".infrastructure.gcp.projectID =\"$(terraform output project | jq -r)\"" --inplace ${{ github.workspace }}/constellation-state.yaml
|
||||||
|
yq eval ".infrastructure.gcp.ipCidrPod =\"$(terraform output ip_cidr_pods | jq -r)\"" --inplace ${{ github.workspace }}/constellation-state.yaml
|
||||||
|
fi
|
2
.github/workflows/e2e-test-daily.yml
vendored
2
.github/workflows/e2e-test-daily.yml
vendored
@ -91,12 +91,14 @@ jobs:
|
|||||||
awsOpenSearchDomain: ${{ secrets.AWS_OPENSEARCH_DOMAIN }}
|
awsOpenSearchDomain: ${{ secrets.AWS_OPENSEARCH_DOMAIN }}
|
||||||
awsOpenSearchUsers: ${{ secrets.AWS_OPENSEARCH_USER }}
|
awsOpenSearchUsers: ${{ secrets.AWS_OPENSEARCH_USER }}
|
||||||
awsOpenSearchPwd: ${{ secrets.AWS_OPENSEARCH_PWD }}
|
awsOpenSearchPwd: ${{ secrets.AWS_OPENSEARCH_PWD }}
|
||||||
|
selfManagedInfra: "false"
|
||||||
|
|
||||||
- name: Always terminate cluster
|
- name: Always terminate cluster
|
||||||
if: always()
|
if: always()
|
||||||
uses: ./.github/actions/constellation_destroy
|
uses: ./.github/actions/constellation_destroy
|
||||||
with:
|
with:
|
||||||
kubeconfig: ${{ steps.e2e_test.outputs.kubeconfig }}
|
kubeconfig: ${{ steps.e2e_test.outputs.kubeconfig }}
|
||||||
|
selfManagedInfra: "false"
|
||||||
|
|
||||||
- name: Always delete IAM configuration
|
- name: Always delete IAM configuration
|
||||||
if: always()
|
if: always()
|
||||||
|
@ -205,12 +205,14 @@ jobs:
|
|||||||
cosignPrivateKey: ${{ secrets.COSIGN_PRIVATE_KEY }}
|
cosignPrivateKey: ${{ secrets.COSIGN_PRIVATE_KEY }}
|
||||||
fetchMeasurements: ${{ contains(needs.find-latest-image.outputs.image, '/stream/stable/') }}
|
fetchMeasurements: ${{ contains(needs.find-latest-image.outputs.image, '/stream/stable/') }}
|
||||||
internalLoadBalancer: true
|
internalLoadBalancer: true
|
||||||
|
selfManagedInfra: "false"
|
||||||
|
|
||||||
- name: Always terminate cluster
|
- name: Always terminate cluster
|
||||||
if: always()
|
if: always()
|
||||||
uses: ./.github/actions/constellation_destroy
|
uses: ./.github/actions/constellation_destroy
|
||||||
with:
|
with:
|
||||||
kubeconfig: ${{ steps.e2e_test.outputs.kubeconfig }}
|
kubeconfig: ${{ steps.e2e_test.outputs.kubeconfig }}
|
||||||
|
selfManagedInfra: "false"
|
||||||
|
|
||||||
- name: Always delete IAM configuration
|
- name: Always delete IAM configuration
|
||||||
if: always()
|
if: always()
|
||||||
|
2
.github/workflows/e2e-test-manual.yml
vendored
2
.github/workflows/e2e-test-manual.yml
vendored
@ -260,12 +260,14 @@ jobs:
|
|||||||
cosignPassword: ${{ secrets.COSIGN_PASSWORD }}
|
cosignPassword: ${{ secrets.COSIGN_PASSWORD }}
|
||||||
cosignPrivateKey: ${{ secrets.COSIGN_PRIVATE_KEY }}
|
cosignPrivateKey: ${{ secrets.COSIGN_PRIVATE_KEY }}
|
||||||
fetchMeasurements: ${{ contains(needs.find-latest-image.outputs.image, '/stream/stable/') }}
|
fetchMeasurements: ${{ contains(needs.find-latest-image.outputs.image, '/stream/stable/') }}
|
||||||
|
selfManagedInfra: "false"
|
||||||
|
|
||||||
- name: Always terminate cluster
|
- name: Always terminate cluster
|
||||||
if: always()
|
if: always()
|
||||||
uses: ./.github/actions/constellation_destroy
|
uses: ./.github/actions/constellation_destroy
|
||||||
with:
|
with:
|
||||||
kubeconfig: ${{ steps.e2e_test.outputs.kubeconfig }}
|
kubeconfig: ${{ steps.e2e_test.outputs.kubeconfig }}
|
||||||
|
selfManagedInfra: "false"
|
||||||
|
|
||||||
- name: Always delete IAM configuration
|
- name: Always delete IAM configuration
|
||||||
if: always()
|
if: always()
|
||||||
|
20
.github/workflows/e2e-test-release.yml
vendored
20
.github/workflows/e2e-test-release.yml
vendored
@ -151,6 +151,24 @@ jobs:
|
|||||||
kubernetes-version: "v1.28"
|
kubernetes-version: "v1.28"
|
||||||
runner: "ubuntu-22.04"
|
runner: "ubuntu-22.04"
|
||||||
|
|
||||||
|
# self-managed infra test on latest k8s version
|
||||||
|
# runs Sonobuoy full test
|
||||||
|
- test: "sonobuoy full"
|
||||||
|
provider: "gcp"
|
||||||
|
kubernetes-version: "v1.28"
|
||||||
|
runner: "ubuntu-22.04"
|
||||||
|
selfManagedInfra: "true"
|
||||||
|
- test: "sonobuoy full"
|
||||||
|
provider: "azure"
|
||||||
|
kubernetes-version: "v1.28"
|
||||||
|
runner: "ubuntu-22.04"
|
||||||
|
selfManagedInfra: "true"
|
||||||
|
- test: "sonobuoy full"
|
||||||
|
provider: "aws"
|
||||||
|
kubernetes-version: "v1.28"
|
||||||
|
runner: "ubuntu-22.04"
|
||||||
|
selfManagedInfra: "true"
|
||||||
|
|
||||||
#
|
#
|
||||||
# Tests on macOS runner
|
# Tests on macOS runner
|
||||||
#
|
#
|
||||||
@ -213,12 +231,14 @@ jobs:
|
|||||||
cosignPassword: ${{ secrets.COSIGN_PASSWORD }}
|
cosignPassword: ${{ secrets.COSIGN_PASSWORD }}
|
||||||
cosignPrivateKey: ${{ secrets.COSIGN_PRIVATE_KEY }}
|
cosignPrivateKey: ${{ secrets.COSIGN_PRIVATE_KEY }}
|
||||||
githubToken: ${{ secrets.GITHUB_TOKEN }}
|
githubToken: ${{ secrets.GITHUB_TOKEN }}
|
||||||
|
selfManagedInfra: ${{ matrix.selfManagedInfra == 'true' }}
|
||||||
|
|
||||||
- name: Always terminate cluster
|
- name: Always terminate cluster
|
||||||
if: always()
|
if: always()
|
||||||
uses: ./.github/actions/constellation_destroy
|
uses: ./.github/actions/constellation_destroy
|
||||||
with:
|
with:
|
||||||
kubeconfig: ${{ steps.e2e_test.outputs.kubeconfig }}
|
kubeconfig: ${{ steps.e2e_test.outputs.kubeconfig }}
|
||||||
|
selfManagedInfra: ${{ matrix.selfManagedInfra == 'true' }}
|
||||||
|
|
||||||
- name: Always delete IAM configuration
|
- name: Always delete IAM configuration
|
||||||
if: always()
|
if: always()
|
||||||
|
21
.github/workflows/e2e-test-weekly.yml
vendored
21
.github/workflows/e2e-test-weekly.yml
vendored
@ -171,6 +171,24 @@ jobs:
|
|||||||
provider: "aws"
|
provider: "aws"
|
||||||
kubernetes-version: "v1.28"
|
kubernetes-version: "v1.28"
|
||||||
|
|
||||||
|
# self-managed infra test on latest k8s version
|
||||||
|
# with Sonobuoy full
|
||||||
|
- test: "sonobuoy full"
|
||||||
|
refStream: "ref/main/stream/debug/?"
|
||||||
|
provider: "gcp"
|
||||||
|
kubernetes-version: "v1.28"
|
||||||
|
selfManagedInfra: "true"
|
||||||
|
- test: "sonobuoy full"
|
||||||
|
refStream: "ref/main/stream/debug/?"
|
||||||
|
provider: "azure"
|
||||||
|
kubernetes-version: "v1.28"
|
||||||
|
selfManagedInfra: "true"
|
||||||
|
- test: "sonobuoy full"
|
||||||
|
provider: "aws"
|
||||||
|
refStream: "ref/main/stream/debug/?"
|
||||||
|
kubernetes-version: "v1.28"
|
||||||
|
selfManagedInfra: "true"
|
||||||
|
|
||||||
#
|
#
|
||||||
# Tests on release-stable refStream
|
# Tests on release-stable refStream
|
||||||
#
|
#
|
||||||
@ -188,6 +206,7 @@ jobs:
|
|||||||
refStream: "ref/release/stream/stable/?"
|
refStream: "ref/release/stream/stable/?"
|
||||||
provider: "aws"
|
provider: "aws"
|
||||||
kubernetes-version: "v1.27"
|
kubernetes-version: "v1.27"
|
||||||
|
|
||||||
runs-on: ubuntu-22.04
|
runs-on: ubuntu-22.04
|
||||||
permissions:
|
permissions:
|
||||||
id-token: write
|
id-token: write
|
||||||
@ -231,12 +250,14 @@ jobs:
|
|||||||
cosignPrivateKey: ${{ secrets.COSIGN_PRIVATE_KEY }}
|
cosignPrivateKey: ${{ secrets.COSIGN_PRIVATE_KEY }}
|
||||||
fetchMeasurements: ${{ matrix.refStream != 'ref/release/stream/stable/?' }}
|
fetchMeasurements: ${{ matrix.refStream != 'ref/release/stream/stable/?' }}
|
||||||
azureSNPEnforcementPolicy: ${{ matrix.azureSNPEnforcementPolicy }}
|
azureSNPEnforcementPolicy: ${{ matrix.azureSNPEnforcementPolicy }}
|
||||||
|
selfManagedInfra: ${{ matrix.selfManagedInfra == 'true' }}
|
||||||
|
|
||||||
- name: Always terminate cluster
|
- name: Always terminate cluster
|
||||||
if: always()
|
if: always()
|
||||||
uses: ./.github/actions/constellation_destroy
|
uses: ./.github/actions/constellation_destroy
|
||||||
with:
|
with:
|
||||||
kubeconfig: ${{ steps.e2e_test.outputs.kubeconfig }}
|
kubeconfig: ${{ steps.e2e_test.outputs.kubeconfig }}
|
||||||
|
selfManagedInfra: ${{ matrix.selfManagedInfra == 'true' }}
|
||||||
|
|
||||||
- name: Always delete IAM configuration
|
- name: Always delete IAM configuration
|
||||||
if: always()
|
if: always()
|
||||||
|
2
.github/workflows/e2e-upgrade.yml
vendored
2
.github/workflows/e2e-upgrade.yml
vendored
@ -182,6 +182,7 @@ jobs:
|
|||||||
awsOpenSearchDomain: ${{ secrets.AWS_OPENSEARCH_DOMAIN }}
|
awsOpenSearchDomain: ${{ secrets.AWS_OPENSEARCH_DOMAIN }}
|
||||||
awsOpenSearchUsers: ${{ secrets.AWS_OPENSEARCH_USER }}
|
awsOpenSearchUsers: ${{ secrets.AWS_OPENSEARCH_USER }}
|
||||||
awsOpenSearchPwd: ${{ secrets.AWS_OPENSEARCH_PWD }}
|
awsOpenSearchPwd: ${{ secrets.AWS_OPENSEARCH_PWD }}
|
||||||
|
selfManagedInfra: "false"
|
||||||
|
|
||||||
- name: Build CLI
|
- name: Build CLI
|
||||||
uses: ./.github/actions/build_cli
|
uses: ./.github/actions/build_cli
|
||||||
@ -287,6 +288,7 @@ jobs:
|
|||||||
uses: ./.github/actions/constellation_destroy
|
uses: ./.github/actions/constellation_destroy
|
||||||
with:
|
with:
|
||||||
kubeconfig: ${{ steps.e2e_test.outputs.kubeconfig }}
|
kubeconfig: ${{ steps.e2e_test.outputs.kubeconfig }}
|
||||||
|
selfManagedInfra: "false"
|
||||||
|
|
||||||
- name: Always delete IAM configuration
|
- name: Always delete IAM configuration
|
||||||
if: always()
|
if: always()
|
||||||
|
@ -10,7 +10,6 @@ go_library(
|
|||||||
"create.go",
|
"create.go",
|
||||||
"iam.go",
|
"iam.go",
|
||||||
"iamupgrade.go",
|
"iamupgrade.go",
|
||||||
"patch.go",
|
|
||||||
"rollback.go",
|
"rollback.go",
|
||||||
"serviceaccount.go",
|
"serviceaccount.go",
|
||||||
"terminate.go",
|
"terminate.go",
|
||||||
@ -36,10 +35,8 @@ go_library(
|
|||||||
"//internal/constants",
|
"//internal/constants",
|
||||||
"//internal/file",
|
"//internal/file",
|
||||||
"//internal/imagefetcher",
|
"//internal/imagefetcher",
|
||||||
|
"//internal/maa",
|
||||||
"//internal/role",
|
"//internal/role",
|
||||||
"@com_github_azure_azure_sdk_for_go//profiles/latest/attestation/attestation",
|
|
||||||
"@com_github_azure_azure_sdk_for_go_sdk_azcore//policy",
|
|
||||||
"@com_github_azure_azure_sdk_for_go_sdk_azidentity//:azidentity",
|
|
||||||
"@com_github_spf13_cobra//:cobra",
|
"@com_github_spf13_cobra//:cobra",
|
||||||
],
|
],
|
||||||
)
|
)
|
||||||
@ -51,7 +48,6 @@ go_test(
|
|||||||
"clusterupgrade_test.go",
|
"clusterupgrade_test.go",
|
||||||
"create_test.go",
|
"create_test.go",
|
||||||
"iam_test.go",
|
"iam_test.go",
|
||||||
"patch_test.go",
|
|
||||||
"rollback_test.go",
|
"rollback_test.go",
|
||||||
"terminate_test.go",
|
"terminate_test.go",
|
||||||
"tfupgrade_test.go",
|
"tfupgrade_test.go",
|
||||||
|
@ -18,6 +18,7 @@ import (
|
|||||||
"github.com/edgelesssys/constellation/v2/internal/cloud/cloudprovider"
|
"github.com/edgelesssys/constellation/v2/internal/cloud/cloudprovider"
|
||||||
"github.com/edgelesssys/constellation/v2/internal/constants"
|
"github.com/edgelesssys/constellation/v2/internal/constants"
|
||||||
"github.com/edgelesssys/constellation/v2/internal/file"
|
"github.com/edgelesssys/constellation/v2/internal/file"
|
||||||
|
"github.com/edgelesssys/constellation/v2/internal/maa"
|
||||||
)
|
)
|
||||||
|
|
||||||
// ClusterUpgrader is responsible for performing Terraform migrations on cluster upgrades.
|
// ClusterUpgrader is responsible for performing Terraform migrations on cluster upgrades.
|
||||||
@ -43,7 +44,7 @@ func NewClusterUpgrader(ctx context.Context, existingWorkspace, upgradeWorkspace
|
|||||||
|
|
||||||
return &ClusterUpgrader{
|
return &ClusterUpgrader{
|
||||||
tf: tfClient,
|
tf: tfClient,
|
||||||
policyPatcher: NewAzurePolicyPatcher(),
|
policyPatcher: maa.NewAzurePolicyPatcher(),
|
||||||
fileHandler: fileHandler,
|
fileHandler: fileHandler,
|
||||||
existingWorkspace: existingWorkspace,
|
existingWorkspace: existingWorkspace,
|
||||||
upgradeWorkspace: upgradeWorkspace,
|
upgradeWorkspace: upgradeWorkspace,
|
||||||
|
@ -25,6 +25,7 @@ import (
|
|||||||
"github.com/edgelesssys/constellation/v2/internal/config"
|
"github.com/edgelesssys/constellation/v2/internal/config"
|
||||||
"github.com/edgelesssys/constellation/v2/internal/constants"
|
"github.com/edgelesssys/constellation/v2/internal/constants"
|
||||||
"github.com/edgelesssys/constellation/v2/internal/imagefetcher"
|
"github.com/edgelesssys/constellation/v2/internal/imagefetcher"
|
||||||
|
"github.com/edgelesssys/constellation/v2/internal/maa"
|
||||||
)
|
)
|
||||||
|
|
||||||
// Creator creates cloud resources.
|
// Creator creates cloud resources.
|
||||||
@ -51,7 +52,7 @@ func NewCreator(out io.Writer) *Creator {
|
|||||||
newRawDownloader: func() rawDownloader {
|
newRawDownloader: func() rawDownloader {
|
||||||
return imagefetcher.NewDownloader()
|
return imagefetcher.NewDownloader()
|
||||||
},
|
},
|
||||||
policyPatcher: NewAzurePolicyPatcher(),
|
policyPatcher: maa.NewAzurePolicyPatcher(),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -43,6 +43,8 @@ func NewInitCmd() *cobra.Command {
|
|||||||
RunE: func(cmd *cobra.Command, args []string) error {
|
RunE: func(cmd *cobra.Command, args []string) error {
|
||||||
// Define flags for apply backend that are not set by init
|
// Define flags for apply backend that are not set by init
|
||||||
cmd.Flags().Bool("yes", false, "")
|
cmd.Flags().Bool("yes", false, "")
|
||||||
|
// We always want to skip the infrastructure phase here, to be aligned with the
|
||||||
|
// functionality of the old init command.
|
||||||
cmd.Flags().StringSlice("skip-phases", []string{string(skipInfrastructurePhase)}, "")
|
cmd.Flags().StringSlice("skip-phases", []string{string(skipInfrastructurePhase)}, "")
|
||||||
cmd.Flags().Duration("timeout", time.Hour, "")
|
cmd.Flags().Duration("timeout", time.Hour, "")
|
||||||
return runApply(cmd, args)
|
return runApply(cmd, args)
|
||||||
|
30
hack/image-fetch/BUILD.bazel
Normal file
30
hack/image-fetch/BUILD.bazel
Normal file
@ -0,0 +1,30 @@
|
|||||||
|
load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library")
|
||||||
|
load("//bazel/go:go_test.bzl", "go_test")
|
||||||
|
|
||||||
|
go_library(
|
||||||
|
name = "image-fetch_lib",
|
||||||
|
srcs = ["main.go"],
|
||||||
|
importpath = "github.com/edgelesssys/constellation/v2/hack/image-fetch",
|
||||||
|
visibility = ["//visibility:private"],
|
||||||
|
deps = [
|
||||||
|
"//internal/api/attestationconfigapi",
|
||||||
|
"//internal/cloud/cloudprovider",
|
||||||
|
"//internal/config",
|
||||||
|
"//internal/constants",
|
||||||
|
"//internal/file",
|
||||||
|
"//internal/imagefetcher",
|
||||||
|
"@com_github_spf13_afero//:afero",
|
||||||
|
],
|
||||||
|
)
|
||||||
|
|
||||||
|
go_binary(
|
||||||
|
name = "image-fetch",
|
||||||
|
embed = [":image-fetch_lib"],
|
||||||
|
visibility = ["//visibility:public"],
|
||||||
|
)
|
||||||
|
|
||||||
|
go_test(
|
||||||
|
name = "image-fetch_test",
|
||||||
|
srcs = ["main_test.go"],
|
||||||
|
embed = [":image-fetch_lib"],
|
||||||
|
)
|
68
hack/image-fetch/main.go
Normal file
68
hack/image-fetch/main.go
Normal file
@ -0,0 +1,68 @@
|
|||||||
|
/*
|
||||||
|
Copyright (c) Edgeless Systems GmbH
|
||||||
|
|
||||||
|
SPDX-License-Identifier: AGPL-3.0-only
|
||||||
|
*/
|
||||||
|
|
||||||
|
/*
|
||||||
|
imagefetch retrieves a CSP image reference from a Constellation config in the CWD.
|
||||||
|
This is especially useful when using self-managed infrastructure, where the image
|
||||||
|
reference needs to be chosen by the user, which would usually happen manually.
|
||||||
|
*/
|
||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"regexp"
|
||||||
|
|
||||||
|
"github.com/edgelesssys/constellation/v2/internal/api/attestationconfigapi"
|
||||||
|
"github.com/edgelesssys/constellation/v2/internal/cloud/cloudprovider"
|
||||||
|
"github.com/edgelesssys/constellation/v2/internal/config"
|
||||||
|
"github.com/edgelesssys/constellation/v2/internal/constants"
|
||||||
|
"github.com/edgelesssys/constellation/v2/internal/file"
|
||||||
|
"github.com/edgelesssys/constellation/v2/internal/imagefetcher"
|
||||||
|
"github.com/spf13/afero"
|
||||||
|
)
|
||||||
|
|
||||||
|
var (
|
||||||
|
caseInsensitiveCommunityGalleriesRegexp = regexp.MustCompile(`(?i)\/communitygalleries\/`)
|
||||||
|
caseInsensitiveImagesRegExp = regexp.MustCompile(`(?i)\/images\/`)
|
||||||
|
caseInsensitiveVersionsRegExp = regexp.MustCompile(`(?i)\/versions\/`)
|
||||||
|
)
|
||||||
|
|
||||||
|
func main() {
|
||||||
|
cwd := os.Getenv("BUILD_WORKING_DIRECTORY") // set by Bazel, for bazel run compatibility
|
||||||
|
ctx := context.Background()
|
||||||
|
|
||||||
|
fh := file.NewHandler(afero.NewOsFs())
|
||||||
|
attFetcher := attestationconfigapi.NewFetcher()
|
||||||
|
conf, err := config.New(fh, filepath.Join(cwd, constants.ConfigFilename), attFetcher, true)
|
||||||
|
var configValidationErr *config.ValidationError
|
||||||
|
if errors.As(err, &configValidationErr) {
|
||||||
|
fmt.Println(configValidationErr.LongMessage())
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
panic(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
imgFetcher := imagefetcher.New()
|
||||||
|
provider := conf.GetProvider()
|
||||||
|
attestationVariant := conf.GetAttestationConfig().GetVariant()
|
||||||
|
region := conf.GetRegion()
|
||||||
|
image, err := imgFetcher.FetchReference(ctx, provider, attestationVariant, conf.Image, region)
|
||||||
|
if err != nil {
|
||||||
|
panic(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if provider == cloudprovider.Azure {
|
||||||
|
image = caseInsensitiveCommunityGalleriesRegexp.ReplaceAllString(image, "/communityGalleries/")
|
||||||
|
image = caseInsensitiveImagesRegExp.ReplaceAllString(image, "/images/")
|
||||||
|
image = caseInsensitiveVersionsRegExp.ReplaceAllString(image, "/versions/")
|
||||||
|
}
|
||||||
|
|
||||||
|
fmt.Println(image)
|
||||||
|
}
|
13
hack/image-fetch/main_test.go
Normal file
13
hack/image-fetch/main_test.go
Normal file
@ -0,0 +1,13 @@
|
|||||||
|
/*
|
||||||
|
Copyright (c) Edgeless Systems GmbH
|
||||||
|
|
||||||
|
SPDX-License-Identifier: AGPL-3.0-only
|
||||||
|
*/
|
||||||
|
|
||||||
|
package main
|
||||||
|
|
||||||
|
import "testing"
|
||||||
|
|
||||||
|
func TestNop(t *testing.T) {
|
||||||
|
t.Skip("This is a nop-test to catch build-time errors in this package.")
|
||||||
|
}
|
22
hack/maa-patch/BUILD.bazel
Normal file
22
hack/maa-patch/BUILD.bazel
Normal file
@ -0,0 +1,22 @@
|
|||||||
|
load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library")
|
||||||
|
load("//bazel/go:go_test.bzl", "go_test")
|
||||||
|
|
||||||
|
go_library(
|
||||||
|
name = "maa-patch_lib",
|
||||||
|
srcs = ["main.go"],
|
||||||
|
importpath = "github.com/edgelesssys/constellation/v2/hack/maa-patch",
|
||||||
|
visibility = ["//visibility:private"],
|
||||||
|
deps = ["//internal/maa"],
|
||||||
|
)
|
||||||
|
|
||||||
|
go_binary(
|
||||||
|
name = "maa-patch",
|
||||||
|
embed = [":maa-patch_lib"],
|
||||||
|
visibility = ["//visibility:public"],
|
||||||
|
)
|
||||||
|
|
||||||
|
go_test(
|
||||||
|
name = "maa-patch_test",
|
||||||
|
srcs = ["main_test.go"],
|
||||||
|
embed = [":maa-patch_lib"],
|
||||||
|
)
|
33
hack/maa-patch/main.go
Normal file
33
hack/maa-patch/main.go
Normal file
@ -0,0 +1,33 @@
|
|||||||
|
/*
|
||||||
|
Copyright (c) Edgeless Systems GmbH
|
||||||
|
|
||||||
|
SPDX-License-Identifier: AGPL-3.0-only
|
||||||
|
*/
|
||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"net/url"
|
||||||
|
"os"
|
||||||
|
|
||||||
|
"github.com/edgelesssys/constellation/v2/internal/maa"
|
||||||
|
)
|
||||||
|
|
||||||
|
func main() {
|
||||||
|
if len(os.Args) != 2 {
|
||||||
|
fmt.Fprintf(os.Stderr, "Usage: %s <attestation-url>\n", os.Args[0])
|
||||||
|
os.Exit(1)
|
||||||
|
}
|
||||||
|
|
||||||
|
attestationURL := os.Args[1]
|
||||||
|
if _, err := url.Parse(attestationURL); err != nil {
|
||||||
|
fmt.Fprintf(os.Stderr, "Invalid attestation URL: %s\n", err)
|
||||||
|
os.Exit(1)
|
||||||
|
}
|
||||||
|
|
||||||
|
p := maa.NewAzurePolicyPatcher()
|
||||||
|
if err := p.Patch(context.Background(), attestationURL); err != nil {
|
||||||
|
panic(err)
|
||||||
|
}
|
||||||
|
}
|
13
hack/maa-patch/main_test.go
Normal file
13
hack/maa-patch/main_test.go
Normal file
@ -0,0 +1,13 @@
|
|||||||
|
/*
|
||||||
|
Copyright (c) Edgeless Systems GmbH
|
||||||
|
|
||||||
|
SPDX-License-Identifier: AGPL-3.0-only
|
||||||
|
*/
|
||||||
|
|
||||||
|
package main
|
||||||
|
|
||||||
|
import "testing"
|
||||||
|
|
||||||
|
func TestNop(t *testing.T) {
|
||||||
|
t.Skip("This is a nop-test to catch build-time errors in this package.")
|
||||||
|
}
|
24
internal/maa/BUILD.bazel
Normal file
24
internal/maa/BUILD.bazel
Normal file
@ -0,0 +1,24 @@
|
|||||||
|
load("@io_bazel_rules_go//go:def.bzl", "go_library")
|
||||||
|
load("//bazel/go:go_test.bzl", "go_test")
|
||||||
|
|
||||||
|
go_library(
|
||||||
|
name = "maa",
|
||||||
|
srcs = [
|
||||||
|
"maa.go",
|
||||||
|
"patch.go",
|
||||||
|
],
|
||||||
|
importpath = "github.com/edgelesssys/constellation/v2/internal/maa",
|
||||||
|
visibility = ["//:__subpackages__"],
|
||||||
|
deps = [
|
||||||
|
"@com_github_azure_azure_sdk_for_go//profiles/latest/attestation/attestation",
|
||||||
|
"@com_github_azure_azure_sdk_for_go_sdk_azcore//policy",
|
||||||
|
"@com_github_azure_azure_sdk_for_go_sdk_azidentity//:azidentity",
|
||||||
|
],
|
||||||
|
)
|
||||||
|
|
||||||
|
go_test(
|
||||||
|
name = "maa_test",
|
||||||
|
srcs = ["patch_test.go"],
|
||||||
|
embed = [":maa"],
|
||||||
|
deps = ["@com_github_stretchr_testify//assert"],
|
||||||
|
)
|
9
internal/maa/maa.go
Normal file
9
internal/maa/maa.go
Normal file
@ -0,0 +1,9 @@
|
|||||||
|
/*
|
||||||
|
Copyright (c) Edgeless Systems GmbH
|
||||||
|
|
||||||
|
SPDX-License-Identifier: AGPL-3.0-only
|
||||||
|
*/
|
||||||
|
|
||||||
|
// Package maa provides an interface for interacting with an MAA service
|
||||||
|
// on an infrastructure level.
|
||||||
|
package maa
|
@ -3,7 +3,7 @@ Copyright (c) Edgeless Systems GmbH
|
|||||||
|
|
||||||
SPDX-License-Identifier: AGPL-3.0-only
|
SPDX-License-Identifier: AGPL-3.0-only
|
||||||
*/
|
*/
|
||||||
package cloudcmd
|
package maa
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
@ -3,7 +3,7 @@ Copyright (c) Edgeless Systems GmbH
|
|||||||
|
|
||||||
SPDX-License-Identifier: AGPL-3.0-only
|
SPDX-License-Identifier: AGPL-3.0-only
|
||||||
*/
|
*/
|
||||||
package cloudcmd
|
package maa
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"testing"
|
"testing"
|
Loading…
Reference in New Issue
Block a user