From 6564fcbf6cc890309206c923dfa7e4d456a33319 Mon Sep 17 00:00:00 2001 From: Fabian Kammel Date: Wed, 14 Dec 2022 16:51:42 +0100 Subject: [PATCH] E2E Test Mini Constellation (#796) * fix: typo to build amd64 for macos * Implement E2E test for mini constellation Signed-off-by: Fabian Kammel --- .github/actions/e2e_mini/.gitignore | 2 + .github/actions/e2e_mini/.terraform.lock.hcl | 74 +++++++++++ .github/actions/e2e_mini/action.yml | 65 ++++++++++ .github/actions/e2e_mini/main.tf | 125 +++++++++++++++++++ .github/actions/e2e_mini/output.tf | 11 ++ .github/actions/e2e_mini/run-e2e.sh | 65 ++++++++++ .github/actions/e2e_mini/wait-for-ssh.sh | 13 ++ .github/docs/release.md | 1 + .github/workflows/build-binaries.yml | 2 +- .github/workflows/e2e-mini.yml | 31 +++++ 10 files changed, 388 insertions(+), 1 deletion(-) create mode 100644 .github/actions/e2e_mini/.gitignore create mode 100644 .github/actions/e2e_mini/.terraform.lock.hcl create mode 100644 .github/actions/e2e_mini/action.yml create mode 100644 .github/actions/e2e_mini/main.tf create mode 100644 .github/actions/e2e_mini/output.tf create mode 100644 .github/actions/e2e_mini/run-e2e.sh create mode 100755 .github/actions/e2e_mini/wait-for-ssh.sh create mode 100644 .github/workflows/e2e-mini.yml diff --git a/.github/actions/e2e_mini/.gitignore b/.github/actions/e2e_mini/.gitignore new file mode 100644 index 000000000..0cf054ac7 --- /dev/null +++ b/.github/actions/e2e_mini/.gitignore @@ -0,0 +1,2 @@ +id_rsa +id_rsa.pub diff --git a/.github/actions/e2e_mini/.terraform.lock.hcl b/.github/actions/e2e_mini/.terraform.lock.hcl new file mode 100644 index 000000000..3e1398484 --- /dev/null +++ b/.github/actions/e2e_mini/.terraform.lock.hcl @@ -0,0 +1,74 @@ +# This file is maintained automatically by "terraform init". +# Manual edits may be lost in future updates. + +provider "registry.terraform.io/hashicorp/azurerm" { + version = "3.32.0" + constraints = "3.32.0" + hashes = [ + "h1:/ziRWmSDLF+zqfYDBiO4lCBJ20gIwPjDErIR8whhYSg=", + "h1:Rl31vGl2LBhpmFRaZ3bTq21VG8W1eZOuP90VpwNw9fM=", + "h1:dfbqrKzCN4csxin1abdn8KJXHv89R4skhkUglCI4poY=", + "h1:vb1ZppEoZ6wAozqq0Ll4E4llcqprKv9Mr319OqMLM0M=", + "h1:zjaayFSTUOxL0J0vE3MSPGbUNVxbUkQv76THmtrAM4Q=", + "zh:3ee1992144e6bf9801c44df0ed1e10413fa83ad605e3ce751cb342dd46904c41", + "zh:4f083079909f929b76c0cb2819b107803ecbf26c761832aaa1e7b4a667025665", + "zh:52ad565c4bd37c2b4f0bba78639277ef98caaebf2c4c00c67a2659561079c21c", + "zh:5ecf7a8470e066cc27b837a8fbc9a02629bb85797007475539983496bcccbc53", + "zh:6348154495cd838862b27a9bc0a2714e8f76cd2919df55fce8da0f64ce240ab1", + "zh:8325c4f5f65e30bba2537c7df702c80ae29999fba6194c258b075b3cbde5a709", + "zh:8b4d33aa76474a9fac9a6859e759c03ffeadb787abf7a9ba5a05b4ca3914c008", + "zh:95ccd31450909582ebcf01548ee20df658049783530d79adcb53a601bb163597", + "zh:c104f977b96c6402276c82a8d9d6fee14381511e832e9c3593e589e5ee4e708c", + "zh:e12372a41a981c24323a467f6c54b0a17e26c85a0fb569e4b733b2a76c9ba6b6", + "zh:e80bf9b674914f91ed00984758288b7266ba5772fad728cd1b4cd2f776851ed8", + "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c", + ] +} + +provider "registry.terraform.io/hashicorp/random" { + version = "3.4.3" + constraints = "3.4.3" + hashes = [ + "h1:hV66lcagXXRwwCW3Y542bI1JgPo8z/taYKT7K+a2Z5U=", + "h1:hXUPrH8igYBhatzatkp80RCeeUJGu9lQFDyKemOlsTo=", + "h1:saZR+mhthL0OZl4SyHXZraxyaBNVMxiZzks78nWcZ2o=", + "h1:tL3katm68lX+4lAncjQA9AXL4GR/VM+RPwqYf4D2X8Q=", + "h1:xZGZf18JjMS06pFa4NErzANI98qi59SEcBsOcS2P2yQ=", + "zh:41c53ba47085d8261590990f8633c8906696fa0a3c4b384ff6a7ecbf84339752", + "zh:59d98081c4475f2ad77d881c4412c5129c56214892f490adf11c7e7a5a47de9b", + "zh:686ad1ee40b812b9e016317e7f34c0d63ef837e084dea4a1f578f64a6314ad53", + "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", + "zh:84103eae7251384c0d995f5a257c72b0096605048f757b749b7b62107a5dccb3", + "zh:8ee974b110adb78c7cd18aae82b2729e5124d8f115d484215fd5199451053de5", + "zh:9dd4561e3c847e45de603f17fa0c01ae14cae8c4b7b4e6423c9ef3904b308dda", + "zh:bb07bb3c2c0296beba0beec629ebc6474c70732387477a65966483b5efabdbc6", + "zh:e891339e96c9e5a888727b45b2e1bb3fcbdfe0fd7c5b4396e4695459b38c8cb1", + "zh:ea4739860c24dfeaac6c100b2a2e357106a89d18751f7693f3c31ecf6a996f8d", + "zh:f0c76ac303fd0ab59146c39bc121c5d7d86f878e9a69294e29444d4c653786f8", + "zh:f143a9a5af42b38fed328a161279906759ff39ac428ebcfe55606e05e1518b93", + ] +} + +provider "registry.terraform.io/hashicorp/tls" { + version = "4.0.4" + constraints = "4.0.4" + hashes = [ + "h1:GZcFizg5ZT2VrpwvxGBHQ/hO9r6g0vYdQqx3bFD3anY=", + "h1:Wd3RqmQW60k2QWPN4sK5CtjGuO1d+CRNXgC+D4rKtXc=", + "h1:bNsvpX5EGuVxgGRXBQVLXlmq40PdoLp8Rfuh1ZmV7yY=", + "h1:pe9vq86dZZKCm+8k1RhzARwENslF3SXb9ErHbQfgjXU=", + "h1:rKKMyIEBZwR+8j6Tx3PwqBrStuH+J+pxcbCR5XN8WAw=", + "zh:23671ed83e1fcf79745534841e10291bbf34046b27d6e68a5d0aab77206f4a55", + "zh:45292421211ffd9e8e3eb3655677700e3c5047f71d8f7650d2ce30242335f848", + "zh:59fedb519f4433c0fdb1d58b27c210b27415fddd0cd73c5312530b4309c088be", + "zh:5a8eec2409a9ff7cd0758a9d818c74bcba92a240e6c5e54b99df68fff312bbd5", + "zh:5e6a4b39f3171f53292ab88058a59e64825f2b842760a4869e64dc1dc093d1fe", + "zh:810547d0bf9311d21c81cc306126d3547e7bd3f194fc295836acf164b9f8424e", + "zh:824a5f3617624243bed0259d7dd37d76017097dc3193dac669be342b90b2ab48", + "zh:9361ccc7048be5dcbc2fafe2d8216939765b3160bd52734f7a9fd917a39ecbd8", + "zh:aa02ea625aaf672e649296bce7580f62d724268189fe9ad7c1b36bb0fa12fa60", + "zh:c71b4cd40d6ec7815dfeefd57d88bc592c0c42f5e5858dcc88245d371b4b8b1e", + "zh:dabcd52f36b43d250a3d71ad7abfa07b5622c69068d989e60b79b2bb4f220316", + "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c", + ] +} diff --git a/.github/actions/e2e_mini/action.yml b/.github/actions/e2e_mini/action.yml new file mode 100644 index 000000000..c87e0437f --- /dev/null +++ b/.github/actions/e2e_mini/action.yml @@ -0,0 +1,65 @@ +name: E2E QEMU test +description: "Test MiniConstellation functionality on Azure VM." + +inputs: + azureClientID: + description: "Azure client to use for login with OIDC" + required: true + azureSubscriptionID: + description: "Azure subscription to use for login with OIDC" + required: true + azureTenantID: + description: "Azure tenant to use for login with OIDC" + required: true + +runs: + using: "composite" + steps: + - name: Install terraform + uses: hashicorp/setup-terraform@633666f66e0061ca3b725c73b2ec20cd13a8fdd1 # v2.0.3 + with: + terraform_wrapper: false + + - name: Setup Go environment + uses: actions/setup-go@c4a742cab115ed795e34d4513e2cf7d472deb55f # tag=v3.3.1 + with: + go-version: "1.19.4" + + - name: Build Constellation CLI + uses: ./.github/actions/build_cli + with: + enterpriseCLI: true + + - name: MiniConstellation E2E + shell: bash + run: | + echo "::group::Terraform" + cd .github/actions/e2e_mini + terraform init + terraform apply -auto-approve + terraform output -raw ssh_private_key > id_rsa + chmod 600 id_rsa + export AZURE_VM_IP=$(terraform output -raw public_ip) + echo "::endgroup::" + + ./wait-for-ssh.sh + # Copy locally build Constellation CLI and run e2e script. + scp -i id_rsa ../../../build/constellation adminuser@${AZURE_VM_IP}:. + ssh -i id_rsa adminuser@${AZURE_VM_IP} 'bash -s' < run-e2e.sh + env: + ARM_CLIENT_ID: ${{ inputs.azureClientID }} + ARM_SUBSCRIPTION_ID: ${{ inputs.azureSubscriptionID }} + ARM_TENANT_ID: ${{ inputs.azureTenantID }} + + - name: Delete deployment + if: always() + shell: bash + run: | + echo "::group::Terminate" + cd .github/actions/e2e_mini + terraform destroy -auto-approve + echo "::endgroup::" + env: + ARM_CLIENT_ID: ${{ inputs.azureClientID }} + ARM_SUBSCRIPTION_ID: ${{ inputs.azureSubscriptionID }} + ARM_TENANT_ID: ${{ inputs.azureTenantID }} diff --git a/.github/actions/e2e_mini/main.tf b/.github/actions/e2e_mini/main.tf new file mode 100644 index 000000000..819991ef8 --- /dev/null +++ b/.github/actions/e2e_mini/main.tf @@ -0,0 +1,125 @@ +terraform { + required_providers { + azurerm = { + source = "hashicorp/azurerm" + version = "3.32.0" + } + random = { + source = "hashicorp/random" + version = "3.4.3" + } + tls = { + source = "hashicorp/tls" + version = "4.0.4" + } + } +} + +provider "azurerm" { + use_oidc = true + features {} +} + +provider "tls" {} + +resource "random_string" "suffix" { + length = 6 + special = false +} + +resource "tls_private_key" "ssh_key" { + algorithm = "RSA" + rsa_bits = 2048 +} + +data "azurerm_resource_group" "main" { + name = "e2e-miniconstellation" +} + +resource "azurerm_virtual_network" "main" { + name = "e2e-mini-${random_string.suffix.result}" + address_space = ["10.0.0.0/16"] + location = data.azurerm_resource_group.main.location + resource_group_name = data.azurerm_resource_group.main.name +} + +resource "azurerm_subnet" "main" { + name = "e2e-mini-${random_string.suffix.result}" + resource_group_name = data.azurerm_resource_group.main.name + virtual_network_name = azurerm_virtual_network.main.name + address_prefixes = ["10.0.2.0/24"] +} + +resource "azurerm_public_ip" "main" { + name = "e2e-mini-${random_string.suffix.result}" + location = data.azurerm_resource_group.main.location + resource_group_name = data.azurerm_resource_group.main.name + allocation_method = "Static" + sku = "Standard" +} + +resource "azurerm_network_interface" "main" { + name = "e2e-mini-${random_string.suffix.result}" + resource_group_name = data.azurerm_resource_group.main.name + location = data.azurerm_resource_group.main.location + + ip_configuration { + name = "main" + subnet_id = azurerm_subnet.main.id + private_ip_address_allocation = "Dynamic" + public_ip_address_id = azurerm_public_ip.main.id + } +} + +resource "azurerm_network_security_group" "ssh" { + name = "e2e-mini-${random_string.suffix.result}" + resource_group_name = data.azurerm_resource_group.main.name + location = data.azurerm_resource_group.main.location + + security_rule { + name = "ssh" + priority = 100 + direction = "Inbound" + access = "Allow" + protocol = "Tcp" + source_port_range = "*" + destination_port_range = "22" + source_address_prefix = "*" + destination_address_prefix = "*" + } +} + +resource "azurerm_subnet_network_security_group_association" "ssh" { + subnet_id = azurerm_subnet.main.id + network_security_group_id = azurerm_network_security_group.ssh.id +} + +resource "azurerm_linux_virtual_machine" "main" { + name = "e2e-mini-${random_string.suffix.result}" + resource_group_name = data.azurerm_resource_group.main.name + location = data.azurerm_resource_group.main.location + # Dv3-series provides nested virtualization support + # https://learn.microsoft.com/en-us/azure/virtual-machines/dv3-dsv3-series#dv3-series + size = "Standard_D8_v3" + admin_username = "adminuser" + network_interface_ids = [ + azurerm_network_interface.main.id, + ] + + admin_ssh_key { + username = "adminuser" + public_key = tls_private_key.ssh_key.public_key_openssh + } + + source_image_reference { + publisher = "Canonical" + offer = "0001-com-ubuntu-server-jammy-daily" + sku = "22_04-daily-lts" + version = "latest" + } + + os_disk { + storage_account_type = "Standard_LRS" + caching = "ReadWrite" + } +} diff --git a/.github/actions/e2e_mini/output.tf b/.github/actions/e2e_mini/output.tf new file mode 100644 index 000000000..14f541264 --- /dev/null +++ b/.github/actions/e2e_mini/output.tf @@ -0,0 +1,11 @@ +output "public_ip" { + value = azurerm_public_ip.main.ip_address + sensitive = false + depends_on = [azurerm_public_ip.main] +} + +output "ssh_private_key" { + value = tls_private_key.ssh_key.private_key_openssh + sensitive = true + depends_on = [tls_private_key.ssh_key] +} diff --git a/.github/actions/e2e_mini/run-e2e.sh b/.github/actions/e2e_mini/run-e2e.sh new file mode 100644 index 000000000..af1d05770 --- /dev/null +++ b/.github/actions/e2e_mini/run-e2e.sh @@ -0,0 +1,65 @@ +#!/usr/bin/env bash +# +# This script installs all dependencies for MiniConstellation to be run on a +# fresh Ubuntu 22.04 LTS installation. +# It expects to find the to be used Constellation CLI to be available at +# $HOME/constellation +# +set -euxo pipefail + +echo "::group::Install dependencies" +cloud-init status --wait + +export DEBIAN_FRONTEND=noninteractive +sudo apt update -y + +sudo apt install -y bridge-utils cpu-checker \ + libvirt-clients libvirt-daemon libvirt-daemon-system \ + qemu qemu-kvm virtinst xsltproc \ + ca-certificates curl gnupg lsb-release + +sudo mkdir -p /etc/apt/keyrings +curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor -o /etc/apt/keyrings/docker.gpg +echo \ + "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu \ + $(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null +sudo apt update -y +sudo apt install -y docker-ce docker-ce-cli containerd.io docker-compose-plugin +sudo usermod -aG docker "$USER" +newgrp docker +echo "::endgroup::" + +echo "::group::Run E2E Test" +mkdir constellation_workspace +cd constellation_workspace +mv "$HOME"/constellation . +chmod u+x constellation + +sudo sh -c 'echo "127.0.0.1 license.confidential.cloud" >> /etc/hosts' + +mkdir -p "$HOME"/.docker +touch "$HOME"/.docker/config.json + +./constellation mini up + +curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl" +sudo install kubectl /usr/local/bin/kubectl + +export KUBECONFIG="$PWD/constellation-admin.conf" + +# Wait for nodes to actually show up in K8s control plane +sleep 10 + +# Wait for nodes +kubectl wait --for=condition=Ready --timeout=600s nodes control-plane-0 +kubectl wait --for=condition=Ready --timeout=600s nodes worker-0 +# Wait for deployments +kubectl -n kube-system wait --for=condition=Available=True --timeout=180s deployment coredns +kubectl -n kube-system wait --for=condition=Available=True --timeout=180s deployment cilium-operator +# Wait for daemon sets +kubectl -n kube-system rollout status --timeout 180s daemonset cilium +kubectl -n kube-system rollout status --timeout 180s daemonset join-service +kubectl -n kube-system rollout status --timeout 180s daemonset kms +kubectl -n kube-system rollout status --timeout 180s daemonset konnectivity-agent +kubectl -n kube-system rollout status --timeout 180s daemonset verification-service +echo "::endgroup::" diff --git a/.github/actions/e2e_mini/wait-for-ssh.sh b/.github/actions/e2e_mini/wait-for-ssh.sh new file mode 100755 index 000000000..0c7741182 --- /dev/null +++ b/.github/actions/e2e_mini/wait-for-ssh.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash + +set +e +echo "Waiting for SSH server to come online..." + +# Wait for SSH to come online, at most 10*30s=5min +COUNT=0 +until ssh -i id_rsa -o StrictHostKeyChecking=no adminuser@"${AZURE_VM_IP}" date || [ $COUNT -eq 10 ]; do + sleep 30 + ((COUNT++)) +done + +echo "Done waiting." diff --git a/.github/docs/release.md b/.github/docs/release.md index 4822688f3..033ca579f 100644 --- a/.github/docs/release.md +++ b/.github/docs/release.md @@ -87,6 +87,7 @@ This checklist will prepare `v1.3.0` from `v1.2.0`. Adjust your version numbers gh workflow run e2e-test-manual-macos.yml --ref release/v$minor -F cloudProvider=azure -F test="sonobuoy full" -F osImage=v$ver -F isDebugImage=false -F keepMeasurements=true gh workflow run e2e-test-manual.yml --ref release/v$minor -F cloudProvider=gcp -F test="sonobuoy full" -F osImage=v$ver -F isDebugImage=false -F keepMeasurements=true gh workflow run e2e-test-manual-macos.yml --ref release/v$minor -F cloudProvider=gcp -F test="sonobuoy full" -F osImage=v$ver -F isDebugImage=false -F keepMeasurements=true + gh workflow run e2e-mini.yml --ref release/v$minor ``` 13. Create a new tag on this release branch. diff --git a/.github/workflows/build-binaries.yml b/.github/workflows/build-binaries.yml index 15662e02d..2e9f5b8e1 100644 --- a/.github/workflows/build-binaries.yml +++ b/.github/workflows/build-binaries.yml @@ -91,7 +91,7 @@ jobs: uses: ./.github/actions/build_cdbg with: targetOS: "darwin" - targetArch: "arm64" + targetArch: "amd64" - name: Build cdbg (macOS, arm64) uses: ./.github/actions/build_cdbg diff --git a/.github/workflows/e2e-mini.yml b/.github/workflows/e2e-mini.yml new file mode 100644 index 000000000..596bc48f2 --- /dev/null +++ b/.github/workflows/e2e-mini.yml @@ -0,0 +1,31 @@ +name: e2e test qemu (MiniConstellation) + +on: + workflow_dispatch: + +permissions: + id-token: write + contents: read + +jobs: + e2e-mini: + runs-on: ubuntu-22.04 + environment: e2e + steps: + - name: Checkout + id: checkout + uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # tag=v3.1.0 + + - name: Azure login OIDC + uses: azure/login@92a5484dfaf04ca78a94597f4f19fea633851fa2 # v1.4.6 + with: + client-id: ${{ secrets.AZURE_E2E_MINI_CLIENT_ID }} + tenant-id: ${{ secrets.AZURE_TENANT_ID }} + subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} + + - name: Run e2e MiniConstellation + uses: ./.github/actions/e2e_mini + with: + azureClientID: ${{ secrets.AZURE_E2E_MINI_CLIENT_ID }} + azureSubscriptionID: ${{ secrets.AZURE_SUBSCRIPTION_ID }} + azureTenantID: ${{ secrets.AZURE_TENANT_ID }}