E2E Test Mini Constellation (#796)

* fix: typo to build amd64 for macos
* Implement E2E test for mini constellation
Signed-off-by: Fabian Kammel <fk@edgeless.systems>

.github/actions/e2e_mini/.gitignore

@@ -0,0 +1,2 @@
+id_rsa
+id_rsa.pub

.github/actions/e2e_mini/.terraform.lock.hcl

@@ -0,0 +1,74 @@
+# This file is maintained automatically by "terraform init".
+# Manual edits may be lost in future updates.
+
+provider "registry.terraform.io/hashicorp/azurerm" {
+  version     = "3.32.0"
+  constraints = "3.32.0"
+  hashes = [
+    "h1:/ziRWmSDLF+zqfYDBiO4lCBJ20gIwPjDErIR8whhYSg=",
+    "h1:Rl31vGl2LBhpmFRaZ3bTq21VG8W1eZOuP90VpwNw9fM=",
+    "h1:dfbqrKzCN4csxin1abdn8KJXHv89R4skhkUglCI4poY=",
+    "h1:vb1ZppEoZ6wAozqq0Ll4E4llcqprKv9Mr319OqMLM0M=",
+    "h1:zjaayFSTUOxL0J0vE3MSPGbUNVxbUkQv76THmtrAM4Q=",
+    "zh:3ee1992144e6bf9801c44df0ed1e10413fa83ad605e3ce751cb342dd46904c41",
+    "zh:4f083079909f929b76c0cb2819b107803ecbf26c761832aaa1e7b4a667025665",
+    "zh:52ad565c4bd37c2b4f0bba78639277ef98caaebf2c4c00c67a2659561079c21c",
+    "zh:5ecf7a8470e066cc27b837a8fbc9a02629bb85797007475539983496bcccbc53",
+    "zh:6348154495cd838862b27a9bc0a2714e8f76cd2919df55fce8da0f64ce240ab1",
+    "zh:8325c4f5f65e30bba2537c7df702c80ae29999fba6194c258b075b3cbde5a709",
+    "zh:8b4d33aa76474a9fac9a6859e759c03ffeadb787abf7a9ba5a05b4ca3914c008",
+    "zh:95ccd31450909582ebcf01548ee20df658049783530d79adcb53a601bb163597",
+    "zh:c104f977b96c6402276c82a8d9d6fee14381511e832e9c3593e589e5ee4e708c",
+    "zh:e12372a41a981c24323a467f6c54b0a17e26c85a0fb569e4b733b2a76c9ba6b6",
+    "zh:e80bf9b674914f91ed00984758288b7266ba5772fad728cd1b4cd2f776851ed8",
+    "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c",
+  ]
+}
+
+provider "registry.terraform.io/hashicorp/random" {
+  version     = "3.4.3"
+  constraints = "3.4.3"
+  hashes = [
+    "h1:hV66lcagXXRwwCW3Y542bI1JgPo8z/taYKT7K+a2Z5U=",
+    "h1:hXUPrH8igYBhatzatkp80RCeeUJGu9lQFDyKemOlsTo=",
+    "h1:saZR+mhthL0OZl4SyHXZraxyaBNVMxiZzks78nWcZ2o=",
+    "h1:tL3katm68lX+4lAncjQA9AXL4GR/VM+RPwqYf4D2X8Q=",
+    "h1:xZGZf18JjMS06pFa4NErzANI98qi59SEcBsOcS2P2yQ=",
+    "zh:41c53ba47085d8261590990f8633c8906696fa0a3c4b384ff6a7ecbf84339752",
+    "zh:59d98081c4475f2ad77d881c4412c5129c56214892f490adf11c7e7a5a47de9b",
+    "zh:686ad1ee40b812b9e016317e7f34c0d63ef837e084dea4a1f578f64a6314ad53",
+    "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3",
+    "zh:84103eae7251384c0d995f5a257c72b0096605048f757b749b7b62107a5dccb3",
+    "zh:8ee974b110adb78c7cd18aae82b2729e5124d8f115d484215fd5199451053de5",
+    "zh:9dd4561e3c847e45de603f17fa0c01ae14cae8c4b7b4e6423c9ef3904b308dda",
+    "zh:bb07bb3c2c0296beba0beec629ebc6474c70732387477a65966483b5efabdbc6",
+    "zh:e891339e96c9e5a888727b45b2e1bb3fcbdfe0fd7c5b4396e4695459b38c8cb1",
+    "zh:ea4739860c24dfeaac6c100b2a2e357106a89d18751f7693f3c31ecf6a996f8d",
+    "zh:f0c76ac303fd0ab59146c39bc121c5d7d86f878e9a69294e29444d4c653786f8",
+    "zh:f143a9a5af42b38fed328a161279906759ff39ac428ebcfe55606e05e1518b93",
+  ]
+}
+
+provider "registry.terraform.io/hashicorp/tls" {
+  version     = "4.0.4"
+  constraints = "4.0.4"
+  hashes = [
+    "h1:GZcFizg5ZT2VrpwvxGBHQ/hO9r6g0vYdQqx3bFD3anY=",
+    "h1:Wd3RqmQW60k2QWPN4sK5CtjGuO1d+CRNXgC+D4rKtXc=",
+    "h1:bNsvpX5EGuVxgGRXBQVLXlmq40PdoLp8Rfuh1ZmV7yY=",
+    "h1:pe9vq86dZZKCm+8k1RhzARwENslF3SXb9ErHbQfgjXU=",
+    "h1:rKKMyIEBZwR+8j6Tx3PwqBrStuH+J+pxcbCR5XN8WAw=",
+    "zh:23671ed83e1fcf79745534841e10291bbf34046b27d6e68a5d0aab77206f4a55",
+    "zh:45292421211ffd9e8e3eb3655677700e3c5047f71d8f7650d2ce30242335f848",
+    "zh:59fedb519f4433c0fdb1d58b27c210b27415fddd0cd73c5312530b4309c088be",
+    "zh:5a8eec2409a9ff7cd0758a9d818c74bcba92a240e6c5e54b99df68fff312bbd5",
+    "zh:5e6a4b39f3171f53292ab88058a59e64825f2b842760a4869e64dc1dc093d1fe",
+    "zh:810547d0bf9311d21c81cc306126d3547e7bd3f194fc295836acf164b9f8424e",
+    "zh:824a5f3617624243bed0259d7dd37d76017097dc3193dac669be342b90b2ab48",
+    "zh:9361ccc7048be5dcbc2fafe2d8216939765b3160bd52734f7a9fd917a39ecbd8",
+    "zh:aa02ea625aaf672e649296bce7580f62d724268189fe9ad7c1b36bb0fa12fa60",
+    "zh:c71b4cd40d6ec7815dfeefd57d88bc592c0c42f5e5858dcc88245d371b4b8b1e",
+    "zh:dabcd52f36b43d250a3d71ad7abfa07b5622c69068d989e60b79b2bb4f220316",
+    "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c",
+  ]
+}

.github/actions/e2e_mini/action.yml

@@ -0,0 +1,65 @@
+name: E2E QEMU test
+description: "Test MiniConstellation functionality on Azure VM."
+
+inputs:
+  azureClientID:
+    description: "Azure client to use for login with OIDC"
+    required: true
+  azureSubscriptionID:
+    description: "Azure subscription to use for login with OIDC"
+    required: true
+  azureTenantID:
+    description: "Azure tenant to use for login with OIDC"
+    required: true
+
+runs:
+  using: "composite"
+  steps:
+    - name: Install terraform
+      uses: hashicorp/setup-terraform@633666f66e0061ca3b725c73b2ec20cd13a8fdd1 # v2.0.3
+      with:
+        terraform_wrapper: false
+
+    - name: Setup Go environment
+      uses: actions/setup-go@c4a742cab115ed795e34d4513e2cf7d472deb55f # tag=v3.3.1
+      with:
+        go-version: "1.19.4"
+
+    - name: Build Constellation CLI
+      uses: ./.github/actions/build_cli
+      with:
+        enterpriseCLI: true
+
+    - name: MiniConstellation E2E
+      shell: bash
+      run: |
+        echo "::group::Terraform"
+        cd .github/actions/e2e_mini
+        terraform init
+        terraform apply -auto-approve
+        terraform output -raw ssh_private_key > id_rsa
+        chmod 600 id_rsa
+        export AZURE_VM_IP=$(terraform output -raw public_ip)
+        echo "::endgroup::"
+
+        ./wait-for-ssh.sh
+        # Copy locally build Constellation CLI and run e2e script.
+        scp -i id_rsa ../../../build/constellation adminuser@${AZURE_VM_IP}:.
+        ssh -i id_rsa adminuser@${AZURE_VM_IP} 'bash -s' < run-e2e.sh
+      env:
+        ARM_CLIENT_ID: ${{ inputs.azureClientID }}
+        ARM_SUBSCRIPTION_ID: ${{ inputs.azureSubscriptionID }}
+        ARM_TENANT_ID: ${{ inputs.azureTenantID }}
+
+    - name: Delete deployment
+      if: always()
+      shell: bash
+      run: |
+        echo "::group::Terminate"
+        cd .github/actions/e2e_mini
+        terraform destroy -auto-approve
+        echo "::endgroup::"
+      env:
+        ARM_CLIENT_ID: ${{ inputs.azureClientID }}
+        ARM_SUBSCRIPTION_ID: ${{ inputs.azureSubscriptionID }}
+        ARM_TENANT_ID: ${{ inputs.azureTenantID }}

.github/actions/e2e_mini/main.tf

@@ -0,0 +1,125 @@
+terraform {
+  required_providers {
+    azurerm = {
+      source  = "hashicorp/azurerm"
+      version = "3.32.0"
+    }
+    random = {
+      source  = "hashicorp/random"
+      version = "3.4.3"
+    }
+    tls = {
+      source  = "hashicorp/tls"
+      version = "4.0.4"
+    }
+  }
+}
+
+provider "azurerm" {
+  use_oidc = true
+  features {}
+}
+
+provider "tls" {}
+
+resource "random_string" "suffix" {
+  length  = 6
+  special = false
+}
+
+resource "tls_private_key" "ssh_key" {
+  algorithm = "RSA"
+  rsa_bits  = 2048
+}
+
+data "azurerm_resource_group" "main" {
+  name = "e2e-miniconstellation"
+}
+
+resource "azurerm_virtual_network" "main" {
+  name                = "e2e-mini-${random_string.suffix.result}"
+  address_space       = ["10.0.0.0/16"]
+  location            = data.azurerm_resource_group.main.location
+  resource_group_name = data.azurerm_resource_group.main.name
+}
+
+resource "azurerm_subnet" "main" {
+  name                 = "e2e-mini-${random_string.suffix.result}"
+  resource_group_name  = data.azurerm_resource_group.main.name
+  virtual_network_name = azurerm_virtual_network.main.name
+  address_prefixes     = ["10.0.2.0/24"]
+}
+
+resource "azurerm_public_ip" "main" {
+  name                = "e2e-mini-${random_string.suffix.result}"
+  location            = data.azurerm_resource_group.main.location
+  resource_group_name = data.azurerm_resource_group.main.name
+  allocation_method   = "Static"
+  sku                 = "Standard"
+}
+
+resource "azurerm_network_interface" "main" {
+  name                = "e2e-mini-${random_string.suffix.result}"
+  resource_group_name = data.azurerm_resource_group.main.name
+  location            = data.azurerm_resource_group.main.location
+
+  ip_configuration {
+    name                          = "main"
+    subnet_id                     = azurerm_subnet.main.id
+    private_ip_address_allocation = "Dynamic"
+    public_ip_address_id          = azurerm_public_ip.main.id
+  }
+}
+
+resource "azurerm_network_security_group" "ssh" {
+  name                = "e2e-mini-${random_string.suffix.result}"
+  resource_group_name = data.azurerm_resource_group.main.name
+  location            = data.azurerm_resource_group.main.location
+
+  security_rule {
+    name                       = "ssh"
+    priority                   = 100
+    direction                  = "Inbound"
+    access                     = "Allow"
+    protocol                   = "Tcp"
+    source_port_range          = "*"
+    destination_port_range     = "22"
+    source_address_prefix      = "*"
+    destination_address_prefix = "*"
+  }
+}
+
+resource "azurerm_subnet_network_security_group_association" "ssh" {
+  subnet_id                 = azurerm_subnet.main.id
+  network_security_group_id = azurerm_network_security_group.ssh.id
+}
+
+resource "azurerm_linux_virtual_machine" "main" {
+  name                = "e2e-mini-${random_string.suffix.result}"
+  resource_group_name = data.azurerm_resource_group.main.name
+  location            = data.azurerm_resource_group.main.location
+  # Dv3-series provides nested virtualization support
+  # https://learn.microsoft.com/en-us/azure/virtual-machines/dv3-dsv3-series#dv3-series
+  size           = "Standard_D8_v3"
+  admin_username = "adminuser"
+  network_interface_ids = [
+    azurerm_network_interface.main.id,
+  ]
+
+  admin_ssh_key {
+    username   = "adminuser"
+    public_key = tls_private_key.ssh_key.public_key_openssh
+  }
+
+  source_image_reference {
+    publisher = "Canonical"
+    offer     = "0001-com-ubuntu-server-jammy-daily"
+    sku       = "22_04-daily-lts"
+    version   = "latest"
+  }
+
+  os_disk {
+    storage_account_type = "Standard_LRS"
+    caching              = "ReadWrite"
+  }
+}

.github/actions/e2e_mini/output.tf

@@ -0,0 +1,11 @@
+output "public_ip" {
+  value      = azurerm_public_ip.main.ip_address
+  sensitive  = false
+  depends_on = [azurerm_public_ip.main]
+}
+
+output "ssh_private_key" {
+  value      = tls_private_key.ssh_key.private_key_openssh
+  sensitive  = true
+  depends_on = [tls_private_key.ssh_key]
+}

.github/actions/e2e_mini/run-e2e.sh

@@ -0,0 +1,65 @@
+#!/usr/bin/env bash
+#
+# This script installs all dependencies for MiniConstellation to be run on a
+# fresh Ubuntu 22.04 LTS installation.
+# It expects to find the to be used Constellation CLI to be available at
+# $HOME/constellation
+#
+set -euxo pipefail
+
+echo "::group::Install dependencies"
+cloud-init status --wait
+
+export DEBIAN_FRONTEND=noninteractive
+sudo apt update -y
+
+sudo apt install -y bridge-utils cpu-checker \
+  libvirt-clients libvirt-daemon libvirt-daemon-system \
+  qemu qemu-kvm virtinst xsltproc \
+  ca-certificates curl gnupg lsb-release
+
+sudo mkdir -p /etc/apt/keyrings
+curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor -o /etc/apt/keyrings/docker.gpg
+echo \
+  "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu \
+  $(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null
+sudo apt update -y
+sudo apt install -y docker-ce docker-ce-cli containerd.io docker-compose-plugin
+sudo usermod -aG docker "$USER"
+newgrp docker
+echo "::endgroup::"
+
+echo "::group::Run E2E Test"
+mkdir constellation_workspace
+cd constellation_workspace
+mv "$HOME"/constellation .
+chmod u+x constellation
+
+sudo sh -c 'echo "127.0.0.1 license.confidential.cloud" >> /etc/hosts'
+
+mkdir -p "$HOME"/.docker
+touch "$HOME"/.docker/config.json
+
+./constellation mini up
+
+curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl"
+sudo install kubectl /usr/local/bin/kubectl
+
+export KUBECONFIG="$PWD/constellation-admin.conf"
+
+# Wait for nodes to actually show up in K8s control plane
+sleep 10
+
+# Wait for nodes
+kubectl wait --for=condition=Ready --timeout=600s nodes control-plane-0
+kubectl wait --for=condition=Ready --timeout=600s nodes worker-0
+# Wait for deployments
+kubectl -n kube-system wait --for=condition=Available=True --timeout=180s deployment coredns
+kubectl -n kube-system wait --for=condition=Available=True --timeout=180s deployment cilium-operator
+# Wait for daemon sets
+kubectl -n kube-system rollout status --timeout 180s daemonset cilium
+kubectl -n kube-system rollout status --timeout 180s daemonset join-service
+kubectl -n kube-system rollout status --timeout 180s daemonset kms
+kubectl -n kube-system rollout status --timeout 180s daemonset konnectivity-agent
+kubectl -n kube-system rollout status --timeout 180s daemonset verification-service
+echo "::endgroup::"

.github/actions/e2e_mini/wait-for-ssh.sh

@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+
+set +e
+echo "Waiting for SSH server to come online..."
+
+# Wait for SSH to come online, at most 10*30s=5min
+COUNT=0
+until ssh -i id_rsa -o StrictHostKeyChecking=no adminuser@"${AZURE_VM_IP}" date || [ $COUNT -eq 10 ]; do
+  sleep 30
+  ((COUNT++))
+done
+
+echo "Done waiting."

.github/docs/release.md

@@ -87,6 +87,7 @@ This checklist will prepare `v1.3.0` from `v1.2.0`. Adjust your version numbers
         gh workflow run e2e-test-manual-macos.yml --ref release/v$minor -F cloudProvider=azure -F test="sonobuoy full" -F osImage=v$ver -F isDebugImage=false -F keepMeasurements=true
         gh workflow run e2e-test-manual.yml --ref release/v$minor -F cloudProvider=gcp -F test="sonobuoy full" -F osImage=v$ver -F isDebugImage=false -F keepMeasurements=true
         gh workflow run e2e-test-manual-macos.yml --ref release/v$minor -F cloudProvider=gcp -F test="sonobuoy full" -F osImage=v$ver -F isDebugImage=false -F keepMeasurements=true
+        gh workflow run e2e-mini.yml --ref release/v$minor
         ```
 
     13. Create a new tag on this release branch.

.github/workflows/build-binaries.yml

@@ -91,7 +91,7 @@ jobs:
         uses: ./.github/actions/build_cdbg
         with:
           targetOS: "darwin"
-          targetArch: "arm64"
+          targetArch: "amd64"
 
       - name: Build cdbg (macOS, arm64)
         uses: ./.github/actions/build_cdbg

.github/workflows/e2e-mini.yml

@@ -0,0 +1,31 @@
+name: e2e test qemu (MiniConstellation)
+
+on:
+  workflow_dispatch:
+
+permissions:
+  id-token: write
+  contents: read
+
+jobs:
+  e2e-mini:
+    runs-on: ubuntu-22.04
+    environment: e2e
+    steps:
+      - name: Checkout
+        id: checkout
+        uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # tag=v3.1.0
+
+      - name: Azure login OIDC
+        uses: azure/login@92a5484dfaf04ca78a94597f4f19fea633851fa2 # v1.4.6
+        with:
+          client-id: ${{ secrets.AZURE_E2E_MINI_CLIENT_ID }}
+          tenant-id: ${{ secrets.AZURE_TENANT_ID }}
+          subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
+
+      - name: Run e2e MiniConstellation
+        uses: ./.github/actions/e2e_mini
+        with:
+          azureClientID: ${{ secrets.AZURE_E2E_MINI_CLIENT_ID }}
+          azureSubscriptionID: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
+          azureTenantID: ${{ secrets.AZURE_TENANT_ID }}