miniconstellation e2e test as bazel target

2024-09-19 15:56:03 +00:00 · 2023-03-22 14:54:46 +01:00 · 2023-03-22 14:54:46 +01:00 · 18661ced48
commit 18661ced48
parent b2df6ba07a
11 changed files with 252 additions and 118 deletions
--- a/.github/actions/e2e_mini/action.yml
+++ b/.github/actions/e2e_mini/action.yml
@ -23,23 +23,12 @@ runs:
      with:
        terraform_wrapper: false

-    - name: Setup Go environment
-      uses: actions/setup-go@6edd4406fa81c3da01a34fa6f6343087c207a568 # v3.5.0
-      with:
-        go-version: "1.20.2"
-
    - name: Setup bazel
      uses: ./.github/actions/setup_bazel
      with:
        useCache: "true"
        buildBuddyApiKey: ${{ inputs.buildBuddyApiKey }}

-    - name: Build Constellation CLI
-      uses: ./.github/actions/build_cli
-      with:
-        enterpriseCLI: true
-        outputPath: build/constellation
-
    - name: MiniConstellation E2E
      shell: bash
      env:
@ -47,29 +36,4 @@ runs:
        ARM_SUBSCRIPTION_ID: ${{ inputs.azureSubscriptionID }}
        ARM_TENANT_ID: ${{ inputs.azureTenantID }}
      run: |
-        echo "::group::Terraform"
-        cd .github/actions/e2e_mini
-        terraform init
-        terraform apply -auto-approve
-        terraform output -raw ssh_private_key > id_rsa
-        chmod 600 id_rsa
-        export AZURE_VM_IP=$(terraform output -raw public_ip)
-        echo "::endgroup::"
-
-        ./wait-for-ssh.sh
-        # Copy locally build Constellation CLI and run e2e script.
-        scp -i id_rsa ../../../build/constellation adminuser@${AZURE_VM_IP}:.
-        ssh -i id_rsa adminuser@${AZURE_VM_IP} 'bash -s' < run-e2e.sh
-
-    - name: Delete deployment
-      if: always()
-      shell: bash
-      env:
-        ARM_CLIENT_ID: ${{ inputs.azureClientID }}
-        ARM_SUBSCRIPTION_ID: ${{ inputs.azureSubscriptionID }}
-        ARM_TENANT_ID: ${{ inputs.azureTenantID }}
-      run: |
-        echo "::group::Terminate"
-        cd .github/actions/e2e_mini
-        terraform destroy -auto-approve
-        echo "::endgroup::"
+        bazelisk run //e2e/miniconstellation:remote_test
--- a/.github/actions/e2e_mini/run-e2e.sh
+++ b/.github/actions/e2e_mini/run-e2e.sh
@ -1,62 +0,0 @@
-#!/usr/bin/env bash
-#
-# This script installs all dependencies for MiniConstellation to be run on a
-# fresh Ubuntu 22.04 LTS installation.
-# It expects to find the to be used Constellation CLI to be available at
-# $HOME/constellation
-#
-set -euxo pipefail
-
-echo "::group::Install dependencies"
-cloud-init status --wait
-
-export DEBIAN_FRONTEND=noninteractive
-sudo apt update -y
-
-sudo apt install -y bridge-utils cpu-checker \
-  libvirt-clients libvirt-daemon libvirt-daemon-system \
-  qemu qemu-kvm virtinst xsltproc \
-  ca-certificates curl gnupg lsb-release
-
-sudo mkdir -p /etc/apt/keyrings
-curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor -o /etc/apt/keyrings/docker.gpg
-echo \
-  "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu \
-  $(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null
-sudo apt update -y
-sudo apt install -y docker-ce docker-ce-cli containerd.io docker-compose-plugin
-sudo usermod -aG docker "$USER"
-newgrp docker
-echo "::endgroup::"
-
-echo "::group::Run E2E Test"
-mkdir constellation_workspace
-cd constellation_workspace
-mv "$HOME"/constellation .
-chmod u+x constellation
-
-sudo sh -c 'echo "127.0.0.1 license.confidential.cloud" >> /etc/hosts'
-
-./constellation mini up
-
-curl -fsSLO "https://dl.k8s.io/release/$(curl -fsSL https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl"
-sudo install kubectl /usr/local/bin/kubectl
-
-export KUBECONFIG="$PWD/constellation-admin.conf"
-
-# Wait for nodes to actually show up in K8s control plane
-sleep 10
-
-# Wait for nodes
-kubectl wait --for=condition=Ready --timeout=600s nodes control-plane-0
-kubectl wait --for=condition=Ready --timeout=600s nodes worker-0
-# Wait for deployments
-kubectl -n kube-system wait --for=condition=Available=True --timeout=180s deployment coredns
-kubectl -n kube-system wait --for=condition=Available=True --timeout=180s deployment cilium-operator
-# Wait for daemon sets
-kubectl -n kube-system rollout status --timeout 180s daemonset cilium
-kubectl -n kube-system rollout status --timeout 180s daemonset join-service
-kubectl -n kube-system rollout status --timeout 180s daemonset key-service
-kubectl -n kube-system rollout status --timeout 180s daemonset konnectivity-agent
-kubectl -n kube-system rollout status --timeout 180s daemonset verification-service
-echo "::endgroup::"
--- a/.github/actions/e2e_mini/wait-for-ssh.sh
+++ b/.github/actions/e2e_mini/wait-for-ssh.sh
@ -1,13 +0,0 @@
-#!/usr/bin/env bash
-
-set +e
-echo "Waiting for SSH server to come online..."
-
-# Wait for SSH to come online, at most 10*30s=5min
-COUNT=0
-until ssh -i id_rsa -o StrictHostKeyChecking=no adminuser@"${AZURE_VM_IP}" date || [ $COUNT -eq 10 ]; do
-  sleep 30
-  ((COUNT++))
-done
-
-echo "Done waiting."
--- a/.github/actions/e2e_mini/.gitignore
+++ b/.github/actions/e2e_mini/.gitignore
--- a/.github/actions/e2e_mini/.terraform.lock.hcl
+++ b/.github/actions/e2e_mini/.terraform.lock.hcl
@ -31,6 +31,30 @@ provider "registry.terraform.io/hashicorp/azurerm" {
  ]
 }

+provider "registry.terraform.io/hashicorp/cloudinit" {
+  version     = "2.3.2"
+  constraints = "2.3.2"
+  hashes = [
+    "h1:2jb+BfT5T96dXxUD2LQ6MtVHpXErd7ZybmMvdWE2jd4=",
+    "h1:Ar/DAbZQ9Nsj0BrqX6camrEE6U+Yq4E87DCNVqxqx8k=",
+    "h1:Vl0aixAYTV/bjathX7VArC5TVNkxBCsi3Vq7R4z1uvc=",
+    "h1:ocyv0lvfyvzW4krenxV5CL4Jq5DiA3EUfoy8DR6zFMw=",
+    "h1:y+6FsU2STOpx6L6JOon4DVZoZPQgNoR2xR2WQ/EVxcQ=",
+    "zh:2487e498736ed90f53de8f66fe2b8c05665b9f8ff1506f751c5ee227c7f457d1",
+    "zh:3d8627d142942336cf65eea6eb6403692f47e9072ff3fa11c3f774a3b93130b3",
+    "zh:434b643054aeafb5df28d5529b72acc20c6f5ded24decad73b98657af2b53f4f",
+    "zh:436aa6c2b07d82aa6a9dd746a3e3a627f72787c27c80552ceda6dc52d01f4b6f",
+    "zh:458274c5aabe65ef4dbd61d43ce759287788e35a2da004e796373f88edcaa422",
+    "zh:54bc70fa6fb7da33292ae4d9ceef5398d637c7373e729ed4fce59bd7b8d67372",
+    "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3",
+    "zh:893ba267e18749c1a956b69be569f0d7bc043a49c3a0eb4d0d09a8e8b2ca3136",
+    "zh:95493b7517bce116f75cdd4c63b7c82a9d0d48ec2ef2f5eb836d262ef96d0aa7",
+    "zh:9ae21ab393be52e3e84e5cce0ef20e690d21f6c10ade7d9d9d22b39851bfeddc",
+    "zh:cc3b01ac2472e6d59358d54d5e4945032efbc8008739a6d4946ca1b621a16040",
+    "zh:f23bfe9758f06a1ec10ea3a81c9deedf3a7b42963568997d84a5153f35c5839a",
+  ]
+}
+
 provider "registry.terraform.io/hashicorp/random" {
  version     = "3.4.3"
  constraints = "3.4.3"
--- a/e2e/miniconstellation/BUILD.bazel
+++ b/e2e/miniconstellation/BUILD.bazel
@ -0,0 +1,33 @@
+load("//bazel/sh:def.bzl", "sh_template")
+
+filegroup(
+    name = "dependencies",
+    srcs = [
+        ".terraform.lock.hcl",
+        "cloud-init.yaml",
+        "main.tf",
+        "output.tf",
+        "test-remote.sh",
+    ],
+)
+
+sh_library(
+    name = "dependencies_lib",
+    srcs = [
+        ":dependencies",
+    ],
+)
+
+sh_template(
+    name = "remote_test",
+    data = [
+        "//cli:cli_enterprise_linux_amd64",
+    ],
+    substitutions = {
+        "@@PATH_CLI@@": "$(rootpath //cli:cli_enterprise_linux_amd64)",
+    },
+    template = "main.sh.in",
+    deps = [
+        ":dependencies_lib",
+    ],
+)
--- a/e2e/miniconstellation/cloud-init.yaml
+++ b/e2e/miniconstellation/cloud-init.yaml
@ -0,0 +1,39 @@
+#cloud-config
+
+users:
+ - default
+ - name: adminuser
+   groups: docker
+   sudo: ALL=(ALL) NOPASSWD:ALL
+   homedir: /home/adminuser
+
+groups:
+  - docker
+
+package_update: true
+packages:
+ - git
+ - cryptsetup
+ - build-essential
+ - libguestfs-tools
+ - apt-transport-https
+ - ca-certificates
+ - curl
+ - gnupg
+ - lsb-release
+ - jq
+ - pv
+ - xsltproc
+ - qemu-guest-agent
+ - libvirt-clients
+ - libvirt-daemon
+ - libvirt-daemon-system
+
+runcmd:
+ - [/bin/bash, -c, "curl -fsSL https://download.docker.com/linux/ubuntu/gpg | gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg"]
+ - [/bin/bash, -c, "echo \"deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable\" | tee /etc/apt/sources.list.d/docker.list > /dev/null "]
+ - [apt-get, update]
+ - [apt-get, install, -y, docker-ce, docker-ce-cli, containerd.io, libssl-dev, pigz]
+ - [/bin/bash, -c, "systemctl enable docker.service && systemctl start docker.service"]
+ - [/bin/bash, -c, "curl -fsSLO \"https://dl.k8s.io/release/$(curl -fsSL https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl\" && install kubectl /usr/local/bin/kubectl"]
+ - [/bin/bash, -c, "echo \"127.0.0.1 license.confidential.cloud\" >> /etc/hosts"]
--- a/e2e/miniconstellation/main.sh.in
+++ b/e2e/miniconstellation/main.sh.in
@ -0,0 +1,51 @@
+#!/usr/bin/env bash
+
+clean_up() {
+  echo "::group::Terminate"
+
+  terraform destroy -auto-approve
+
+  echo "::endgroup::"
+}
+
+path_lib=$(realpath @@BASE_LIB@@) || exit 1
+path_cli=$(realpath @@PATH_CLI@@) || exit 1
+
+# shellcheck source=../../bazel/sh/lib.bash
+if ! source "${path_lib}"; then
+  echo "Error: could not find import"
+  exit 1
+fi
+
+registerExitHandler clean_up
+
+cd e2e/miniconstellation
+
+echo "::group::Terraform"
+
+terraform init
+terraform apply -auto-approve
+terraform output -raw ssh_private_key > id_rsa
+chmod 600 id_rsa
+
+azure_vm_ip=$(terraform output -raw public_ip)
+
+echo "::endgroup::"
+
+echo "Waiting for SSH server to come online..."
+
+# Wait for SSH to come online, at most 10*30s=5min
+count=0
+until ssh -i id_rsa -o StrictHostKeyChecking=no adminuser@"${azure_vm_ip}" date || [[ ${count} -eq 10 ]]; do
+  sleep 30
+  count=$((count + 1))
+done
+
+echo "Done waiting."
+
+# Copy locally build Constellation CLI and run e2e script.
+echo "::group::Copy files to remote VM"
+scp -i id_rsa "${path_cli}" adminuser@"${azure_vm_ip}":constellation
+echo "::endgroup::"
+
+ssh -i id_rsa adminuser@"${azure_vm_ip}" 'bash -s' < test-remote.sh
--- a/.github/actions/e2e_mini/main.tf
+++ b/.github/actions/e2e_mini/main.tf
@ -12,6 +12,10 @@ terraform {
      source  = "hashicorp/tls"
      version = "4.0.4"
    }
+    cloudinit = {
+      source  = "hashicorp/cloudinit"
+      version = "2.3.2"
+    }
  }
 }

@ -32,6 +36,16 @@ resource "tls_private_key" "ssh_key" {
  rsa_bits  = 2048
 }

+data "cloudinit_config" "cloud_init" {
+  base64_encode = true
+  part {
+    filename     = "cloud-init.yaml"
+    content_type = "text/cloud-config"
+
+    content = file("${path.module}/cloud-init.yaml")
+  }
+}
+
 data "azurerm_resource_group" "main" {
  name = "e2e-miniconstellation"
 }
@ -98,19 +112,21 @@ resource "azurerm_linux_virtual_machine" "main" {
  name                = "e2e-mini-${random_string.suffix.result}"
  resource_group_name = data.azurerm_resource_group.main.name
  location            = data.azurerm_resource_group.main.location
-  # Dv3-series provides nested virtualization support
-  # https://learn.microsoft.com/en-us/azure/virtual-machines/dv3-dsv3-series#dv3-series
-  size           = "Standard_D8_v3"
+
+  # Standard_D8as_v5 provides nested virtualization support
+  size = "Standard_D8as_v5"
+
  admin_username = "adminuser"
-  network_interface_ids = [
-    azurerm_network_interface.main.id,
-  ]

  admin_ssh_key {
    username   = "adminuser"
    public_key = tls_private_key.ssh_key.public_key_openssh
  }

+  network_interface_ids = [
+    azurerm_network_interface.main.id,
+  ]
+
  source_image_reference {
    publisher = "Canonical"
    offer     = "0001-com-ubuntu-server-jammy-daily"
@ -122,4 +138,6 @@ resource "azurerm_linux_virtual_machine" "main" {
    storage_account_type = "Standard_LRS"
    caching              = "ReadWrite"
  }
+
+  user_data = data.cloudinit_config.cloud_init.rendered
 }
--- a/.github/actions/e2e_mini/output.tf
+++ b/.github/actions/e2e_mini/output.tf
--- a/e2e/miniconstellation/test-remote.sh
+++ b/e2e/miniconstellation/test-remote.sh
@ -0,0 +1,80 @@
+#!/usr/bin/env bash
+#
+# This script expects:
+# Constellation CLI @ $PWD/constellation
+# kubectl @ PATH
+
+set -euo pipefail
+
+debug_info() {
+  arg=$?
+
+  echo "Getting debug info"
+
+  sha256sum ./*.raw
+
+  ls -lisah
+
+  cat ./constellation-conf.yaml
+
+  exit "${arg}"
+}
+
+trap debug_info ERR
+
+echo "::group::Run E2E Test"
+mkdir constellation_workspace
+cd constellation_workspace
+cp ../constellation .
+chmod u+x constellation
+
+# wait for docker to come up
+echo "Waiting for docker service to be active..."
+
+# Wait at most 5min
+count=0
+until systemctl is-active docker || [[ ${count} -eq 30 ]]; do
+  sleep 10
+  count=$((count + 1))
+done
+
+if [[ ${count} -eq 30 ]]; then
+  echo "Docker service did not come up in time."
+  exit 1
+fi
+
+echo "Done waiting."
+
+./constellation mini up --debug
+
+export KUBECONFIG="${PWD}/constellation-admin.conf"
+
+# Wait for nodes to actually show up in K8s
+count=0
+until kubectl wait --for=condition=Ready --timeout=2s nodes control-plane-0 2> /dev/null || [[ ${count} -eq 30 ]]; do
+  echo "Control-planes are not registered in Kubernetes yet. Waiting..."
+  sleep 10
+  count=$((count + 1))
+done
+
+count=0
+until kubectl wait --for=condition=Ready --timeout=2s nodes worker-0 2> /dev/null || [[ ${count} -eq 30 ]]; do
+  echo "Worker nodes are not registered in Kubernetes yet. Waiting..."
+  sleep 10
+  count=$((count + 1))
+done
+
+# Wait for deployments
+kubectl -n kube-system wait --for=condition=Available=True --timeout=180s deployment coredns
+kubectl -n kube-system wait --for=condition=Available=True --timeout=180s deployment cilium-operator
+# Wait for daemon sets
+kubectl -n kube-system rollout status --timeout 180s daemonset cilium
+kubectl -n kube-system rollout status --timeout 180s daemonset join-service
+kubectl -n kube-system rollout status --timeout 180s daemonset key-service
+kubectl -n kube-system rollout status --timeout 180s daemonset konnectivity-agent
+kubectl -n kube-system rollout status --timeout 180s daemonset verification-service
+
+echo "Miniconstellation started successfully. Shutting down..."
+
+./constellation mini down -y
+echo "::endgroup::"