Use terraform in CLI to create QEMU cluster (#172)

* Use terraform in CLI to create QEMU cluster

* Dont allow qemu creation on os/arch other than linux/amd64

* Allow usage of --name flag for QEMU resources

Signed-off-by: Daniel Weiße <dw@edgeless.systems>

cli/internal/terraform/terraform/qemu/main.tf

@@ -0,0 +1,103 @@
+terraform {
+  required_providers {
+    libvirt = {
+      source  = "dmacvicar/libvirt"
+      version = "0.6.14"
+    }
+    docker = {
+      source  = "kreuzwerker/docker"
+      version = "2.17.0"
+    }
+  }
+}
+
+provider "libvirt" {
+  uri = "qemu:///session"
+}
+
+provider "docker" {
+  host = "unix:///var/run/docker.sock"
+
+  registry_auth {
+    address     = "ghcr.io"
+    config_file = pathexpand("~/.docker/config.json")
+  }
+}
+
+resource "docker_image" "qemu-metadata" {
+  name         = "${var.metadata_api_image}"
+  keep_locally = true
+}
+
+resource "docker_container" "qemu-metadata" {
+  name         = "${var.name}-qemu-metadata"
+  image        = docker_image.qemu-metadata.latest
+  network_mode = "host"
+  rm           = true
+  command = [ 
+    "--network",
+    "${var.name}-network",
+  ]
+  mounts {
+    source = "/var/run/libvirt/libvirt-sock"
+    target = "/var/run/libvirt/libvirt-sock"
+    type   = "bind"
+  }
+}
+
+module "control_plane" {
+  source          = "./modules/instance_group"
+  role            = "control-plane"
+  amount          = var.control_plane_count
+  vcpus           = var.vcpus
+  memory          = var.memory
+  state_disk_size = var.state_disk_size
+  ip_range_start  = var.ip_range_start
+  cidr            = "10.42.1.0/24"
+  network_id      = libvirt_network.constellation.id
+  pool            = libvirt_pool.cluster.name
+  boot_volume_id  = libvirt_volume.constellation_coreos_image.id
+  machine         = var.machine
+  name            = var.name
+}
+
+module "worker" {
+  source          = "./modules/instance_group"
+  role            = "worker"
+  amount          = var.worker_count
+  vcpus           = var.vcpus
+  memory          = var.memory
+  state_disk_size = var.state_disk_size
+  ip_range_start  = var.ip_range_start
+  cidr            = "10.42.2.0/24"
+  network_id      = libvirt_network.constellation.id
+  pool            = libvirt_pool.cluster.name
+  boot_volume_id  = libvirt_volume.constellation_coreos_image.id
+  machine         = var.machine
+  name            = var.name
+}
+
+resource "libvirt_pool" "cluster" {
+  name = "${var.name}-storage-pool"
+  type = "dir"
+  path = "/var/lib/libvirt/images"
+}
+
+resource "libvirt_volume" "constellation_coreos_image" {
+  name   = "${var.name}-node-image"
+  pool   = libvirt_pool.cluster.name
+  source = var.constellation_coreos_image
+  format = var.image_format
+}
+
+resource "libvirt_network" "constellation" {
+  name      = "${var.name}-network"
+  mode      = "nat"
+  addresses = ["10.42.0.0/16"]
+  dhcp {
+    enabled = true
+  }
+  dns {
+    enabled = true
+  }
+}

cli/internal/terraform/terraform/qemu/modules/instance_group/domain.xsl

@@ -0,0 +1,24 @@
+<xsl:stylesheet version="2.0" xmlns:xsl="http://www.w3.org/1999/XSL/Transform">
+  <xsl:output omit-xml-declaration="yes" indent="yes"/>
+    <xsl:template match="node()|@*">
+        <xsl:copy>
+            <xsl:apply-templates select="node()|@*"/>
+        </xsl:copy>
+    </xsl:template>
+    <xsl:template match="os">
+        <os firmware="efi">
+            <xsl:apply-templates select="@*|node()"/>
+        </os>
+    </xsl:template>
+    <xsl:template match="/domain/devices/tpm/backend">
+    <xsl:copy>
+        <xsl:apply-templates select="node()|@*"/>
+        <xsl:element name ="active_pcr_banks">
+            <xsl:element name="sha1"></xsl:element>
+            <xsl:element name="sha256"></xsl:element>
+            <xsl:element name="sha384"></xsl:element>
+            <xsl:element name="sha512"></xsl:element>
+        </xsl:element>
+    </xsl:copy>
+  </xsl:template>
+</xsl:stylesheet>

cli/internal/terraform/terraform/qemu/modules/instance_group/main.tf

@@ -0,0 +1,72 @@
+terraform {
+  required_providers {
+    libvirt = {
+      source  = "dmacvicar/libvirt"
+      version = "0.6.14"
+    }
+  }
+}
+
+locals {
+  state_disk_size_byte = 1073741824 * var.state_disk_size
+}
+
+resource "libvirt_domain" "instance_group" {
+  name    = "${var.name}-${var.role}-${count.index}"
+  count   = var.amount
+  memory  = var.memory
+  vcpu    = var.vcpus
+  machine = var.machine
+  tpm {
+    backend_type    = "emulator"
+    backend_version = "2.0"
+  }
+  disk = [
+    {
+      volume_id = element(libvirt_volume.boot_volume.*.id, count.index)
+      scsi : true,
+      // fix for https://github.com/dmacvicar/terraform-provider-libvirt/issues/728
+      block_device : null,
+      file : null,
+      url : null,
+      wwn : null
+    },
+    {
+      volume_id = element(libvirt_volume.state_volume.*.id, count.index)
+      // fix for https://github.com/dmacvicar/terraform-provider-libvirt/issues/728
+      block_device : null,
+      file : null,
+      scsi : null,
+      url : null,
+      wwn : null
+    },
+  ]
+  network_interface {
+    network_id     = var.network_id
+    hostname       = "${var.role}-${count.index}"
+    addresses      = [cidrhost(var.cidr, var.ip_range_start + count.index)]
+    wait_for_lease = true
+  }
+  console {
+    type        = "pty"
+    target_port = "0"
+  }
+  xml {
+    xslt = file("modules/instance_group/domain.xsl")
+  }
+}
+
+resource "libvirt_volume" "boot_volume" {
+  name           = "constellation-${var.role}-${count.index}-boot"
+  count          = var.amount
+  pool           = var.pool
+  base_volume_id = var.boot_volume_id
+}
+
+resource "libvirt_volume" "state_volume" {
+  name   = "constellation-${var.role}-${count.index}-state"
+  count  = var.amount
+  pool   = var.pool
+  size   = local.state_disk_size_byte
+  format = "qcow2"
+}

cli/internal/terraform/terraform/qemu/modules/instance_group/outputs.tf

@@ -0,0 +1,3 @@
+output "instance_ips" {
+  value = flatten(libvirt_domain.instance_group[*].network_interface[*].addresses[*])
+}

cli/internal/terraform/terraform/qemu/modules/instance_group/variables.tf

@@ -0,0 +1,59 @@
+variable "amount" {
+  type        = number
+  description = "amount of nodes"
+}
+
+variable "vcpus" {
+  type        = number
+  description = "amount of vcpus per instance"
+}
+
+variable "memory" {
+  type        = number
+  description = "amount of memory per instance (MiB)"
+}
+
+variable "state_disk_size" {
+  type        = number
+  description = "size of state disk (GiB)"
+}
+
+variable "ip_range_start" {
+  type        = number
+  description = "first ip address to use within subnet"
+}
+
+variable "cidr" {
+  type        = string
+  description = "subnet to use for dhcp"
+}
+
+variable "network_id" {
+  type        = string
+  description = "id of the network to use"
+}
+
+variable "pool" {
+  type        = string
+  description = "name of the storage pool to use"
+}
+
+variable "boot_volume_id" {
+  type        = string
+  description = "id of the constellation boot disk"
+}
+
+variable "role" {
+  type        = string
+  description = "role of the node in the constellation. either 'control-plane' or 'worker'"
+}
+
+variable "machine" {
+  type        = string
+  description = "machine type. use 'q35' for secure boot and 'pc' for non secure boot. See 'qemu-system-x86_64 -machine help'"
+}
+
+variable "name" {
+  type        = string
+  description = "name prefix of the cluster VMs"
+}

cli/internal/terraform/terraform/qemu/outputs.tf

@@ -0,0 +1,3 @@
+output "ip" {
+  value = module.control_plane.instance_ips[0]
+}

cli/internal/terraform/terraform/qemu/variables.tf

@@ -0,0 +1,57 @@
+variable "constellation_coreos_image" {
+  type        = string
+  description = "constellation OS file path"
+}
+
+variable "image_format" {
+  type        = string
+  default     = "qcow2"
+  description = "image format"
+}
+
+variable "control_plane_count" {
+  type        = number
+  description = "amount of control plane nodes"
+}
+
+variable "worker_count" {
+  type        = number
+  description = "amount of worker nodes"
+}
+
+variable "vcpus" {
+  type        = number
+  description = "amount of vcpus per instance"
+}
+
+variable "memory" {
+  type        = number
+  description = "amount of memory per instance (MiB)"
+}
+
+variable "state_disk_size" {
+  type        = number
+  description = "size of state disk (GiB)"
+}
+
+variable "ip_range_start" {
+  type        = number
+  description = "first ip address to use within subnet"
+}
+
+variable "machine" {
+  type        = string
+  default     = "q35"
+  description = "machine type. use 'q35' for secure boot and 'pc' for non secure boot. See 'qemu-system-x86_64 -machine help'"
+}
+
+variable "metadata_api_image" {
+  type        = string
+  description = "container image of the QEMU metadata api server"
+}
+
+variable "name" {
+  type        = string
+  default     = "constellation"
+  description = "name prefix of the cluster VMs"
+}