terraform: QEMU node groups (#1961)

* init

add variables

add amount to instance_group again

fix tf validate

rollback old names

make fields optional

fix image ref mini

daniel comments

use latest

* Update cli/internal/terraform/terraform/qemu/main.tf

Co-authored-by: Malte Poll <1780588+malt3@users.noreply.github.com>

* add uid to resource name

* make machine a global variable again

* fix tf

---------

Co-authored-by: Malte Poll <1780588+malt3@users.noreply.github.com>
This commit is contained in:
Adrian Stobbe 2023-06-28 14:42:34 +02:00 committed by GitHub
parent 161bb37cba
commit 9bb91ca447
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
11 changed files with 202 additions and 188 deletions

View File

@ -406,31 +406,43 @@ func (c *Creator) createQEMU(ctx context.Context, cl terraformClient, lv libvirt
}
vars := terraform.QEMUVariables{
CommonVariables: terraform.CommonVariables{
Name: opts.Config.Name,
CountControlPlanes: opts.ControlPlaneCount,
CountWorkers: opts.WorkerCount,
StateDiskSizeGB: opts.Config.StateDiskSizeGB,
},
Name: opts.Config.Name,
LibvirtURI: libvirtURI,
LibvirtSocketPath: libvirtSocketPath,
// TODO(malt3): auto select boot mode based on attestation variant.
// requires image info v2.
BootMode: "uefi",
ImagePath: imagePath,
ImageFormat: opts.Config.Provider.QEMU.ImageFormat,
CPUCount: opts.Config.Provider.QEMU.VCPUs,
MemorySizeMiB: opts.Config.Provider.QEMU.Memory,
BootMode: "uefi",
ImagePath: imagePath,
ImageFormat: opts.Config.Provider.QEMU.ImageFormat,
NodeGroups: map[string]terraform.QEMUNodeGroup{
"control_plane_default": {
Role: role.ControlPlane.TFString(),
InstanceCount: opts.ControlPlaneCount,
DiskSize: opts.Config.StateDiskSizeGB,
CPUCount: opts.Config.Provider.QEMU.VCPUs,
MemorySize: opts.Config.Provider.QEMU.Memory,
},
"worker_default": {
Role: role.Worker.TFString(),
InstanceCount: opts.WorkerCount,
DiskSize: opts.Config.StateDiskSizeGB,
CPUCount: opts.Config.Provider.QEMU.VCPUs,
MemorySize: opts.Config.Provider.QEMU.Memory,
},
},
Machine: "q35", // TODO(elchead): make configurable AB#3225
MetadataAPIImage: opts.Config.Provider.QEMU.MetadataAPIImage,
MetadataLibvirtURI: metadataLibvirtURI,
NVRAM: opts.Config.Provider.QEMU.NVRAM,
Firmware: opts.Config.Provider.QEMU.Firmware,
// TODO(malt3) enable once we have a way to auto-select values for these
// requires image info v2.
// BzImagePath: placeholder,
// InitrdPath: placeholder,
// KernelCmdline: placeholder,
}
if opts.Config.Provider.QEMU.Firmware != "" {
vars.Firmware = toPtr(opts.Config.Provider.QEMU.Firmware)
}
if err := cl.PrepareWorkspace(path.Join("terraform", strings.ToLower(cloudprovider.QEMU.String())), &vars); err != nil {
return clusterid.File{}, fmt.Errorf("prepare workspace: %w", err)

View File

@ -103,6 +103,7 @@ go_test(
"//internal/cloud/cloudprovider",
"//internal/constants",
"//internal/file",
"//internal/role",
"@com_github_azure_azure_sdk_for_go_sdk_azcore//to",
"@com_github_hashicorp_terraform_exec//tfexec",
"@com_github_hashicorp_terraform_json//:terraform-json",

View File

@ -28,25 +28,26 @@ provider "registry.terraform.io/dmacvicar/libvirt" {
}
provider "registry.terraform.io/hashicorp/random" {
version = "3.4.3"
version = "3.5.1"
constraints = "3.5.1"
hashes = [
"h1:hV66lcagXXRwwCW3Y542bI1JgPo8z/taYKT7K+a2Z5U=",
"h1:hXUPrH8igYBhatzatkp80RCeeUJGu9lQFDyKemOlsTo=",
"h1:saZR+mhthL0OZl4SyHXZraxyaBNVMxiZzks78nWcZ2o=",
"h1:tL3katm68lX+4lAncjQA9AXL4GR/VM+RPwqYf4D2X8Q=",
"h1:xZGZf18JjMS06pFa4NErzANI98qi59SEcBsOcS2P2yQ=",
"zh:41c53ba47085d8261590990f8633c8906696fa0a3c4b384ff6a7ecbf84339752",
"zh:59d98081c4475f2ad77d881c4412c5129c56214892f490adf11c7e7a5a47de9b",
"zh:686ad1ee40b812b9e016317e7f34c0d63ef837e084dea4a1f578f64a6314ad53",
"h1:3hjTP5tQBspPcFAJlfafnWrNrKnr7J4Cp0qB9jbqf30=",
"h1:6FVyQ/aG6tawPam6B+oFjgdidKd83uG9n7dOSQ66HBA=",
"h1:IL9mSatmwov+e0+++YX2V6uel+dV6bn+fC/cnGDK3Ck=",
"h1:VSnd9ZIPyfKHOObuQCaKfnjIHRtR7qTw19Rz8tJxm+k=",
"h1:sZ7MTSD4FLekNN2wSNFGpM+5slfvpm5A/NLVZiB7CO0=",
"zh:04e3fbd610cb52c1017d282531364b9c53ef72b6bc533acb2a90671957324a64",
"zh:119197103301ebaf7efb91df8f0b6e0dd31e6ff943d231af35ee1831c599188d",
"zh:4d2b219d09abf3b1bb4df93d399ed156cadd61f44ad3baf5cf2954df2fba0831",
"zh:6130bdde527587bbe2dcaa7150363e96dbc5250ea20154176d82bc69df5d4ce3",
"zh:6cc326cd4000f724d3086ee05587e7710f032f94fc9af35e96a386a1c6f2214f",
"zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3",
"zh:84103eae7251384c0d995f5a257c72b0096605048f757b749b7b62107a5dccb3",
"zh:8ee974b110adb78c7cd18aae82b2729e5124d8f115d484215fd5199451053de5",
"zh:9dd4561e3c847e45de603f17fa0c01ae14cae8c4b7b4e6423c9ef3904b308dda",
"zh:bb07bb3c2c0296beba0beec629ebc6474c70732387477a65966483b5efabdbc6",
"zh:e891339e96c9e5a888727b45b2e1bb3fcbdfe0fd7c5b4396e4695459b38c8cb1",
"zh:ea4739860c24dfeaac6c100b2a2e357106a89d18751f7693f3c31ecf6a996f8d",
"zh:f0c76ac303fd0ab59146c39bc121c5d7d86f878e9a69294e29444d4c653786f8",
"zh:f143a9a5af42b38fed328a161279906759ff39ac428ebcfe55606e05e1518b93",
"zh:b6d88e1d28cf2dfa24e9fdcc3efc77adcdc1c3c3b5c7ce503a423efbdd6de57b",
"zh:ba74c592622ecbcef9dc2a4d81ed321c4e44cddf7da799faa324da9bf52a22b2",
"zh:c7c5cde98fe4ef1143bd1b3ec5dc04baf0d4cc3ca2c5c7d40d17c0e9b2076865",
"zh:dac4bad52c940cd0dfc27893507c1e92393846b024c5a9db159a93c534a3da03",
"zh:de8febe2a2acd9ac454b844a4106ed295ae9520ef54dc8ed2faf29f12716b602",
"zh:eab0d0495e7e711cca367f7d4df6e322e6c562fc52151ec931176115b83ed014",
]
}

View File

@ -50,47 +50,27 @@ resource "docker_container" "qemu_metadata" {
}
module "control_plane" {
module "node_group" {
source = "./modules/instance_group"
role = "control-plane"
amount = var.control_plane_count
vcpus = var.vcpus
memory = var.memory
state_disk_size = var.state_disk_size
cidr = "10.42.1.0/24"
base_name = var.name
for_each = var.node_groups
node_group_name = each.key
role = each.value.role
amount = each.value.instance_count
state_disk_size = each.value.disk_size
vcpus = each.value.vcpus
memory = each.value.memory
machine = var.machine
cidr = each.value.role == "control-plane" ? "10.42.1.0/24" : "10.42.2.0/24"
network_id = libvirt_network.constellation.id
pool = libvirt_pool.cluster.name
boot_mode = var.constellation_boot_mode
boot_volume_id = libvirt_volume.constellation_os_image.id
kernel_volume_id = local.kernel_volume_id
initrd_volume_id = local.initrd_volume_id
kernel_cmdline = local.kernel_cmdline
machine = var.machine
kernel_cmdline = each.value.role == "control-plane" ? local.kernel_cmdline : var.constellation_cmdline
firmware = var.firmware
nvram = var.nvram
name = var.name
}
module "worker" {
source = "./modules/instance_group"
role = "worker"
amount = var.worker_count
vcpus = var.vcpus
memory = var.memory
state_disk_size = var.state_disk_size
cidr = "10.42.2.0/24"
network_id = libvirt_network.constellation.id
pool = libvirt_pool.cluster.name
boot_mode = var.constellation_boot_mode
boot_volume_id = libvirt_volume.constellation_os_image.id
kernel_volume_id = local.kernel_volume_id
initrd_volume_id = local.initrd_volume_id
kernel_cmdline = var.constellation_cmdline
machine = var.machine
firmware = var.firmware
nvram = var.nvram
name = var.name
}
resource "libvirt_pool" "cluster" {

View File

@ -4,12 +4,15 @@ terraform {
source = "dmacvicar/libvirt"
version = "0.7.1"
}
random = {
source = "hashicorp/random"
version = "3.5.1"
}
}
}
resource "libvirt_domain" "instance_group" {
name = "${var.name}-${var.role}-${count.index}"
count = var.amount
name = "${var.base_name}-${var.role}-${local.group_uid}-${count.index}"
memory = var.memory
vcpu = var.vcpus
machine = var.machine
@ -56,21 +59,36 @@ resource "libvirt_domain" "instance_group" {
}
resource "libvirt_volume" "boot_volume" {
name = "constellation-${var.role}-${count.index}-boot"
count = var.amount
name = "constellation-${var.role}-${local.group_uid}-${count.index}-boot"
pool = var.pool
base_volume_id = var.boot_volume_id
lifecycle {
ignore_changes = [
name, # required. Allow legacy scale sets to keep their old names
]
}
}
resource "libvirt_volume" "state_volume" {
name = "constellation-${var.role}-${count.index}-state"
count = var.amount
name = "constellation-${var.role}-${local.group_uid}-${count.index}-state"
pool = var.pool
size = local.state_disk_size_byte
format = "qcow2"
lifecycle {
ignore_changes = [
name, # required. Allow legacy scale sets to keep their old names
]
}
}
resource "random_id" "uid" {
byte_length = 4
}
locals {
group_uid = random_id.uid.hex
state_disk_size_byte = 1073741824 * var.state_disk_size
ip_range_start = 100
kernel = var.boot_mode == "direct-linux-boot" ? var.kernel_volume_id : null

View File

@ -84,8 +84,12 @@ variable "nvram" {
type = string
description = "path to UEFI NVRAM template file. Used for secure boot."
}
variable "name" {
variable "base_name" {
type = string
description = "name prefix of the cluster VMs"
}
variable "node_group_name" {
type = string
description = "name of the node group"
}

View File

@ -1,5 +1,5 @@
output "ip" {
value = module.control_plane.instance_ips[0]
value = module.node_group["control_plane_default"].instance_ips[0]
}
output "uid" {

View File

@ -1,3 +1,25 @@
variable "node_groups" {
type = map(object({
role = string
instance_count = number // number of instances in the node group
disk_size = number // size of state disk (GiB)
vcpus = number
memory = number // amount of memory per instance (MiB)
}))
validation {
condition = can([for group in var.node_groups : group.role == "control-plane" || group.role == "worker"])
error_message = "The role has to be 'control-plane' or 'worker'."
}
description = "A map of node group names to node group configurations."
}
variable "machine" {
type = string
default = "q35"
description = "machine type. use 'q35' for secure boot and 'pc' for non secure boot. See 'qemu-system-x86_64 -machine help'"
}
variable "libvirt_uri" {
type = string
description = "libvirt socket uri"
@ -43,38 +65,6 @@ variable "image_format" {
default = "qcow2"
description = "image format"
}
variable "control_plane_count" {
type = number
description = "amount of control plane nodes"
}
variable "worker_count" {
type = number
description = "amount of worker nodes"
}
variable "vcpus" {
type = number
description = "amount of vcpus per instance"
}
variable "memory" {
type = number
description = "amount of memory per instance (MiB)"
}
variable "state_disk_size" {
type = number
description = "size of state disk (GiB)"
}
variable "machine" {
type = string
default = "q35"
description = "machine type. use 'q35' for secure boot and 'pc' for non secure boot. See 'qemu-system-x86_64 -machine help'"
}
variable "firmware" {
type = string
default = "/usr/share/OVMF/OVMF_CODE.secboot.fd"

View File

@ -19,6 +19,7 @@ import (
"github.com/edgelesssys/constellation/v2/internal/cloud/cloudprovider"
"github.com/edgelesssys/constellation/v2/internal/constants"
"github.com/edgelesssys/constellation/v2/internal/file"
"github.com/edgelesssys/constellation/v2/internal/role"
"github.com/hashicorp/terraform-exec/tfexec"
tfjson "github.com/hashicorp/terraform-json"
"github.com/spf13/afero"
@ -33,17 +34,16 @@ func TestMain(m *testing.M) {
func TestPrepareCluster(t *testing.T) {
qemuVars := &QEMUVariables{
CommonVariables: CommonVariables{
Name: "name",
CountControlPlanes: 1,
CountWorkers: 2,
StateDiskSizeGB: 11,
Name: "name",
NodeGroups: map[string]QEMUNodeGroup{
"control-plane": {
Role: role.ControlPlane.TFString(),
DiskSize: 30,
CPUCount: 1,
MemorySize: 1024,
},
},
CPUCount: 1,
MemorySizeMiB: 1024,
ImagePath: "path",
ImageFormat: "format",
MetadataAPIImage: "api",
Machine: "q35",
}
testCases := map[string]struct {
@ -248,14 +248,16 @@ func TestCreateCluster(t *testing.T) {
return &workingState
}
qemuVars := &QEMUVariables{
CommonVariables: CommonVariables{
Name: "name",
CountControlPlanes: 1,
CountWorkers: 2,
StateDiskSizeGB: 11,
Name: "name",
NodeGroups: map[string]QEMUNodeGroup{
"control-plane": {
Role: role.ControlPlane.TFString(),
DiskSize: 11,
CPUCount: 1,
MemorySize: 1024,
},
},
CPUCount: 1,
MemorySizeMiB: 1024,
Machine: "q35",
ImagePath: "path",
ImageFormat: "format",
MetadataAPIImage: "api",

View File

@ -277,71 +277,68 @@ func (v *OpenStackClusterVariables) String() string {
// QEMUVariables is user configuration for creating a QEMU cluster with Terraform.
type QEMUVariables struct {
// CommonVariables contains common variables.
CommonVariables
// Name is the name to use for the cluster.
Name string `hcl:"name" cty:"name"`
// NodeGroups is a map of node groups to create.
NodeGroups map[string]QEMUNodeGroup `hcl:"node_groups" cty:"node_groups"`
// Machine is the type of machine to use. use 'q35' for secure boot and 'pc' for non secure boot. See 'qemu-system-x86_64 -machine help'
Machine string `hcl:"machine" cty:"machine"`
// LibvirtURI is the libvirt connection URI.
LibvirtURI string
LibvirtURI string `hcl:"libvirt_uri" cty:"libvirt_uri"`
// LibvirtSocketPath is the path to the libvirt socket in case of unix socket.
LibvirtSocketPath string
LibvirtSocketPath string `hcl:"libvirt_socket_path" cty:"libvirt_socket_path"`
// BootMode is the boot mode to use.
// Can be either "uefi" or "direct-linux-boot".
BootMode string
// CPUCount is the number of CPUs to allocate to each node.
CPUCount int
// MemorySizeMiB is the amount of memory to allocate to each node, in MiB.
MemorySizeMiB int
// IPRangeStart is the first IP address in the IP range to allocate to the cluster.
ImagePath string
BootMode string `hcl:"constellation_boot_mode" cty:"constellation_boot_mode"`
// ImagePath is the path to the image to use for the nodes.
ImagePath string `hcl:"constellation_os_image" cty:"constellation_os_image"`
// ImageFormat is the format of the image from ImagePath.
ImageFormat string
ImageFormat string `hcl:"image_format" cty:"image_format"`
// MetadataAPIImage is the container image to use for the metadata API.
MetadataAPIImage string
MetadataAPIImage string `hcl:"metadata_api_image" cty:"metadata_api_image"`
// MetadataLibvirtURI is the libvirt connection URI used by the metadata container.
// In case of unix socket, this should be "qemu:///system".
// Other wise it should be the same as LibvirtURI.
MetadataLibvirtURI string
MetadataLibvirtURI string `hcl:"metadata_libvirt_uri" cty:"metadata_libvirt_uri"`
// NVRAM is the path to the NVRAM template.
NVRAM string
NVRAM string `hcl:"nvram" cty:"nvram"`
// Firmware is the path to the firmware.
Firmware string
Firmware *string `hcl:"firmware" cty:"firmware"`
// BzImagePath is the path to the bzImage (kernel).
BzImagePath string
BzImagePath *string `hcl:"constellation_kernel" cty:"constellation_kernel"`
// InitrdPath is the path to the initrd.
InitrdPath string
InitrdPath *string `hcl:"constellation_initrd" cty:"constellation_initrd"`
// KernelCmdline is the kernel command line.
KernelCmdline string
KernelCmdline *string `hcl:"constellation_cmdline" cty:"constellation_cmdline"`
}
// String returns a string representation of the variables, formatted as Terraform variables.
func (v *QEMUVariables) String() string {
b := &strings.Builder{}
b.WriteString(v.CommonVariables.String())
writeLinef(b, "libvirt_uri = %q", v.LibvirtURI)
writeLinef(b, "libvirt_socket_path = %q", v.LibvirtSocketPath)
writeLinef(b, "constellation_os_image = %q", v.ImagePath)
writeLinef(b, "image_format = %q", v.ImageFormat)
writeLinef(b, "constellation_boot_mode = %q", v.BootMode)
writeLinef(b, "constellation_kernel = %q", v.BzImagePath)
writeLinef(b, "constellation_initrd = %q", v.InitrdPath)
writeLinef(b, "constellation_cmdline = %q", v.KernelCmdline)
writeLinef(b, "vcpus = %d", v.CPUCount)
writeLinef(b, "memory = %d", v.MemorySizeMiB)
writeLinef(b, "metadata_api_image = %q", v.MetadataAPIImage)
writeLinef(b, "metadata_libvirt_uri = %q", v.MetadataLibvirtURI)
switch v.NVRAM {
// copy v object
vCopy := *v
switch vCopy.NVRAM {
case "production":
b.WriteString("nvram = \"/usr/share/OVMF/constellation_vars.production.fd\"\n")
vCopy.NVRAM = "/usr/share/OVMF/constellation_vars.production.fd"
case "testing":
b.WriteString("nvram = \"/usr/share/OVMF/constellation_vars.testing.fd\"\n")
default:
writeLinef(b, "nvram = %q", v.NVRAM)
}
if v.Firmware != "" {
writeLinef(b, "firmware = %q", v.Firmware)
vCopy.NVRAM = "/usr/share/OVMF/constellation_vars.testing.fd"
}
f := hclwrite.NewEmptyFile()
gohcl.EncodeIntoBody(vCopy, f.Body())
return string(f.Bytes())
}
return b.String()
// QEMUNodeGroup is a node group for a QEMU cluster.
type QEMUNodeGroup struct {
// Role is the role of the node group.
Role string `hcl:"role" cty:"role"`
// InstanceCount is the number of instances to create.
InstanceCount int `hcl:"instance_count" cty:"instance_count"`
// DiskSize is the size of the disk to allocate to each node, in GiB.
DiskSize int `hcl:"disk_size" cty:"disk_size"`
// CPUCount is the number of CPUs to allocate to each node.
CPUCount int `hcl:"vcpus" cty:"vcpus"`
// MemorySize is the amount of memory to allocate to each node, in MiB.
MemorySize int `hcl:"memory" cty:"memory"`
}
func writeLinef(builder *strings.Builder, format string, a ...any) {

View File

@ -10,6 +10,7 @@ import (
"testing"
"github.com/Azure/azure-sdk-for-go/sdk/azcore/to"
"github.com/edgelesssys/constellation/v2/internal/role"
"github.com/stretchr/testify/assert"
)
@ -245,49 +246,57 @@ debug = true
}
func TestQEMUClusterVariables(t *testing.T) {
vars := QEMUVariables{
CommonVariables: CommonVariables{
Name: "cluster-name",
CountControlPlanes: 1,
CountWorkers: 2,
StateDiskSizeGB: 30,
vars := &QEMUVariables{
Name: "cluster-name",
NodeGroups: map[string]QEMUNodeGroup{
"control-plane": {
Role: role.ControlPlane.TFString(),
InstanceCount: 1,
DiskSize: 30,
CPUCount: 4,
MemorySize: 8192,
},
},
Machine: "q35",
LibvirtURI: "qemu:///system",
LibvirtSocketPath: "/var/run/libvirt/libvirt-sock",
BootMode: "uefi",
CPUCount: 4,
MemorySizeMiB: 8192,
ImagePath: "/var/lib/libvirt/images/cluster-name.qcow2",
ImageFormat: "raw",
MetadataAPIImage: "example.com/metadata-api:latest",
MetadataLibvirtURI: "qemu:///system",
NVRAM: "production",
Firmware: "/usr/share/OVMF/OVMF_CODE.fd",
BzImagePath: "/var/lib/libvirt/images/cluster-name-bzimage",
InitrdPath: "/var/lib/libvirt/images/cluster-name-initrd",
KernelCmdline: "console=ttyS0,115200n8",
InitrdPath: toPtr("/var/lib/libvirt/images/cluster-name-initrd"),
KernelCmdline: toPtr("console=ttyS0,115200n8"),
}
// test that the variables are correctly rendered
want := `name = "cluster-name"
control_plane_count = 1
worker_count = 2
state_disk_size = 30
libvirt_uri = "qemu:///system"
libvirt_socket_path = "/var/run/libvirt/libvirt-sock"
constellation_os_image = "/var/lib/libvirt/images/cluster-name.qcow2"
image_format = "raw"
node_groups = {
control-plane = {
disk_size = 30
instance_count = 1
memory = 8192
role = "control-plane"
vcpus = 4
}
}
machine = "q35"
libvirt_uri = "qemu:///system"
libvirt_socket_path = "/var/run/libvirt/libvirt-sock"
constellation_boot_mode = "uefi"
constellation_kernel = "/var/lib/libvirt/images/cluster-name-bzimage"
constellation_initrd = "/var/lib/libvirt/images/cluster-name-initrd"
constellation_cmdline = "console=ttyS0,115200n8"
vcpus = 4
memory = 8192
metadata_api_image = "example.com/metadata-api:latest"
metadata_libvirt_uri = "qemu:///system"
nvram = "/usr/share/OVMF/constellation_vars.production.fd"
firmware = "/usr/share/OVMF/OVMF_CODE.fd"
constellation_os_image = "/var/lib/libvirt/images/cluster-name.qcow2"
image_format = "raw"
metadata_api_image = "example.com/metadata-api:latest"
metadata_libvirt_uri = "qemu:///system"
nvram = "/usr/share/OVMF/constellation_vars.production.fd"
constellation_initrd = "/var/lib/libvirt/images/cluster-name-initrd"
constellation_cmdline = "console=ttyS0,115200n8"
`
got := vars.String()
assert.Equal(t, want, got)
}
func toPtr[T any](v T) *T {
return &v
}