terraform: gcp node groups (#1941)

* terraform: GCP node groups

* cli: marshal GCP node groups to terraform variables

This does not have any side effects for users.
We still strictly create one control-plane and one worker group.
This is a preparation for enabling customizable node groups in the future.

bazel/toolchains/go_module_deps.bzl

@@ -97,6 +97,14 @@ def go_dependencies():
         sum = "h1:rFw4nCn9iMW+Vajsk51NtYIcwSTkXr+JGrMd36kTDJw=",
         version = "v0.0.0-20180502004556-fa1af6a1f4f5",
     )
+    go_repository(
+        name = "com_github_agext_levenshtein",
+        build_file_generation = "on",
+        build_file_proto_mode = "disable_global",
+        importpath = "github.com/agext/levenshtein",
+        sum = "h1:QmvMAjj2aEICytGiWzmxoE0x2KZvE0fvmqMOfy2tjT8=",
+        version = "v1.2.1",
+    )
 
     go_repository(
         name = "com_github_alcortesm_tgz",
@@ -249,6 +257,14 @@ def go_dependencies():
         sum = "h1:I4z+fAUqvKfvZV/CHi5dV0QuwbmIvYYFDjG0Ss5QpAs=",
         version = "v0.2.0",
     )
+    go_repository(
+        name = "com_github_apparentlymart_go_dump",
+        build_file_generation = "on",
+        build_file_proto_mode = "disable_global",
+        importpath = "github.com/apparentlymart/go-dump",
+        sum = "h1:ZSTrOEhiM5J5RFxEaFvMZVEAM1KvT1YzbEOwB2EAGjA=",
+        version = "v0.0.0-20180507223929-23540a00eaa3",
+    )
 
     go_repository(
         name = "com_github_apparentlymart_go_textseg",
@@ -3312,6 +3328,14 @@ def go_dependencies():
         sum = "h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4=",
         version = "v1.0.0",
     )
+    go_repository(
+        name = "com_github_hashicorp_hcl_v2",
+        build_file_generation = "on",
+        build_file_proto_mode = "disable_global",
+        importpath = "github.com/hashicorp/hcl/v2",
+        sum = "h1:z1XvSUyXd1HP10U4lrLg5e0JMVz6CPaJvAgxM0KNZVY=",
+        version = "v2.17.0",
+    )
 
     go_repository(
         name = "com_github_hashicorp_logutils",

cli/internal/cloudcmd/create.go

@@ -166,20 +166,30 @@ func (c *Creator) createAWS(ctx context.Context, cl terraformClient, opts Create
 
 func (c *Creator) createGCP(ctx context.Context, cl terraformClient, opts CreateOptions) (idFile clusterid.File, retErr error) {
 	vars := terraform.GCPClusterVariables{
-		CommonVariables: terraform.CommonVariables{
+		Name: opts.Config.Name,
-			Name:               opts.Config.Name,
+		NodeGroups: map[string]terraform.GCPNodeGroup{
-			CountControlPlanes: opts.ControlPlaneCount,
+			"control_plane_default": {
-			CountWorkers:       opts.WorkerCount,
+				Role:            "ControlPlane",
-			StateDiskSizeGB:    opts.Config.StateDiskSizeGB,
+				StateDiskSizeGB: opts.Config.StateDiskSizeGB,
+				InitialCount:    opts.ControlPlaneCount,
+				Zone:            opts.Config.Provider.GCP.Zone,
+				InstanceType:    opts.InsType,
+				DiskType:        opts.Config.Provider.GCP.StateDiskType,
+			},
+			"worker_default": {
+				Role:            "Worker",
+				StateDiskSizeGB: opts.Config.StateDiskSizeGB,
+				InitialCount:    opts.WorkerCount,
+				Zone:            opts.Config.Provider.GCP.Zone,
+				InstanceType:    opts.InsType,
+				DiskType:        opts.Config.Provider.GCP.StateDiskType,
+			},
 		},
-		Project:         opts.Config.Provider.GCP.Project,
+		Project: opts.Config.Provider.GCP.Project,
-		Region:          opts.Config.Provider.GCP.Region,
+		Region:  opts.Config.Provider.GCP.Region,
-		Zone:            opts.Config.Provider.GCP.Zone,
+		Zone:    opts.Config.Provider.GCP.Zone,
-		CredentialsFile: opts.Config.Provider.GCP.ServiceAccountKeyPath,
+		ImageID: opts.image,
-		InstanceType:    opts.InsType,
+		Debug:   opts.Config.IsDebugCluster(),
-		StateDiskType:   opts.Config.Provider.GCP.StateDiskType,
-		ImageID:         opts.image,
-		Debug:           opts.Config.IsDebugCluster(),
 	}
 
 	if err := cl.PrepareWorkspace(path.Join("terraform", strings.ToLower(cloudprovider.GCP.String())), &vars); err != nil {

cli/internal/cmd/upgradeapply.go

@@ -172,6 +172,7 @@ func (u *upgradeApplyCmd) migrateTerraform(cmd *cobra.Command, file file.Handler
 
 	if hasDiff {
 		// If there are any Terraform migrations to apply, ask for confirmation
+		fmt.Fprintln(cmd.OutOrStdout(), "The upgrade requires a migration of Constellation cloud resources by applying an updated Terraform template. Please manually review the suggested changes below.")
 		if !flags.yes {
 			ok, err := askToConfirm(cmd, "Do you want to apply the Terraform migrations?")
 			if err != nil {
@@ -258,15 +259,28 @@ func (u *upgradeApplyCmd) parseUpgradeVars(cmd *cobra.Command, conf *config.Conf
 		targets := []string{}
 
 		vars := &terraform.GCPClusterVariables{
-			CommonVariables: commonVariables,
+			Name: conf.Name,
-			Project:         conf.Provider.GCP.Project,
+			NodeGroups: map[string]terraform.GCPNodeGroup{
-			Region:          conf.Provider.GCP.Region,
+				"control_plane_default": {
-			Zone:            conf.Provider.GCP.Zone,
+					Role:            "ControlPlane",
-			CredentialsFile: conf.Provider.GCP.ServiceAccountKeyPath,
+					StateDiskSizeGB: conf.StateDiskSizeGB,
-			InstanceType:    conf.Provider.GCP.InstanceType,
+					Zone:            conf.Provider.GCP.Zone,
-			StateDiskType:   conf.Provider.GCP.StateDiskType,
+					InstanceType:    conf.Provider.GCP.InstanceType,
-			ImageID:         imageRef,
+					DiskType:        conf.Provider.GCP.StateDiskType,
-			Debug:           conf.IsDebugCluster(),
+				},
+				"worker_default": {
+					Role:            "Worker",
+					StateDiskSizeGB: conf.StateDiskSizeGB,
+					Zone:            conf.Provider.GCP.Zone,
+					InstanceType:    conf.Provider.GCP.InstanceType,
+					DiskType:        conf.Provider.GCP.StateDiskType,
+				},
+			},
+			Project: conf.Provider.GCP.Project,
+			Region:  conf.Provider.GCP.Region,
+			Zone:    conf.Provider.GCP.Zone,
+			ImageID: imageRef,
+			Debug:   conf.IsDebugCluster(),
 		}
 		return targets, vars, nil
 	default:

cli/internal/terraform/BUILD.bazel

@@ -83,6 +83,8 @@ go_library(
         "@com_github_hashicorp_hc_install//product",
         "@com_github_hashicorp_hc_install//releases",
         "@com_github_hashicorp_hc_install//src",
+        "@com_github_hashicorp_hcl_v2//gohcl",
+        "@com_github_hashicorp_hcl_v2//hclwrite",
         "@com_github_hashicorp_terraform_exec//tfexec",
         "@com_github_hashicorp_terraform_json//:terraform-json",
         "@com_github_spf13_afero//:afero",
@@ -94,6 +96,7 @@ go_test(
     srcs = [
         "loader_test.go",
         "terraform_test.go",
+        "variables_test.go",
     ],
     embed = [":terraform"],
     deps = [
@@ -105,5 +108,6 @@ go_test(
         "@com_github_spf13_afero//:afero",
         "@com_github_stretchr_testify//assert",
         "@com_github_stretchr_testify//require",
+        "@org_uber_go_goleak//:goleak",
     ],
 )

cli/internal/terraform/terraform/gcp/main.tf

@@ -44,6 +44,20 @@ locals {
   cidr_vpc_subnet_nodes = "192.168.178.0/24"
   cidr_vpc_subnet_pods  = "10.10.0.0/16"
   kube_env              = "AUTOSCALER_ENV_VARS: kube_reserved=cpu=1060m,memory=1019Mi,ephemeral-storage=41Gi;node_labels=;os=linux;os_distribution=cos;evictionHard="
+  control_plane_named_ports = flatten([
+    { name = "kubernetes", port = local.ports_kubernetes },
+    { name = "bootstrapper", port = local.ports_bootstrapper },
+    { name = "verify", port = local.ports_verify },
+    { name = "konnectivity", port = local.ports_konnectivity },
+    { name = "recovery", port = local.ports_recovery },
+    var.debug ? [{ name = "debugd", port = local.ports_debugd }] : [],
+  ])
+  node_groups_by_role = {
+    for name, node_group in var.node_groups : node_group.role => name...
+  }
+  control_plane_instance_groups = [
+    for control_plane in local.node_groups_by_role["ControlPlane"] : module.instance_group[control_plane].instance_group
+  ]
 }
 
 resource "random_id" "uid" {
@@ -134,48 +148,26 @@ resource "google_compute_firewall" "firewall_internal_pods" {
   allow { protocol = "icmp" }
 }
 
-module "instance_group_control_plane" {
-  source              = "./modules/instance_group"
-  name                = local.name
-  role                = "ControlPlane"
-  uid                 = local.uid
-  instance_type       = var.instance_type
-  instance_count      = var.control_plane_count
-  image_id            = var.image_id
-  disk_size           = var.state_disk_size
-  disk_type           = var.state_disk_type
-  network             = google_compute_network.vpc_network.id
-  subnetwork          = google_compute_subnetwork.vpc_subnetwork.id
-  alias_ip_range_name = google_compute_subnetwork.vpc_subnetwork.secondary_ip_range[0].range_name
-  kube_env            = local.kube_env
-  debug               = var.debug
-  named_ports = flatten([
-    { name = "kubernetes", port = local.ports_kubernetes },
-    { name = "bootstrapper", port = local.ports_bootstrapper },
-    { name = "verify", port = local.ports_verify },
-    { name = "konnectivity", port = local.ports_konnectivity },
-    { name = "recovery", port = local.ports_recovery },
-    var.debug ? [{ name = "debugd", port = local.ports_debugd }] : [],
-  ])
-  labels           = local.labels
-  init_secret_hash = local.initSecretHash
-}
 
-module "instance_group_worker" {
+module "instance_group" {
   source              = "./modules/instance_group"
-  name                = "${local.name}-1"
+  for_each            = var.node_groups
-  role                = "Worker"
+  base_name           = local.name
+  node_group_name     = each.key
+  role                = each.value.role
+  zone                = each.value.zone
   uid                 = local.uid
-  instance_type       = var.instance_type
+  instance_type       = each.value.instance_type
-  instance_count      = var.worker_count
+  instance_count      = each.value.initial_count
   image_id            = var.image_id
-  disk_size           = var.state_disk_size
+  disk_size           = each.value.disk_size
-  disk_type           = var.state_disk_type
+  disk_type           = each.value.disk_type
   network             = google_compute_network.vpc_network.id
   subnetwork          = google_compute_subnetwork.vpc_subnetwork.id
   alias_ip_range_name = google_compute_subnetwork.vpc_subnetwork.secondary_ip_range[0].range_name
   kube_env            = local.kube_env
   debug               = var.debug
+  named_ports         = each.value.role == "ControlPlane" ? local.control_plane_named_ports : []
   labels              = local.labels
   init_secret_hash    = local.initSecretHash
 }
@@ -185,68 +177,78 @@ resource "google_compute_global_address" "loadbalancer_ip" {
 }
 
 module "loadbalancer_kube" {
-  source                 = "./modules/loadbalancer"
+  source                  = "./modules/loadbalancer"
-  name                   = local.name
+  name                    = local.name
-  health_check           = "HTTPS"
+  health_check            = "HTTPS"
-  backend_port_name      = "kubernetes"
+  backend_port_name       = "kubernetes"
-  backend_instance_group = module.instance_group_control_plane.instance_group
+  backend_instance_groups = local.control_plane_instance_groups
-  ip_address             = google_compute_global_address.loadbalancer_ip.self_link
+  ip_address              = google_compute_global_address.loadbalancer_ip.self_link
-  port                   = local.ports_kubernetes
+  port                    = local.ports_kubernetes
-  frontend_labels        = merge(local.labels, { constellation-use = "kubernetes" })
+  frontend_labels         = merge(local.labels, { constellation-use = "kubernetes" })
 }
 
 module "loadbalancer_boot" {
-  source                 = "./modules/loadbalancer"
+  source                  = "./modules/loadbalancer"
-  name                   = local.name
+  name                    = local.name
-  health_check           = "TCP"
+  health_check            = "TCP"
-  backend_port_name      = "bootstrapper"
+  backend_port_name       = "bootstrapper"
-  backend_instance_group = module.instance_group_control_plane.instance_group
+  backend_instance_groups = local.control_plane_instance_groups
-  ip_address             = google_compute_global_address.loadbalancer_ip.self_link
+  ip_address              = google_compute_global_address.loadbalancer_ip.self_link
-  port                   = local.ports_bootstrapper
+  port                    = local.ports_bootstrapper
-  frontend_labels        = merge(local.labels, { constellation-use = "bootstrapper" })
+  frontend_labels         = merge(local.labels, { constellation-use = "bootstrapper" })
 }
 
 module "loadbalancer_verify" {
-  source                 = "./modules/loadbalancer"
+  source                  = "./modules/loadbalancer"
-  name                   = local.name
+  name                    = local.name
-  health_check           = "TCP"
+  health_check            = "TCP"
-  backend_port_name      = "verify"
+  backend_port_name       = "verify"
-  backend_instance_group = module.instance_group_control_plane.instance_group
+  backend_instance_groups = local.control_plane_instance_groups
-  ip_address             = google_compute_global_address.loadbalancer_ip.self_link
+  ip_address              = google_compute_global_address.loadbalancer_ip.self_link
-  port                   = local.ports_verify
+  port                    = local.ports_verify
-  frontend_labels        = merge(local.labels, { constellation-use = "verify" })
+  frontend_labels         = merge(local.labels, { constellation-use = "verify" })
 }
 
 module "loadbalancer_konnectivity" {
-  source                 = "./modules/loadbalancer"
+  source                  = "./modules/loadbalancer"
-  name                   = local.name
+  name                    = local.name
-  health_check           = "TCP"
+  health_check            = "TCP"
-  backend_port_name      = "konnectivity"
+  backend_port_name       = "konnectivity"
-  backend_instance_group = module.instance_group_control_plane.instance_group
+  backend_instance_groups = local.control_plane_instance_groups
-  ip_address             = google_compute_global_address.loadbalancer_ip.self_link
+  ip_address              = google_compute_global_address.loadbalancer_ip.self_link
-  port                   = local.ports_konnectivity
+  port                    = local.ports_konnectivity
-  frontend_labels        = merge(local.labels, { constellation-use = "konnectivity" })
+  frontend_labels         = merge(local.labels, { constellation-use = "konnectivity" })
 }
 
 module "loadbalancer_recovery" {
-  source                 = "./modules/loadbalancer"
+  source                  = "./modules/loadbalancer"
-  name                   = local.name
+  name                    = local.name
-  health_check           = "TCP"
+  health_check            = "TCP"
-  backend_port_name      = "recovery"
+  backend_port_name       = "recovery"
-  backend_instance_group = module.instance_group_control_plane.instance_group
+  backend_instance_groups = local.control_plane_instance_groups
-  ip_address             = google_compute_global_address.loadbalancer_ip.self_link
+  ip_address              = google_compute_global_address.loadbalancer_ip.self_link
-  port                   = local.ports_recovery
+  port                    = local.ports_recovery
-  frontend_labels        = merge(local.labels, { constellation-use = "recovery" })
+  frontend_labels         = merge(local.labels, { constellation-use = "recovery" })
 }
 
 module "loadbalancer_debugd" {
-  count                  = var.debug ? 1 : 0 // only deploy debugd in debug mode
+  count                   = var.debug ? 1 : 0 // only deploy debugd in debug mode
-  source                 = "./modules/loadbalancer"
+  source                  = "./modules/loadbalancer"
-  name                   = local.name
+  name                    = local.name
-  health_check           = "TCP"
+  health_check            = "TCP"
-  backend_port_name      = "debugd"
+  backend_port_name       = "debugd"
-  backend_instance_group = module.instance_group_control_plane.instance_group
+  backend_instance_groups = local.control_plane_instance_groups
-  ip_address             = google_compute_global_address.loadbalancer_ip.self_link
+  ip_address              = google_compute_global_address.loadbalancer_ip.self_link
-  port                   = local.ports_debugd
+  port                    = local.ports_debugd
-  frontend_labels        = merge(local.labels, { constellation-use = "debugd" })
+  frontend_labels         = merge(local.labels, { constellation-use = "debugd" })
+}
+
+moved {
+  from = module.instance_group_control_plane
+  to   = module.instance_group["control_plane_default"]
+}
+
+moved {
+  from = module.instance_group_worker
+  to   = module.instance_group["worker_default"]
 }

cli/internal/terraform/terraform/gcp/modules/instance_group/main.tf

@@ -4,20 +4,42 @@ terraform {
       source  = "hashicorp/google"
       version = "4.69.1"
     }
+
+    random = {
+      source  = "hashicorp/random"
+      version = "3.5.1"
+    }
   }
 }
 
 locals {
+  # migration: allow the old node group names to work since they were created without the uid
+  # and without multiple node groups in mind
+  # node_group: worker_default => name == "<base>-1-worker"
+  # node_group: control_plane_default => name:  "<base>-control-plane"
+  # new names:
+  # node_group: foo, role: Worker => name == "<base>-worker-<uid>"
+  # node_group: bar, role: ControlPlane => name == "<base>-control-plane-<uid>"
   role_dashed     = var.role == "ControlPlane" ? "control-plane" : "worker"
-  name            = "${var.name}-${local.role_dashed}"
+  group_uid       = random_id.uid.hex
+  maybe_uid       = (var.node_group_name == "control_plane_default" || var.node_group_name == "worker_default") ? "" : "-${local.group_uid}"
+  maybe_one       = var.node_group_name == "worker_default" ? "-1" : ""
+  name            = "${var.base_name}${local.maybe_one}-${local.role_dashed}${local.maybe_uid}"
   state_disk_name = "state-disk"
 }
 
+resource "random_id" "uid" {
+  byte_length = 4
+}
+
 resource "google_compute_instance_template" "template" {
   name         = local.name
   machine_type = var.instance_type
   tags         = ["constellation-${var.uid}"] // Note that this is also applied as a label
-  labels       = merge(var.labels, { constellation-role = local.role_dashed })
+  labels = merge(var.labels, {
+    constellation-role       = local.role_dashed,
+    constellation-node-group = var.node_group_name,
+  })
 
   confidential_instance_config {
     enable_confidential_compute = true
@@ -98,6 +120,7 @@ resource "google_compute_instance_group_manager" "instance_group_manager" {
   name               = local.name
   description        = "Instance group manager for Constellation"
   base_instance_name = local.name
+  zone               = var.zone
   target_size        = var.instance_count
 
   dynamic "stateful_disk" {

cli/internal/terraform/terraform/gcp/modules/instance_group/variables.tf

@@ -1,8 +1,13 @@
-variable "name" {
+variable "base_name" {
   type        = string
   description = "Base name of the instance group."
 }
 
+variable "node_group_name" {
+  type        = string
+  description = "Constellation name for the node group (used for configuration and CSP-independent naming)."
+}
+
 variable "role" {
   type        = string
   description = "The role of the instance group."
@@ -84,3 +89,8 @@ variable "alias_ip_range_name" {
   type        = string
   description = "Name of the alias IP range to use."
 }
+
+variable "zone" {
+  type        = string
+  description = "Zone to deploy the instance group in."
+}

cli/internal/terraform/terraform/gcp/modules/loadbalancer/main.tf

@@ -41,9 +41,12 @@ resource "google_compute_backend_service" "backend" {
   port_name             = var.backend_port_name
   timeout_sec           = 240
 
-  backend {
+  dynamic "backend" {
-    group          = var.backend_instance_group
+    for_each = var.backend_instance_groups
-    balancing_mode = "UTILIZATION"
+    content {
+      group          = backend.value
+      balancing_mode = "UTILIZATION"
+    }
   }
 }
 

cli/internal/terraform/terraform/gcp/modules/loadbalancer/variables.tf

@@ -13,9 +13,9 @@ variable "backend_port_name" {
   description = "Name of backend port. The same name should appear in the instance groups referenced by this service."
 }
 
-variable "backend_instance_group" {
+variable "backend_instance_groups" {
-  type        = string
+  type        = list(string)
-  description = "The URL of the instance group resource from which the load balancer will direct traffic."
+  description = "The URLs of the instance group resources from which the load balancer will direct traffic."
 }
 
 variable "ip_address" {

cli/internal/terraform/terraform/gcp/variables.tf

@@ -4,20 +4,16 @@ variable "name" {
   description = "Base name of the cluster."
 }
 
-variable "control_plane_count" {
+variable "node_groups" {
-  type        = number
+  type = map(object({
-  description = "The number of control plane nodes to deploy."
+    role          = string
-}
+    zone          = string
-
+    instance_type = string
-variable "worker_count" {
+    disk_size     = number
-  type        = number
+    disk_type     = string
-  description = "The number of worker nodes to deploy."
+    initial_count = number
-}
+  }))
-
+  description = "A map of node group names to node group configurations."
-variable "state_disk_size" {
-  type        = number
-  default     = 30
-  description = "The size of the state disk in GB."
 }
 
 variable "project" {
@@ -35,17 +31,6 @@ variable "zone" {
   description = "The GCP zone to deploy the cluster in."
 }
 
-variable "instance_type" {
-  type        = string
-  description = "The GCP instance type to deploy."
-}
-
-variable "state_disk_type" {
-  type        = string
-  default     = "pd-ssd"
-  description = "The type of the state disk."
-}
-
 variable "image_id" {
   type        = string
   description = "The GCP image to use for the cluster nodes."

cli/internal/terraform/terraform_test.go

@@ -24,8 +24,13 @@ import (
 	"github.com/spf13/afero"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
+	"go.uber.org/goleak"
 )
 
+func TestMain(m *testing.M) {
+	goleak.VerifyTestMain(m)
+}
+
 func TestPrepareCluster(t *testing.T) {
 	qemuVars := &QEMUVariables{
 		CommonVariables: CommonVariables{

cli/internal/terraform/variables.go

@@ -9,6 +9,9 @@ package terraform
 import (
 	"fmt"
 	"strings"
+
+	"github.com/hashicorp/hcl/v2/gohcl"
+	"github.com/hashicorp/hcl/v2/hclwrite"
 )
 
 // Variables is a struct that holds all variables that are passed to Terraform.
@@ -98,40 +101,40 @@ func (v *AWSIAMVariables) String() string {
 
 // GCPClusterVariables is user configuration for creating resources with Terraform on GCP.
 type GCPClusterVariables struct {
-	// CommonVariables contains common variables.
+	// Name of the cluster.
-	CommonVariables
+	Name string `hcl:"name" cty:"name"`
-
 	// Project is the ID of the GCP project to use.
-	Project string
+	Project string `hcl:"project" cty:"project"`
 	// Region is the GCP region to use.
-	Region string
+	Region string `hcl:"region" cty:"region"`
 	// Zone is the GCP zone to use.
-	Zone string
+	Zone string `hcl:"zone" cty:"zone"`
-	// CredentialsFile is the path to the GCP credentials file.
-	CredentialsFile string
-	// InstanceType is the GCP instance type to use.
-	InstanceType string
-	// StateDiskType is the GCP disk type to use for the state disk.
-	StateDiskType string
 	// ImageID is the ID of the GCP image to use.
-	ImageID string
+	ImageID string `hcl:"image_id" cty:"image_id"`
 	// Debug is true if debug mode is enabled.
-	Debug bool
+	Debug bool `hcl:"debug" cty:"debug"`
+	// NodeGroups is a map of node groups to create.
+	NodeGroups map[string]GCPNodeGroup `hcl:"node_groups" cty:"node_groups"`
+}
+
+// GCPNodeGroup is a node group to create on GCP.
+type GCPNodeGroup struct {
+	// Role is the role of the node group.
+	Role string `hcl:"role" cty:"role"`
+	// StateDiskSizeGB is the size of the state disk to allocate to each node, in GB.
+	StateDiskSizeGB int `hcl:"disk_size" cty:"disk_size"`
+	// InitialCount is the initial number of nodes to create in the node group.
+	InitialCount int    `hcl:"initial_count" cty:"initial_count"`
+	Zone         string `hcl:"zone" cty:"zone"`
+	InstanceType string `hcl:"instance_type" cty:"instance_type"`
+	DiskType     string `hcl:"disk_type" cty:"disk_type"`
 }
 
 // String returns a string representation of the variables, formatted as Terraform variables.
 func (v *GCPClusterVariables) String() string {
-	b := &strings.Builder{}
+	f := hclwrite.NewEmptyFile()
-	b.WriteString(v.CommonVariables.String())
+	gohcl.EncodeIntoBody(v, f.Body())
-	writeLinef(b, "project = %q", v.Project)
+	return string(f.Bytes())
-	writeLinef(b, "region = %q", v.Region)
-	writeLinef(b, "zone = %q", v.Zone)
-	writeLinef(b, "instance_type = %q", v.InstanceType)
-	writeLinef(b, "state_disk_type = %q", v.StateDiskType)
-	writeLinef(b, "image_id = %q", v.ImageID)
-	writeLinef(b, "debug = %t", v.Debug)
-
-	return b.String()
 }
 
 // GCPIAMVariables is user configuration for creating the IAM confioguration with Terraform on GCP.

cli/internal/terraform/variables_test.go

@@ -0,0 +1,287 @@
+/*
+Copyright (c) Edgeless Systems GmbH
+
+SPDX-License-Identifier: AGPL-3.0-only
+*/
+
+package terraform
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestAWSClusterVariables(t *testing.T) {
+	vars := AWSClusterVariables{
+		CommonVariables: CommonVariables{
+			Name:               "cluster-name",
+			CountControlPlanes: 1,
+			CountWorkers:       2,
+			StateDiskSizeGB:    30,
+		},
+		Region:                 "eu-central-1",
+		Zone:                   "eu-central-1a",
+		AMIImageID:             "ami-0123456789abcdef",
+		InstanceType:           "x1.foo",
+		StateDiskType:          "bardisk",
+		IAMProfileControlPlane: "arn:aws:iam::123456789012:instance-profile/cluster-name-controlplane",
+		IAMProfileWorkerNodes:  "arn:aws:iam::123456789012:instance-profile/cluster-name-worker",
+		Debug:                  true,
+		EnableSNP:              true,
+	}
+
+	// test that the variables are correctly rendered
+	want := `name = "cluster-name"
+control_plane_count = 1
+worker_count = 2
+state_disk_size = 30
+region = "eu-central-1"
+zone = "eu-central-1a"
+ami = "ami-0123456789abcdef"
+instance_type = "x1.foo"
+state_disk_type = "bardisk"
+iam_instance_profile_control_plane = "arn:aws:iam::123456789012:instance-profile/cluster-name-controlplane"
+iam_instance_profile_worker_nodes = "arn:aws:iam::123456789012:instance-profile/cluster-name-worker"
+debug = true
+enable_snp = true
+`
+	got := vars.String()
+	assert.Equal(t, want, got)
+}
+
+func TestAWSIAMVariables(t *testing.T) {
+	vars := AWSIAMVariables{
+		Region: "eu-central-1",
+		Prefix: "my-prefix",
+	}
+
+	// test that the variables are correctly rendered
+	want := `name_prefix = "my-prefix"
+region = "eu-central-1"
+`
+	got := vars.String()
+	assert.Equal(t, want, got)
+}
+
+func TestGCPClusterVariables(t *testing.T) {
+	vars := GCPClusterVariables{
+		Name:    "cluster-name",
+		Project: "my-project",
+		Region:  "eu-central-1",
+		Zone:    "eu-central-1a",
+		ImageID: "image-0123456789abcdef",
+		Debug:   true,
+		NodeGroups: map[string]GCPNodeGroup{
+			"control_plane_default": {
+				Role:            "ControlPlane",
+				StateDiskSizeGB: 30,
+				InitialCount:    1,
+				Zone:            "eu-central-1a",
+				InstanceType:    "n2d-standard-4",
+				DiskType:        "pd-ssd",
+			},
+			"worker_default": {
+				Role:            "Worker",
+				StateDiskSizeGB: 10,
+				InitialCount:    1,
+				Zone:            "eu-central-1b",
+				InstanceType:    "n2d-standard-8",
+				DiskType:        "pd-ssd",
+			},
+		},
+	}
+
+	// test that the variables are correctly rendered
+	want := `name     = "cluster-name"
+project  = "my-project"
+region   = "eu-central-1"
+zone     = "eu-central-1a"
+image_id = "image-0123456789abcdef"
+debug    = true
+node_groups = {
+  control_plane_default = {
+    disk_size     = 30
+    disk_type     = "pd-ssd"
+    initial_count = 1
+    instance_type = "n2d-standard-4"
+    role          = "ControlPlane"
+    zone          = "eu-central-1a"
+  }
+  worker_default = {
+    disk_size     = 10
+    disk_type     = "pd-ssd"
+    initial_count = 1
+    instance_type = "n2d-standard-8"
+    role          = "Worker"
+    zone          = "eu-central-1b"
+  }
+}
+`
+	got := vars.String()
+	assert.Equal(t, want, got)
+}
+
+func TestGCPIAMVariables(t *testing.T) {
+	vars := GCPIAMVariables{
+		Project:          "my-project",
+		Region:           "eu-central-1",
+		Zone:             "eu-central-1a",
+		ServiceAccountID: "my-service-account",
+	}
+
+	// test that the variables are correctly rendered
+	want := `project_id = "my-project"
+region = "eu-central-1"
+zone = "eu-central-1a"
+service_account_id = "my-service-account"
+`
+	got := vars.String()
+	assert.Equal(t, want, got)
+}
+
+func TestAzureClusterVariables(t *testing.T) {
+	vars := AzureClusterVariables{
+		CommonVariables: CommonVariables{
+			Name:               "cluster-name",
+			CountControlPlanes: 1,
+			CountWorkers:       2,
+			StateDiskSizeGB:    30,
+		},
+		ResourceGroup:        "my-resource-group",
+		Location:             "eu-central-1",
+		UserAssignedIdentity: "my-user-assigned-identity",
+		InstanceType:         "Standard_D2s_v3",
+		StateDiskType:        "StandardSSD_LRS",
+		ImageID:              "image-0123456789abcdef",
+		ConfidentialVM:       true,
+		SecureBoot:           false,
+		CreateMAA:            true,
+		Debug:                true,
+	}
+
+	// test that the variables are correctly rendered
+	want := `name = "cluster-name"
+control_plane_count = 1
+worker_count = 2
+state_disk_size = 30
+resource_group = "my-resource-group"
+location = "eu-central-1"
+user_assigned_identity = "my-user-assigned-identity"
+instance_type = "Standard_D2s_v3"
+state_disk_type = "StandardSSD_LRS"
+image_id = "image-0123456789abcdef"
+confidential_vm = true
+secure_boot = false
+create_maa = true
+debug = true
+`
+	got := vars.String()
+	assert.Equal(t, want, got)
+}
+
+func TestAzureIAMVariables(t *testing.T) {
+	vars := AzureIAMVariables{
+		Region:           "eu-central-1",
+		ServicePrincipal: "my-service-principal",
+		ResourceGroup:    "my-resource-group",
+	}
+
+	// test that the variables are correctly rendered
+	want := `service_principal_name = "my-service-principal"
+region = "eu-central-1"
+resource_group_name = "my-resource-group"
+`
+	got := vars.String()
+	assert.Equal(t, want, got)
+}
+
+func TestOpenStackClusterVariables(t *testing.T) {
+	vars := OpenStackClusterVariables{
+		CommonVariables: CommonVariables{
+			Name:               "cluster-name",
+			CountControlPlanes: 1,
+			CountWorkers:       2,
+			StateDiskSizeGB:    30,
+		},
+		Cloud:                   "my-cloud",
+		AvailabilityZone:        "az-01",
+		FlavorID:                "flavor-0123456789abcdef",
+		FloatingIPPoolID:        "fip-pool-0123456789abcdef",
+		StateDiskType:           "performance-8",
+		ImageURL:                "https://example.com/image.raw",
+		DirectDownload:          true,
+		OpenstackUserDomainName: "my-user-domain",
+		OpenstackUsername:       "my-username",
+		OpenstackPassword:       "my-password",
+		Debug:                   true,
+	}
+
+	// test that the variables are correctly rendered
+	want := `name = "cluster-name"
+control_plane_count = 1
+worker_count = 2
+state_disk_size = 30
+cloud = "my-cloud"
+availability_zone = "az-01"
+flavor_id = "flavor-0123456789abcdef"
+floating_ip_pool_id = "fip-pool-0123456789abcdef"
+image_url = "https://example.com/image.raw"
+direct_download = true
+state_disk_type = "performance-8"
+openstack_user_domain_name = "my-user-domain"
+openstack_username = "my-username"
+openstack_password = "my-password"
+debug = true
+`
+	got := vars.String()
+	assert.Equal(t, want, got)
+}
+
+func TestQEMUClusterVariables(t *testing.T) {
+	vars := QEMUVariables{
+		CommonVariables: CommonVariables{
+			Name:               "cluster-name",
+			CountControlPlanes: 1,
+			CountWorkers:       2,
+			StateDiskSizeGB:    30,
+		},
+		LibvirtURI:         "qemu:///system",
+		LibvirtSocketPath:  "/var/run/libvirt/libvirt-sock",
+		BootMode:           "uefi",
+		CPUCount:           4,
+		MemorySizeMiB:      8192,
+		ImagePath:          "/var/lib/libvirt/images/cluster-name.qcow2",
+		ImageFormat:        "raw",
+		MetadataAPIImage:   "example.com/metadata-api:latest",
+		MetadataLibvirtURI: "qemu:///system",
+		NVRAM:              "production",
+		Firmware:           "/usr/share/OVMF/OVMF_CODE.fd",
+		BzImagePath:        "/var/lib/libvirt/images/cluster-name-bzimage",
+		InitrdPath:         "/var/lib/libvirt/images/cluster-name-initrd",
+		KernelCmdline:      "console=ttyS0,115200n8",
+	}
+
+	// test that the variables are correctly rendered
+	want := `name = "cluster-name"
+control_plane_count = 1
+worker_count = 2
+state_disk_size = 30
+libvirt_uri = "qemu:///system"
+libvirt_socket_path = "/var/run/libvirt/libvirt-sock"
+constellation_os_image = "/var/lib/libvirt/images/cluster-name.qcow2"
+image_format = "raw"
+constellation_boot_mode = "uefi"
+constellation_kernel = "/var/lib/libvirt/images/cluster-name-bzimage"
+constellation_initrd = "/var/lib/libvirt/images/cluster-name-initrd"
+constellation_cmdline = "console=ttyS0,115200n8"
+vcpus = 4
+memory = 8192
+metadata_api_image = "example.com/metadata-api:latest"
+metadata_libvirt_uri = "qemu:///system"
+nvram = "/usr/share/OVMF/constellation_vars.production.fd"
+firmware = "/usr/share/OVMF/OVMF_CODE.fd"
+`
+	got := vars.String()
+	assert.Equal(t, want, got)
+}

go.mod

@@ -126,6 +126,7 @@ require (
 require (
 	github.com/AdaLogics/go-fuzz-headers v0.0.0-20230106234847-43070de90fa1 // indirect
 	github.com/ProtonMail/go-crypto v0.0.0-20230217124315-7d5c6f04bbb8 // indirect
+	github.com/agext/levenshtein v1.2.1 // indirect
 	github.com/cloudflare/circl v1.3.3 // indirect
 	github.com/gabriel-vasile/mimetype v1.4.2 // indirect
 	github.com/go-logr/stdr v1.2.2 // indirect
@@ -248,6 +249,7 @@ require (
 	github.com/hashicorp/go-retryablehttp v0.7.2 // indirect
 	github.com/hashicorp/go-secure-stdlib/awsutil v0.1.6 // indirect
 	github.com/hashicorp/go-uuid v1.0.3 // indirect
+	github.com/hashicorp/hcl/v2 v2.17.0
 	github.com/huandu/xstrings v1.4.0 // indirect
 	github.com/imdario/mergo v0.3.13 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect

go.sum

@@ -188,6 +188,8 @@ github.com/VividCortex/gohistogram v1.0.0/go.mod h1:Pf5mBqqDxYaXu3hDrrU+w6nw50o/
 github.com/a8m/expect v1.0.0/go.mod h1:4IwSCMumY49ScypDnjNbYEjgVeqy1/U2cEs3Lat96eA=
 github.com/acomagu/bufpipe v1.0.4 h1:e3H4WUzM3npvo5uv95QuJM3cQspFNtFBzvJ2oNjKIDQ=
 github.com/afex/hystrix-go v0.0.0-20180502004556-fa1af6a1f4f5/go.mod h1:SkGFH1ia65gfNATL8TAiHDNxPzPdmEL5uirI2Uyuz6c=
+github.com/agext/levenshtein v1.2.1 h1:QmvMAjj2aEICytGiWzmxoE0x2KZvE0fvmqMOfy2tjT8=
+github.com/agext/levenshtein v1.2.1/go.mod h1:JEDfjyjHDjOF/1e4FlBE/PkbqA9OfWu2ki2W0IB5558=
 github.com/alcortesm/tgz v0.0.0-20161220082320-9c5fe88206d7/go.mod h1:6zEj6s6u/ghQa61ZWa/C2Aw3RkjiTBOix7dkqa1VLIs=
 github.com/alecthomas/kingpin v2.2.6+incompatible/go.mod h1:59OFYbFVLKQKq+mqrL6Rw5bR0c3ACQaawgXx0QYndlE=
 github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
@@ -841,6 +843,8 @@ github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+l
 github.com/hashicorp/hc-install v0.5.2 h1:SfwMFnEXVVirpwkDuSF5kymUOhrUxrTq3udEseZdOD0=
 github.com/hashicorp/hc-install v0.5.2/go.mod h1:9QISwe6newMWIfEiXpzuu1k9HAGtQYgnSH8H9T8wmoI=
 github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
+github.com/hashicorp/hcl/v2 v2.17.0 h1:z1XvSUyXd1HP10U4lrLg5e0JMVz6CPaJvAgxM0KNZVY=
+github.com/hashicorp/hcl/v2 v2.17.0/go.mod h1:gJyW2PTShkJqQBKpAmPO3yxMxIuoXkOF2TpqXzrQyx4=
 github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64=
 github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ=
 github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I=

hack/go.mod

@@ -87,6 +87,7 @@ require (
 	github.com/Microsoft/go-winio v0.6.0 // indirect
 	github.com/ProtonMail/go-crypto v0.0.0-20230217124315-7d5c6f04bbb8 // indirect
 	github.com/acomagu/bufpipe v1.0.4 // indirect
+	github.com/agext/levenshtein v1.2.1 // indirect
 	github.com/apparentlymart/go-textseg/v13 v13.0.0 // indirect
 	github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect
 	github.com/aws/aws-sdk-go-v2 v1.18.0 // indirect
@@ -192,6 +193,7 @@ require (
 	github.com/hashicorp/go-uuid v1.0.3 // indirect
 	github.com/hashicorp/go-version v1.6.0 // indirect
 	github.com/hashicorp/hc-install v0.5.2 // indirect
+	github.com/hashicorp/hcl/v2 v2.17.0 // indirect
 	github.com/hashicorp/terraform-exec v0.18.1 // indirect
 	github.com/hashicorp/terraform-json v0.15.0 // indirect
 	github.com/huandu/xstrings v1.4.0 // indirect

hack/go.sum

@@ -167,6 +167,8 @@ github.com/a8m/expect v1.0.0/go.mod h1:4IwSCMumY49ScypDnjNbYEjgVeqy1/U2cEs3Lat96
 github.com/acomagu/bufpipe v1.0.4 h1:e3H4WUzM3npvo5uv95QuJM3cQspFNtFBzvJ2oNjKIDQ=
 github.com/acomagu/bufpipe v1.0.4/go.mod h1:mxdxdup/WdsKVreO5GpW4+M/1CE2sMG4jeGJ2sYmHc4=
 github.com/afex/hystrix-go v0.0.0-20180502004556-fa1af6a1f4f5/go.mod h1:SkGFH1ia65gfNATL8TAiHDNxPzPdmEL5uirI2Uyuz6c=
+github.com/agext/levenshtein v1.2.1 h1:QmvMAjj2aEICytGiWzmxoE0x2KZvE0fvmqMOfy2tjT8=
+github.com/agext/levenshtein v1.2.1/go.mod h1:JEDfjyjHDjOF/1e4FlBE/PkbqA9OfWu2ki2W0IB5558=
 github.com/alcortesm/tgz v0.0.0-20161220082320-9c5fe88206d7/go.mod h1:6zEj6s6u/ghQa61ZWa/C2Aw3RkjiTBOix7dkqa1VLIs=
 github.com/alecthomas/kingpin v2.2.6+incompatible/go.mod h1:59OFYbFVLKQKq+mqrL6Rw5bR0c3ACQaawgXx0QYndlE=
 github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
@@ -821,6 +823,8 @@ github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+l
 github.com/hashicorp/hc-install v0.5.2 h1:SfwMFnEXVVirpwkDuSF5kymUOhrUxrTq3udEseZdOD0=
 github.com/hashicorp/hc-install v0.5.2/go.mod h1:9QISwe6newMWIfEiXpzuu1k9HAGtQYgnSH8H9T8wmoI=
 github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
+github.com/hashicorp/hcl/v2 v2.17.0 h1:z1XvSUyXd1HP10U4lrLg5e0JMVz6CPaJvAgxM0KNZVY=
+github.com/hashicorp/hcl/v2 v2.17.0/go.mod h1:gJyW2PTShkJqQBKpAmPO3yxMxIuoXkOF2TpqXzrQyx4=
 github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64=
 github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ=
 github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I=