diff --git a/cli/internal/cloudcmd/BUILD.bazel b/cli/internal/cloudcmd/BUILD.bazel index d21a3796a..89f54a418 100644 --- a/cli/internal/cloudcmd/BUILD.bazel +++ b/cli/internal/cloudcmd/BUILD.bazel @@ -29,6 +29,7 @@ go_library( "//internal/config", "//internal/constants", "//internal/imagefetcher", + "//internal/role", "@com_github_azure_azure_sdk_for_go//profiles/latest/attestation/attestation", "@com_github_azure_azure_sdk_for_go_sdk_azcore//policy", "@com_github_azure_azure_sdk_for_go_sdk_azidentity//:azidentity", diff --git a/cli/internal/cloudcmd/create.go b/cli/internal/cloudcmd/create.go index 87d956b4a..a4fe8870b 100644 --- a/cli/internal/cloudcmd/create.go +++ b/cli/internal/cloudcmd/create.go @@ -26,6 +26,7 @@ import ( "github.com/edgelesssys/constellation/v2/internal/config" "github.com/edgelesssys/constellation/v2/internal/constants" "github.com/edgelesssys/constellation/v2/internal/imagefetcher" + "github.com/edgelesssys/constellation/v2/internal/role" ) // Creator creates cloud resources. @@ -214,25 +215,35 @@ func (c *Creator) createGCP(ctx context.Context, cl terraformClient, opts Create func (c *Creator) createAzure(ctx context.Context, cl terraformClient, opts CreateOptions) (idFile clusterid.File, retErr error) { vars := terraform.AzureClusterVariables{ - CommonVariables: terraform.CommonVariables{ - Name: opts.Config.Name, - CountControlPlanes: opts.ControlPlaneCount, - CountWorkers: opts.WorkerCount, - StateDiskSizeGB: opts.Config.StateDiskSizeGB, + Name: opts.Config.Name, + NodeGroups: map[string]terraform.AzureNodeGroup{ + "control_plane_default": { + Role: role.ControlPlane.TFString(), + InstanceCount: toPtr(opts.ControlPlaneCount), + InstanceType: opts.InsType, + DiskSizeGB: opts.Config.StateDiskSizeGB, + DiskType: opts.Config.Provider.Azure.StateDiskType, + Zones: nil, // TODO(elchead): support zones AB#3225 + }, + "worker_default": { + Role: role.Worker.TFString(), + InstanceCount: toPtr(opts.WorkerCount), + InstanceType: opts.InsType, + DiskSizeGB: opts.Config.StateDiskSizeGB, + DiskType: opts.Config.Provider.Azure.StateDiskType, + Zones: nil, + }, }, Location: opts.Config.Provider.Azure.Location, - ResourceGroup: opts.Config.Provider.Azure.ResourceGroup, - UserAssignedIdentity: opts.Config.Provider.Azure.UserAssignedIdentity, - InstanceType: opts.InsType, - StateDiskType: opts.Config.Provider.Azure.StateDiskType, ImageID: opts.image, - SecureBoot: *opts.Config.Provider.Azure.SecureBoot, - CreateMAA: opts.Config.GetAttestationConfig().GetVariant().Equal(variant.AzureSEVSNP{}), - Debug: opts.Config.IsDebugCluster(), + CreateMAA: toPtr(opts.Config.GetAttestationConfig().GetVariant().Equal(variant.AzureSEVSNP{})), + Debug: toPtr(opts.Config.IsDebugCluster()), + ConfidentialVM: toPtr(opts.Config.GetAttestationConfig().GetVariant().Equal(variant.AzureSEVSNP{})), + SecureBoot: opts.Config.Provider.Azure.SecureBoot, + UserAssignedIdentity: opts.Config.Provider.Azure.UserAssignedIdentity, + ResourceGroup: opts.Config.Provider.Azure.ResourceGroup, } - vars.ConfidentialVM = opts.Config.GetAttestationConfig().GetVariant().Equal(variant.AzureSEVSNP{}) - vars = normalizeAzureURIs(vars) if err := cl.PrepareWorkspace(path.Join("terraform", strings.ToLower(cloudprovider.Azure.String())), &vars); err != nil { @@ -245,7 +256,7 @@ func (c *Creator) createAzure(ctx context.Context, cl terraformClient, opts Crea return clusterid.File{}, err } - if vars.CreateMAA { + if vars.CreateMAA != nil && *vars.CreateMAA { // Patch the attestation policy to allow the cluster to boot while having secure boot disabled. if err := c.policyPatcher.Patch(ctx, tfOutput.AttestationURL); err != nil { return clusterid.File{}, err @@ -442,3 +453,7 @@ func (c *Creator) createQEMU(ctx context.Context, cl terraformClient, lv libvirt UID: tfOutput.UID, }, nil } + +func toPtr[T any](v T) *T { + return &v +} diff --git a/cli/internal/cmd/upgradeapply.go b/cli/internal/cmd/upgradeapply.go index 0fbb2a7fb..401de35a2 100644 --- a/cli/internal/cmd/upgradeapply.go +++ b/cli/internal/cmd/upgradeapply.go @@ -156,6 +156,10 @@ func (u *upgradeApplyCmd) migrateTerraform(cmd *cobra.Command, file file.Handler if err != nil { return fmt.Errorf("parsing upgrade variables: %w", err) } + if len(targets) == 0 { + u.log.Debugf("No targets specified. Skipping Terraform migration") + return nil + } u.log.Debugf("Using migration targets:\n%v", targets) u.log.Debugf("Using Terraform variables:\n%v", vars) @@ -240,7 +244,7 @@ func parseTerraformUpgradeVars(cmd *cobra.Command, conf *config.Config, fetcher } return targets, vars, nil case cloudprovider.Azure: - targets := []string{"azurerm_attestation_provider.attestation_provider"} + targets := []string{"azurerm_attestation_provider.attestation_provider", "module.scale_set_group", "module.scale_set_control_plane", "module.scale_set_worker"} // Azure Terraform provider is very strict about it's casing imageRef = strings.Replace(imageRef, "CommunityGalleries", "communityGalleries", 1) @@ -248,16 +252,28 @@ func parseTerraformUpgradeVars(cmd *cobra.Command, conf *config.Config, fetcher imageRef = strings.Replace(imageRef, "Versions", "versions", 1) vars := &terraform.AzureClusterVariables{ - CommonVariables: commonVariables, - Location: conf.Provider.Azure.Location, + Name: conf.Name, ResourceGroup: conf.Provider.Azure.ResourceGroup, UserAssignedIdentity: conf.Provider.Azure.UserAssignedIdentity, - InstanceType: conf.Provider.Azure.InstanceType, - StateDiskType: conf.Provider.Azure.StateDiskType, ImageID: imageRef, - SecureBoot: *conf.Provider.Azure.SecureBoot, - CreateMAA: conf.GetAttestationConfig().GetVariant().Equal(variant.AzureSEVSNP{}), - Debug: conf.IsDebugCluster(), + NodeGroups: map[string]terraform.AzureNodeGroup{ + "control_plane_default": { + Role: "control-plane", + InstanceType: conf.Provider.Azure.InstanceType, + DiskSizeGB: conf.StateDiskSizeGB, + DiskType: conf.Provider.Azure.StateDiskType, + }, + "worker_default": { + Role: "worker", + InstanceType: conf.Provider.Azure.InstanceType, + DiskSizeGB: conf.StateDiskSizeGB, + DiskType: conf.Provider.Azure.StateDiskType, + }, + }, + Location: conf.Provider.Azure.Location, + SecureBoot: conf.Provider.Azure.SecureBoot, + CreateMAA: toPtr(conf.GetAttestationConfig().GetVariant().Equal(variant.AzureSEVSNP{})), + Debug: toPtr(conf.IsDebugCluster()), } return targets, vars, nil case cloudprovider.GCP: @@ -427,3 +443,7 @@ type cloudUpgrader interface { CheckTerraformMigrations(fileHandler file.Handler) error CleanUpTerraformMigrations(fileHandler file.Handler) error } + +func toPtr[T any](v T) *T { + return &v +} diff --git a/cli/internal/terraform/BUILD.bazel b/cli/internal/terraform/BUILD.bazel index 81892bd1c..1462c6018 100644 --- a/cli/internal/terraform/BUILD.bazel +++ b/cli/internal/terraform/BUILD.bazel @@ -103,6 +103,7 @@ go_test( "//internal/cloud/cloudprovider", "//internal/constants", "//internal/file", + "@com_github_azure_azure_sdk_for_go_sdk_azcore//to", "@com_github_hashicorp_terraform_exec//tfexec", "@com_github_hashicorp_terraform_json//:terraform-json", "@com_github_spf13_afero//:afero", diff --git a/cli/internal/terraform/terraform/azure/main.tf b/cli/internal/terraform/terraform/azure/main.tf index 9e726196d..774998e27 100644 --- a/cli/internal/terraform/terraform/azure/main.tf +++ b/cli/internal/terraform/terraform/azure/main.tf @@ -221,58 +221,46 @@ resource "azurerm_network_security_group" "security_group" { } } -module "scale_set_control_plane" { - source = "./modules/scale_set" - - name = "${local.name}-control-plane" - instance_count = var.control_plane_count - state_disk_size = var.state_disk_size - state_disk_type = var.state_disk_type - resource_group = var.resource_group - location = var.location - instance_type = var.instance_type - confidential_vm = var.confidential_vm - secure_boot = var.secure_boot +module "scale_set_group" { + source = "./modules/scale_set" + for_each = var.node_groups + base_name = local.name + node_group_name = each.key + role = each.value.role + zones = each.value.zones tags = merge( local.tags, - { constellation-role = "control-plane" }, { constellation-init-secret-hash = local.initSecretHash }, { constellation-maa-url = var.create_maa ? azurerm_attestation_provider.attestation_provider[0].attestation_uri : "" }, ) - image_id = var.image_id + + instance_count = each.value.instance_count + state_disk_size = each.value.disk_size + state_disk_type = each.value.disk_type + location = var.location + instance_type = each.value.instance_type + confidential_vm = var.confidential_vm + secure_boot = var.secure_boot + resource_group = var.resource_group user_assigned_identity = var.user_assigned_identity + image_id = var.image_id network_security_group_id = azurerm_network_security_group.security_group.id subnet_id = azurerm_subnet.node_subnet.id - backend_address_pool_ids = [ + backend_address_pool_ids = each.value.role == "control-plane" ? [ azurerm_lb_backend_address_pool.all.id, module.loadbalancer_backend_control_plane.backendpool_id - ] -} - -module "scale_set_worker" { - source = "./modules/scale_set" - - name = "${local.name}-worker" - instance_count = var.worker_count - state_disk_size = var.state_disk_size - state_disk_type = var.state_disk_type - resource_group = var.resource_group - location = var.location - instance_type = var.instance_type - confidential_vm = var.confidential_vm - secure_boot = var.secure_boot - tags = merge( - local.tags, - { constellation-role = "worker" }, - { constellation-init-secret-hash = local.initSecretHash }, - { constellation-maa-url = var.create_maa ? azurerm_attestation_provider.attestation_provider[0].attestation_uri : "" }, - ) - image_id = var.image_id - user_assigned_identity = var.user_assigned_identity - network_security_group_id = azurerm_network_security_group.security_group.id - subnet_id = azurerm_subnet.node_subnet.id - backend_address_pool_ids = [ + ] : [ azurerm_lb_backend_address_pool.all.id, - module.loadbalancer_backend_worker.backendpool_id, + module.loadbalancer_backend_worker.backendpool_id ] } + +moved { + from = module.scale_set_control_plane + to = module.scale_set_group["control_plane_default"] +} + +moved { + from = module.scale_set_worker + to = module.scale_set_group["worker_default"] +} diff --git a/cli/internal/terraform/terraform/azure/modules/scale_set/main.tf b/cli/internal/terraform/terraform/azure/modules/scale_set/main.tf index 722c68a76..b2aadd5bb 100644 --- a/cli/internal/terraform/terraform/azure/modules/scale_set/main.tf +++ b/cli/internal/terraform/terraform/azure/modules/scale_set/main.tf @@ -11,6 +11,19 @@ terraform { } } +locals { + tags = merge( + var.tags, + { constellation-role = var.role }, + { constellation-node-group = var.node_group_name }, + ) + group_uid = random_id.uid.hex + name = "${var.base_name}-${var.role}${local.group_uid}" +} + +resource "random_id" "uid" { + byte_length = 4 +} resource "random_password" "password" { length = 16 min_lower = 1 @@ -20,7 +33,7 @@ resource "random_password" "password" { } resource "azurerm_linux_virtual_machine_scale_set" "scale_set" { - name = var.name + name = local.name resource_group_name = var.resource_group location = var.location sku = var.instance_type @@ -34,8 +47,8 @@ resource "azurerm_linux_virtual_machine_scale_set" "scale_set" { upgrade_mode = "Manual" secure_boot_enabled = var.secure_boot source_image_id = var.image_id - tags = var.tags - + tags = local.tags + zones = var.zones identity { type = "UserAssigned" identity_ids = [var.user_assigned_identity] @@ -81,6 +94,7 @@ resource "azurerm_linux_virtual_machine_scale_set" "scale_set" { lifecycle { ignore_changes = [ + name, # required. Allow legacy scale sets to keep their old names instances, # required. autoscaling modifies the instance count externally source_image_id, # required. update procedure modifies the image id externally ] diff --git a/cli/internal/terraform/terraform/azure/modules/scale_set/variables.tf b/cli/internal/terraform/terraform/azure/modules/scale_set/variables.tf index 151b5cb11..80be273bb 100644 --- a/cli/internal/terraform/terraform/azure/modules/scale_set/variables.tf +++ b/cli/internal/terraform/terraform/azure/modules/scale_set/variables.tf @@ -1,7 +1,31 @@ -variable "name" { +variable "base_name" { type = string - default = "constell" - description = "Base name of the cluster." + description = "Base name of the instance group." +} + +variable "node_group_name" { + type = string + description = "Constellation name for the node group (used for configuration and CSP-independent naming)." +} + +variable "role" { + type = string + description = "The role of the instance group." + validation { + condition = contains(["control-plane", "worker"], var.role) + error_message = "The role has to be 'control-plane' or 'worker'." + } +} + +variable "tags" { + type = map(string) + description = "Tags to include in the scale_set." +} + +variable "zones" { + type = list(string) + description = "List of availability zones." + default = null } variable "instance_count" { @@ -61,11 +85,6 @@ variable "subnet_id" { description = "The ID of the subnet to use for the scale set." } -variable "tags" { - type = map(string) - description = "The tags to add to the scale set." -} - variable "confidential_vm" { type = bool default = true diff --git a/cli/internal/terraform/terraform/azure/variables.tf b/cli/internal/terraform/terraform/azure/variables.tf index 0d87ba765..ff91cc0fa 100644 --- a/cli/internal/terraform/terraform/azure/variables.tf +++ b/cli/internal/terraform/terraform/azure/variables.tf @@ -1,28 +1,22 @@ variable "name" { type = string - default = "constell" description = "Base name of the cluster." } -variable "control_plane_count" { - type = number - description = "The number of control plane nodes to deploy." -} - -variable "worker_count" { - type = number - description = "The number of worker nodes to deploy." -} - -variable "state_disk_size" { - type = number - default = 30 - description = "The size of the state disk in GB." -} - -variable "resource_group" { - type = string - description = "The name of the Azure resource group to create the Constellation cluster in." +variable "node_groups" { + type = map(object({ + role = string + instance_count = optional(number) + instance_type = string + disk_size = number + disk_type = string + zones = optional(list(string)) + })) + description = "A map of node group names to node group configurations." + validation { + condition = can([for group in var.node_groups : group.role == "control-plane" || group.role == "worker"]) + error_message = "The role has to be 'control-plane' or 'worker'." + } } variable "location" { @@ -30,27 +24,23 @@ variable "location" { description = "The Azure location to deploy the cluster in." } -variable "user_assigned_identity" { - type = string - description = "The name of the user assigned identity to attache to the nodes of the cluster." -} - -variable "instance_type" { - type = string - description = "The Azure instance type to deploy." -} - -variable "state_disk_type" { - type = string - default = "Premium_LRS" - description = "The type of the state disk." -} - variable "image_id" { type = string description = "The image to use for the cluster nodes." } +variable "create_maa" { + type = bool + default = false + description = "Whether to create a Microsoft Azure attestation provider." +} + +variable "debug" { + type = bool + default = false + description = "Enable debug mode. This opens up a debugd port that can be used to deploy a custom bootstrapper." +} + variable "confidential_vm" { type = bool default = true @@ -63,14 +53,11 @@ variable "secure_boot" { description = "Whether to deploy the cluster nodes with secure boot." } -variable "create_maa" { - type = bool - default = false - description = "Whether to create a Microsoft Azure attestation provider." +variable "resource_group" { + type = string + description = "The name of the Azure resource group to create the Constellation cluster in." } - -variable "debug" { - type = bool - default = false - description = "Enable debug mode. This opens up a debugd port that can be used to deploy a custom bootstrapper." +variable "user_assigned_identity" { + type = string + description = "The name of the user assigned identity to attache to the nodes of the cluster." } diff --git a/cli/internal/terraform/variables.go b/cli/internal/terraform/variables.go index c5759fe44..8fbfb024f 100644 --- a/cli/internal/terraform/variables.go +++ b/cli/internal/terraform/variables.go @@ -162,47 +162,45 @@ func (v *GCPIAMVariables) String() string { // AzureClusterVariables is user configuration for creating a cluster with Terraform on Azure. type AzureClusterVariables struct { - // CommonVariables contains common variables. - CommonVariables - - // ResourceGroup is the name of the Azure resource group to use. - ResourceGroup string - // Location is the Azure location to use. - Location string - // UserAssignedIdentity is the name of the Azure user-assigned identity to use. - UserAssignedIdentity string - // InstanceType is the Azure instance type to use. - InstanceType string - // StateDiskType is the Azure disk type to use for the state disk. - StateDiskType string + // Name of the cluster. + Name string `hcl:"name" cty:"name"` // ImageID is the ID of the Azure image to use. - ImageID string - // ConfidentialVM sets the VM to be confidential. - ConfidentialVM bool - // SecureBoot sets the VM to use secure boot. - SecureBoot bool + ImageID string `hcl:"image_id" cty:"image_id"` // CreateMAA sets whether a Microsoft Azure attestation provider should be created. - CreateMAA bool + CreateMAA *bool `hcl:"create_maa" cty:"create_maa"` // Debug is true if debug mode is enabled. - Debug bool + Debug *bool `hcl:"debug" cty:"debug"` + // ResourceGroup is the name of the Azure resource group to use. + ResourceGroup string `hcl:"resource_group" cty:"resource_group"` + // Location is the Azure location to use. + Location string `hcl:"location" cty:"location"` + // UserAssignedIdentity is the name of the Azure user-assigned identity to use. + UserAssignedIdentity string `hcl:"user_assigned_identity" cty:"user_assigned_identity"` + // ConfidentialVM sets the VM to be confidential. + ConfidentialVM *bool `hcl:"confidential_vm" cty:"confidential_vm"` + // SecureBoot sets the VM to use secure boot. + SecureBoot *bool `hcl:"secure_boot" cty:"secure_boot"` + // NodeGroups is a map of node groups to create. + NodeGroups map[string]AzureNodeGroup `hcl:"node_groups" cty:"node_groups"` } // String returns a string representation of the variables, formatted as Terraform variables. func (v *AzureClusterVariables) String() string { - b := &strings.Builder{} - b.WriteString(v.CommonVariables.String()) - writeLinef(b, "resource_group = %q", v.ResourceGroup) - writeLinef(b, "location = %q", v.Location) - writeLinef(b, "user_assigned_identity = %q", v.UserAssignedIdentity) - writeLinef(b, "instance_type = %q", v.InstanceType) - writeLinef(b, "state_disk_type = %q", v.StateDiskType) - writeLinef(b, "image_id = %q", v.ImageID) - writeLinef(b, "confidential_vm = %t", v.ConfidentialVM) - writeLinef(b, "secure_boot = %t", v.SecureBoot) - writeLinef(b, "create_maa = %t", v.CreateMAA) - writeLinef(b, "debug = %t", v.Debug) + f := hclwrite.NewEmptyFile() + gohcl.EncodeIntoBody(v, f.Body()) + return string(f.Bytes()) +} - return b.String() +// AzureNodeGroup is a node group to create on Azure. +type AzureNodeGroup struct { + // Role is the role of the node group. + Role string `hcl:"role" cty:"role"` + // InstanceCount is optional for upgrades. + InstanceCount *int `hcl:"instance_count" cty:"instance_count"` + InstanceType string `hcl:"instance_type" cty:"instance_type"` + DiskSizeGB int `hcl:"disk_size" cty:"disk_size"` + DiskType string `hcl:"disk_type" cty:"disk_type"` + Zones *[]string `hcl:"zones" cty:"zones"` } // AzureIAMVariables is user configuration for creating the IAM configuration with Terraform on Microsoft Azure. diff --git a/cli/internal/terraform/variables_test.go b/cli/internal/terraform/variables_test.go index 6c0be6e2b..645d09a03 100644 --- a/cli/internal/terraform/variables_test.go +++ b/cli/internal/terraform/variables_test.go @@ -9,6 +9,7 @@ package terraform import ( "testing" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/to" "github.com/stretchr/testify/assert" ) @@ -142,39 +143,44 @@ service_account_id = "my-service-account" func TestAzureClusterVariables(t *testing.T) { vars := AzureClusterVariables{ - CommonVariables: CommonVariables{ - Name: "cluster-name", - CountControlPlanes: 1, - CountWorkers: 2, - StateDiskSizeGB: 30, + Name: "cluster-name", + NodeGroups: map[string]AzureNodeGroup{ + "control_plane_default": { + Role: "ControlPlane", + InstanceCount: to.Ptr(1), + InstanceType: "Standard_D2s_v3", + DiskType: "StandardSSD_LRS", + DiskSizeGB: 100, + }, }, + ConfidentialVM: to.Ptr(true), ResourceGroup: "my-resource-group", - Location: "eu-central-1", UserAssignedIdentity: "my-user-assigned-identity", - InstanceType: "Standard_D2s_v3", - StateDiskType: "StandardSSD_LRS", ImageID: "image-0123456789abcdef", - ConfidentialVM: true, - SecureBoot: false, - CreateMAA: true, - Debug: true, + CreateMAA: to.Ptr(true), + Debug: to.Ptr(true), + Location: "eu-central-1", } // test that the variables are correctly rendered - want := `name = "cluster-name" -control_plane_count = 1 -worker_count = 2 -state_disk_size = 30 -resource_group = "my-resource-group" -location = "eu-central-1" + want := `name = "cluster-name" +image_id = "image-0123456789abcdef" +create_maa = true +debug = true +resource_group = "my-resource-group" +location = "eu-central-1" user_assigned_identity = "my-user-assigned-identity" -instance_type = "Standard_D2s_v3" -state_disk_type = "StandardSSD_LRS" -image_id = "image-0123456789abcdef" -confidential_vm = true -secure_boot = false -create_maa = true -debug = true +confidential_vm = true +node_groups = { + control_plane_default = { + disk_size = 100 + disk_type = "StandardSSD_LRS" + instance_count = 1 + instance_type = "Standard_D2s_v3" + role = "ControlPlane" + zones = null + } +} ` got := vars.String() assert.Equal(t, want, got) diff --git a/internal/role/role.go b/internal/role/role.go index fdb868cce..4288bdae8 100644 --- a/internal/role/role.go +++ b/internal/role/role.go @@ -25,6 +25,18 @@ const ( Worker ) +// TFString returns the role as a string for Terraform. +func (r Role) TFString() string { + switch r { + case ControlPlane: + return "control-plane" + case Worker: + return "worker" + default: + return "unknown" + } +} + // MarshalJSON marshals the Role to JSON string. func (r Role) MarshalJSON() ([]byte, error) { return json.Marshal(r.String())