Git-Mirrors/constellation
1
0
Fork 0
mirror of https://github.com/edgelesssys/constellation.git synced 2024-12-24 15:09:39 -05:00
Code Issues Packages Projects Releases Wiki Activity

terraform: azure node groups (#1955)

Browse Source 
* init

* migration working

* make tf variables with default value optional in go through ptr type

* fix CI build

* pr feedback

* add azure targets tf

* skip migration for empty targets

* make instance_count optional

* change role naming to dashed + add validation

* make node_group.zones optional

* Update cli/internal/terraform/terraform/azure/main.tf

Co-authored-by: Malte Poll <1780588+malt3@users.noreply.github.com>

* malte feedback

---------

Co-authored-by: Malte Poll <1780588+malt3@users.noreply.github.com>
This commit is contained in:
Adrian Stobbe 2023-06-22 16:53:40 +02:00 committed by GitHub
parent 224c74f883
commit 487fa1e397
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
11 changed files with 240 additions and 179 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
cli/internal/cloudcmd/BUILD.bazel
View File

@ -29,6 +29,7 @@ go_library(
"//internal/config",
"//internal/constants",
"//internal/imagefetcher",
"//internal/role",
"@com_github_azure_azure_sdk_for_go//profiles/latest/attestation/attestation",
"@com_github_azure_azure_sdk_for_go_sdk_azcore//policy",
"@com_github_azure_azure_sdk_for_go_sdk_azidentity//:azidentity",

45
cli/internal/cloudcmd/create.go
View File

@ -26,6 +26,7 @@ import (
"github.com/edgelesssys/constellation/v2/internal/config"
"github.com/edgelesssys/constellation/v2/internal/constants"
"github.com/edgelesssys/constellation/v2/internal/imagefetcher"
"github.com/edgelesssys/constellation/v2/internal/role"
)
// Creator creates cloud resources.
@ -214,25 +215,35 @@ func (c *Creator) createGCP(ctx context.Context, cl terraformClient, opts Create
func (c *Creator) createAzure(ctx context.Context, cl terraformClient, opts CreateOptions) (idFile clusterid.File, retErr error) {
vars := terraform.AzureClusterVariables{
CommonVariables: terraform.CommonVariables{
Name: opts.Config.Name,
CountControlPlanes: opts.ControlPlaneCount,
CountWorkers: opts.WorkerCount,
StateDiskSizeGB: opts.Config.StateDiskSizeGB,
Name: opts.Config.Name,
NodeGroups: map[string]terraform.AzureNodeGroup{
"control_plane_default": {
Role: role.ControlPlane.TFString(),
InstanceCount: toPtr(opts.ControlPlaneCount),
InstanceType: opts.InsType,
DiskSizeGB: opts.Config.StateDiskSizeGB,
DiskType: opts.Config.Provider.Azure.StateDiskType,
Zones: nil, // TODO(elchead): support zones AB#3225
},
"worker_default": {
Role: role.Worker.TFString(),
InstanceCount: toPtr(opts.WorkerCount),
InstanceType: opts.InsType,
DiskSizeGB: opts.Config.StateDiskSizeGB,
DiskType: opts.Config.Provider.Azure.StateDiskType,
Zones: nil,
},
},
Location: opts.Config.Provider.Azure.Location,
ResourceGroup: opts.Config.Provider.Azure.ResourceGroup,
UserAssignedIdentity: opts.Config.Provider.Azure.UserAssignedIdentity,
InstanceType: opts.InsType,
StateDiskType: opts.Config.Provider.Azure.StateDiskType,
ImageID: opts.image,
SecureBoot: *opts.Config.Provider.Azure.SecureBoot,
CreateMAA: opts.Config.GetAttestationConfig().GetVariant().Equal(variant.AzureSEVSNP{}),
Debug: opts.Config.IsDebugCluster(),
CreateMAA: toPtr(opts.Config.GetAttestationConfig().GetVariant().Equal(variant.AzureSEVSNP{})),
Debug: toPtr(opts.Config.IsDebugCluster()),
ConfidentialVM: toPtr(opts.Config.GetAttestationConfig().GetVariant().Equal(variant.AzureSEVSNP{})),
SecureBoot: opts.Config.Provider.Azure.SecureBoot,
UserAssignedIdentity: opts.Config.Provider.Azure.UserAssignedIdentity,
ResourceGroup: opts.Config.Provider.Azure.ResourceGroup,
}
vars.ConfidentialVM = opts.Config.GetAttestationConfig().GetVariant().Equal(variant.AzureSEVSNP{})
vars = normalizeAzureURIs(vars)
if err := cl.PrepareWorkspace(path.Join("terraform", strings.ToLower(cloudprovider.Azure.String())), &vars); err != nil {
@ -245,7 +256,7 @@ func (c *Creator) createAzure(ctx context.Context, cl terraformClient, opts Crea
return clusterid.File{}, err
}
if vars.CreateMAA {
if vars.CreateMAA != nil && *vars.CreateMAA {
// Patch the attestation policy to allow the cluster to boot while having secure boot disabled.
if err := c.policyPatcher.Patch(ctx, tfOutput.AttestationURL); err != nil {
return clusterid.File{}, err
@ -442,3 +453,7 @@ func (c *Creator) createQEMU(ctx context.Context, cl terraformClient, lv libvirt
UID: tfOutput.UID,
}, nil
}
func toPtr[T any](v T) *T {
return &v
}

36
cli/internal/cmd/upgradeapply.go
View File

@ -156,6 +156,10 @@ func (u *upgradeApplyCmd) migrateTerraform(cmd *cobra.Command, file file.Handler
if err != nil {
return fmt.Errorf("parsing upgrade variables: %w", err)
}
if len(targets) == 0 {
u.log.Debugf("No targets specified. Skipping Terraform migration")
return nil
}
u.log.Debugf("Using migration targets:\n%v", targets)
u.log.Debugf("Using Terraform variables:\n%v", vars)
@ -240,7 +244,7 @@ func parseTerraformUpgradeVars(cmd *cobra.Command, conf *config.Config, fetcher
}
return targets, vars, nil
case cloudprovider.Azure:
targets := []string{"azurerm_attestation_provider.attestation_provider"}
targets := []string{"azurerm_attestation_provider.attestation_provider", "module.scale_set_group", "module.scale_set_control_plane", "module.scale_set_worker"}
// Azure Terraform provider is very strict about it's casing
imageRef = strings.Replace(imageRef, "CommunityGalleries", "communityGalleries", 1)
@ -248,16 +252,28 @@ func parseTerraformUpgradeVars(cmd *cobra.Command, conf *config.Config, fetcher
imageRef = strings.Replace(imageRef, "Versions", "versions", 1)
vars := &terraform.AzureClusterVariables{
CommonVariables: commonVariables,
Location: conf.Provider.Azure.Location,
Name: conf.Name,
ResourceGroup: conf.Provider.Azure.ResourceGroup,
UserAssignedIdentity: conf.Provider.Azure.UserAssignedIdentity,
InstanceType: conf.Provider.Azure.InstanceType,
StateDiskType: conf.Provider.Azure.StateDiskType,
ImageID: imageRef,
SecureBoot: *conf.Provider.Azure.SecureBoot,
CreateMAA: conf.GetAttestationConfig().GetVariant().Equal(variant.AzureSEVSNP{}),
Debug: conf.IsDebugCluster(),
NodeGroups: map[string]terraform.AzureNodeGroup{
"control_plane_default": {
Role: "control-plane",
InstanceType: conf.Provider.Azure.InstanceType,
DiskSizeGB: conf.StateDiskSizeGB,
DiskType: conf.Provider.Azure.StateDiskType,
},
"worker_default": {
Role: "worker",
InstanceType: conf.Provider.Azure.InstanceType,
DiskSizeGB: conf.StateDiskSizeGB,
DiskType: conf.Provider.Azure.StateDiskType,
},
},
Location: conf.Provider.Azure.Location,
SecureBoot: conf.Provider.Azure.SecureBoot,
CreateMAA: toPtr(conf.GetAttestationConfig().GetVariant().Equal(variant.AzureSEVSNP{})),
Debug: toPtr(conf.IsDebugCluster()),
}
return targets, vars, nil
case cloudprovider.GCP:
@ -427,3 +443,7 @@ type cloudUpgrader interface {
CheckTerraformMigrations(fileHandler file.Handler) error
CleanUpTerraformMigrations(fileHandler file.Handler) error
}
func toPtr[T any](v T) *T {
return &v
}

1
cli/internal/terraform/BUILD.bazel
View File

@ -103,6 +103,7 @@ go_test(
"//internal/cloud/cloudprovider",
"//internal/constants",
"//internal/file",
"@com_github_azure_azure_sdk_for_go_sdk_azcore//to",
"@com_github_hashicorp_terraform_exec//tfexec",
"@com_github_hashicorp_terraform_json//:terraform-json",
"@com_github_spf13_afero//:afero",

72
cli/internal/terraform/terraform/azure/main.tf
View File

@ -221,58 +221,46 @@ resource "azurerm_network_security_group" "security_group" {
}
}
module "scale_set_control_plane" {
source = "./modules/scale_set"
name = "${local.name}-control-plane"
instance_count = var.control_plane_count
state_disk_size = var.state_disk_size
state_disk_type = var.state_disk_type
resource_group = var.resource_group
location = var.location
instance_type = var.instance_type
confidential_vm = var.confidential_vm
secure_boot = var.secure_boot
module "scale_set_group" {
source = "./modules/scale_set"
for_each = var.node_groups
base_name = local.name
node_group_name = each.key
role = each.value.role
zones = each.value.zones
tags = merge(
local.tags,
{ constellation-role = "control-plane" },
{ constellation-init-secret-hash = local.initSecretHash },
{ constellation-maa-url = var.create_maa ? azurerm_attestation_provider.attestation_provider[0].attestation_uri : "" },
)
image_id = var.image_id
instance_count = each.value.instance_count
state_disk_size = each.value.disk_size
state_disk_type = each.value.disk_type
location = var.location
instance_type = each.value.instance_type
confidential_vm = var.confidential_vm
secure_boot = var.secure_boot
resource_group = var.resource_group
user_assigned_identity = var.user_assigned_identity
image_id = var.image_id
network_security_group_id = azurerm_network_security_group.security_group.id
subnet_id = azurerm_subnet.node_subnet.id
backend_address_pool_ids = [
backend_address_pool_ids = each.value.role == "control-plane" ? [
azurerm_lb_backend_address_pool.all.id,
module.loadbalancer_backend_control_plane.backendpool_id
]
}
module "scale_set_worker" {
source = "./modules/scale_set"
name = "${local.name}-worker"
instance_count = var.worker_count
state_disk_size = var.state_disk_size
state_disk_type = var.state_disk_type
resource_group = var.resource_group
location = var.location
instance_type = var.instance_type
confidential_vm = var.confidential_vm
secure_boot = var.secure_boot
tags = merge(
local.tags,
{ constellation-role = "worker" },
{ constellation-init-secret-hash = local.initSecretHash },
{ constellation-maa-url = var.create_maa ? azurerm_attestation_provider.attestation_provider[0].attestation_uri : "" },
)
image_id = var.image_id
user_assigned_identity = var.user_assigned_identity
network_security_group_id = azurerm_network_security_group.security_group.id
subnet_id = azurerm_subnet.node_subnet.id
backend_address_pool_ids = [
] : [
azurerm_lb_backend_address_pool.all.id,
module.loadbalancer_backend_worker.backendpool_id,
module.loadbalancer_backend_worker.backendpool_id
]
}
moved {
from = module.scale_set_control_plane
to = module.scale_set_group["control_plane_default"]
}
moved {
from = module.scale_set_worker
to = module.scale_set_group["worker_default"]
}

20
cli/internal/terraform/terraform/azure/modules/scale_set/main.tf
View File

@ -11,6 +11,19 @@ terraform {
}
}
locals {
tags = merge(
var.tags,
{ constellation-role = var.role },
{ constellation-node-group = var.node_group_name },
)
group_uid = random_id.uid.hex
name = "${var.base_name}-${var.role}${local.group_uid}"
}
resource "random_id" "uid" {
byte_length = 4
}
resource "random_password" "password" {
length = 16
min_lower = 1
@ -20,7 +33,7 @@ resource "random_password" "password" {
}
resource "azurerm_linux_virtual_machine_scale_set" "scale_set" {
name = var.name
name = local.name
resource_group_name = var.resource_group
location = var.location
sku = var.instance_type
@ -34,8 +47,8 @@ resource "azurerm_linux_virtual_machine_scale_set" "scale_set" {
upgrade_mode = "Manual"
secure_boot_enabled = var.secure_boot
source_image_id = var.image_id
tags = var.tags
tags = local.tags
zones = var.zones
identity {
type = "UserAssigned"
identity_ids = [var.user_assigned_identity]
@ -81,6 +94,7 @@ resource "azurerm_linux_virtual_machine_scale_set" "scale_set" {
lifecycle {
ignore_changes = [
name, # required. Allow legacy scale sets to keep their old names
instances, # required. autoscaling modifies the instance count externally
source_image_id, # required. update procedure modifies the image id externally
]

35
cli/internal/terraform/terraform/azure/modules/scale_set/variables.tf
View File

@ -1,7 +1,31 @@
variable "name" {
variable "base_name" {
type = string
default = "constell"
description = "Base name of the cluster."
description = "Base name of the instance group."
}
variable "node_group_name" {
type = string
description = "Constellation name for the node group (used for configuration and CSP-independent naming)."
}
variable "role" {
type = string
description = "The role of the instance group."
validation {
condition = contains(["control-plane", "worker"], var.role)
error_message = "The role has to be 'control-plane' or 'worker'."
}
}
variable "tags" {
type = map(string)
description = "Tags to include in the scale_set."
}
variable "zones" {
type = list(string)
description = "List of availability zones."
default = null
}
variable "instance_count" {
@ -61,11 +85,6 @@ variable "subnet_id" {
description = "The ID of the subnet to use for the scale set."
}
variable "tags" {
type = map(string)
description = "The tags to add to the scale set."
}
variable "confidential_vm" {
type = bool
default = true

77
cli/internal/terraform/terraform/azure/variables.tf
View File

@ -1,28 +1,22 @@
variable "name" {
type = string
default = "constell"
description = "Base name of the cluster."
}
variable "control_plane_count" {
type = number
description = "The number of control plane nodes to deploy."
}
variable "worker_count" {
type = number
description = "The number of worker nodes to deploy."
}
variable "state_disk_size" {
type = number
default = 30
description = "The size of the state disk in GB."
}
variable "resource_group" {
type = string
description = "The name of the Azure resource group to create the Constellation cluster in."
variable "node_groups" {
type = map(object({
role = string
instance_count = optional(number)
instance_type = string
disk_size = number
disk_type = string
zones = optional(list(string))
}))
description = "A map of node group names to node group configurations."
validation {
condition = can([for group in var.node_groups : group.role == "control-plane" || group.role == "worker"])
error_message = "The role has to be 'control-plane' or 'worker'."
}
}
variable "location" {
@ -30,27 +24,23 @@ variable "location" {
description = "The Azure location to deploy the cluster in."
}
variable "user_assigned_identity" {
type = string
description = "The name of the user assigned identity to attache to the nodes of the cluster."
}
variable "instance_type" {
type = string
description = "The Azure instance type to deploy."
}
variable "state_disk_type" {
type = string
default = "Premium_LRS"
description = "The type of the state disk."
}
variable "image_id" {
type = string
description = "The image to use for the cluster nodes."
}
variable "create_maa" {
type = bool
default = false
description = "Whether to create a Microsoft Azure attestation provider."
}
variable "debug" {
type = bool
default = false
description = "Enable debug mode. This opens up a debugd port that can be used to deploy a custom bootstrapper."
}
variable "confidential_vm" {
type = bool
default = true
@ -63,14 +53,11 @@ variable "secure_boot" {
description = "Whether to deploy the cluster nodes with secure boot."
}
variable "create_maa" {
type = bool
default = false
description = "Whether to create a Microsoft Azure attestation provider."
variable "resource_group" {
type = string
description = "The name of the Azure resource group to create the Constellation cluster in."
}
variable "debug" {
type = bool
default = false
description = "Enable debug mode. This opens up a debugd port that can be used to deploy a custom bootstrapper."
variable "user_assigned_identity" {
type = string
description = "The name of the user assigned identity to attache to the nodes of the cluster."
}

64
cli/internal/terraform/variables.go
View File

@ -162,47 +162,45 @@ func (v *GCPIAMVariables) String() string {
// AzureClusterVariables is user configuration for creating a cluster with Terraform on Azure.
type AzureClusterVariables struct {
// CommonVariables contains common variables.
CommonVariables
// ResourceGroup is the name of the Azure resource group to use.
ResourceGroup string
// Location is the Azure location to use.
Location string
// UserAssignedIdentity is the name of the Azure user-assigned identity to use.
UserAssignedIdentity string
// InstanceType is the Azure instance type to use.
InstanceType string
// StateDiskType is the Azure disk type to use for the state disk.
StateDiskType string
// Name of the cluster.
Name string `hcl:"name" cty:"name"`
// ImageID is the ID of the Azure image to use.
ImageID string
// ConfidentialVM sets the VM to be confidential.
ConfidentialVM bool
// SecureBoot sets the VM to use secure boot.
SecureBoot bool
ImageID string `hcl:"image_id" cty:"image_id"`
// CreateMAA sets whether a Microsoft Azure attestation provider should be created.
CreateMAA bool
CreateMAA *bool `hcl:"create_maa" cty:"create_maa"`
// Debug is true if debug mode is enabled.
Debug bool
Debug *bool `hcl:"debug" cty:"debug"`
// ResourceGroup is the name of the Azure resource group to use.
ResourceGroup string `hcl:"resource_group" cty:"resource_group"`
// Location is the Azure location to use.
Location string `hcl:"location" cty:"location"`
// UserAssignedIdentity is the name of the Azure user-assigned identity to use.
UserAssignedIdentity string `hcl:"user_assigned_identity" cty:"user_assigned_identity"`
// ConfidentialVM sets the VM to be confidential.
ConfidentialVM *bool `hcl:"confidential_vm" cty:"confidential_vm"`
// SecureBoot sets the VM to use secure boot.
SecureBoot *bool `hcl:"secure_boot" cty:"secure_boot"`
// NodeGroups is a map of node groups to create.
NodeGroups map[string]AzureNodeGroup `hcl:"node_groups" cty:"node_groups"`
}
// String returns a string representation of the variables, formatted as Terraform variables.
func (v *AzureClusterVariables) String() string {
b := &strings.Builder{}
b.WriteString(v.CommonVariables.String())
writeLinef(b, "resource_group = %q", v.ResourceGroup)
writeLinef(b, "location = %q", v.Location)
writeLinef(b, "user_assigned_identity = %q", v.UserAssignedIdentity)
writeLinef(b, "instance_type = %q", v.InstanceType)
writeLinef(b, "state_disk_type = %q", v.StateDiskType)
writeLinef(b, "image_id = %q", v.ImageID)
writeLinef(b, "confidential_vm = %t", v.ConfidentialVM)
writeLinef(b, "secure_boot = %t", v.SecureBoot)
writeLinef(b, "create_maa = %t", v.CreateMAA)
writeLinef(b, "debug = %t", v.Debug)
f := hclwrite.NewEmptyFile()
gohcl.EncodeIntoBody(v, f.Body())
return string(f.Bytes())
}
return b.String()
// AzureNodeGroup is a node group to create on Azure.
type AzureNodeGroup struct {
// Role is the role of the node group.
Role string `hcl:"role" cty:"role"`
// InstanceCount is optional for upgrades.
InstanceCount *int `hcl:"instance_count" cty:"instance_count"`
InstanceType string `hcl:"instance_type" cty:"instance_type"`
DiskSizeGB int `hcl:"disk_size" cty:"disk_size"`
DiskType string `hcl:"disk_type" cty:"disk_type"`
Zones *[]string `hcl:"zones" cty:"zones"`
}
// AzureIAMVariables is user configuration for creating the IAM configuration with Terraform on Microsoft Azure.

56
cli/internal/terraform/variables_test.go
View File

@ -9,6 +9,7 @@ package terraform
import (
"testing"
"github.com/Azure/azure-sdk-for-go/sdk/azcore/to"
"github.com/stretchr/testify/assert"
)
@ -142,39 +143,44 @@ service_account_id = "my-service-account"
func TestAzureClusterVariables(t *testing.T) {
vars := AzureClusterVariables{
CommonVariables: CommonVariables{
Name: "cluster-name",
CountControlPlanes: 1,
CountWorkers: 2,
StateDiskSizeGB: 30,
Name: "cluster-name",
NodeGroups: map[string]AzureNodeGroup{
"control_plane_default": {
Role: "ControlPlane",
InstanceCount: to.Ptr(1),
InstanceType: "Standard_D2s_v3",
DiskType: "StandardSSD_LRS",
DiskSizeGB: 100,
},
},
ConfidentialVM: to.Ptr(true),
ResourceGroup: "my-resource-group",
Location: "eu-central-1",
UserAssignedIdentity: "my-user-assigned-identity",
InstanceType: "Standard_D2s_v3",
StateDiskType: "StandardSSD_LRS",
ImageID: "image-0123456789abcdef",
ConfidentialVM: true,
SecureBoot: false,
CreateMAA: true,
Debug: true,
CreateMAA: to.Ptr(true),
Debug: to.Ptr(true),
Location: "eu-central-1",
}
// test that the variables are correctly rendered
want := `name = "cluster-name"
control_plane_count = 1
worker_count = 2
state_disk_size = 30
resource_group = "my-resource-group"
location = "eu-central-1"
want := `name = "cluster-name"
image_id = "image-0123456789abcdef"
create_maa = true
debug = true
resource_group = "my-resource-group"
location = "eu-central-1"
user_assigned_identity = "my-user-assigned-identity"
instance_type = "Standard_D2s_v3"
state_disk_type = "StandardSSD_LRS"
image_id = "image-0123456789abcdef"
confidential_vm = true
secure_boot = false
create_maa = true
debug = true
confidential_vm = true
node_groups = {
control_plane_default = {
disk_size = 100
disk_type = "StandardSSD_LRS"
instance_count = 1
instance_type = "Standard_D2s_v3"
role = "ControlPlane"
zones = null
}
}
`
got := vars.String()
assert.Equal(t, want, got)

12
internal/role/role.go
View File

@ -25,6 +25,18 @@ const (
Worker
)
// TFString returns the role as a string for Terraform.
func (r Role) TFString() string {
switch r {
case ControlPlane:
return "control-plane"
case Worker:
return "worker"
default:
return "unknown"
}
}
// MarshalJSON marshals the Role to JSON string.
func (r Role) MarshalJSON() ([]byte, error) {
return json.Marshal(r.String())