bootstrapper: add fallback endpoint and custom endpoint to SAN field (#2108)

terraform: collect apiserver cert SANs and support custom endpoint

constants: add new constants for cluster configuration and custom endpoint

cloud: support apiserver cert sans and prepare for endpoint migration on AWS

config: add customEndpoint field

bootstrapper: use per-CSP apiserver cert SANs

cli: route customEndpoint to terraform and add migration for apiserver cert SANs

bootstrapper: change interface of GetLoadBalancerEndpoint to return host and port separately

cli/internal/terraform/terraform.go

@@ -103,53 +103,66 @@ func (c *Client) PrepareUpgradeWorkspace(path, oldWorkingDir, newWorkingDir, bac
 }
 
 // CreateCluster creates a Constellation cluster using Terraform.
-func (c *Client) CreateCluster(ctx context.Context, logLevel LogLevel) (CreateOutput, error) {
+func (c *Client) CreateCluster(ctx context.Context, logLevel LogLevel) (ApplyOutput, error) {
 	if err := c.setLogLevel(logLevel); err != nil {
-		return CreateOutput{}, fmt.Errorf("set terraform log level %s: %w", logLevel.String(), err)
+		return ApplyOutput{}, fmt.Errorf("set terraform log level %s: %w", logLevel.String(), err)
 	}
 
 	if err := c.tf.Init(ctx); err != nil {
-		return CreateOutput{}, fmt.Errorf("terraform init: %w", err)
+		return ApplyOutput{}, fmt.Errorf("terraform init: %w", err)
 	}
 
 	if err := c.applyManualStateMigrations(ctx); err != nil {
-		return CreateOutput{}, fmt.Errorf("apply manual state migrations: %w", err)
+		return ApplyOutput{}, fmt.Errorf("apply manual state migrations: %w", err)
 	}
 
 	if err := c.tf.Apply(ctx); err != nil {
-		return CreateOutput{}, fmt.Errorf("terraform apply: %w", err)
+		return ApplyOutput{}, fmt.Errorf("terraform apply: %w", err)
 	}
 
 	tfState, err := c.tf.Show(ctx)
 	if err != nil {
-		return CreateOutput{}, fmt.Errorf("terraform show: %w", err)
+		return ApplyOutput{}, fmt.Errorf("terraform show: %w", err)
 	}
 
 	ipOutput, ok := tfState.Values.Outputs["ip"]
 	if !ok {
-		return CreateOutput{}, errors.New("no IP output found")
+		return ApplyOutput{}, errors.New("no IP output found")
 	}
 	ip, ok := ipOutput.Value.(string)
 	if !ok {
-		return CreateOutput{}, errors.New("invalid type in IP output: not a string")
+		return ApplyOutput{}, errors.New("invalid type in IP output: not a string")
+	}
+
+	apiServerCertSANsOutput, ok := tfState.Values.Outputs["api_server_cert_sans"]
+	if !ok {
+		return ApplyOutput{}, errors.New("no api_server_cert_sans output found")
+	}
+	apiServerCertSANsUntyped, ok := apiServerCertSANsOutput.Value.([]any)
+	if !ok {
+		return ApplyOutput{}, fmt.Errorf("invalid type in api_server_cert_sans output: %s is not a list of elements", apiServerCertSANsOutput.Type.FriendlyName())
+	}
+	apiServerCertSANs, err := toStringSlice(apiServerCertSANsUntyped)
+	if err != nil {
+		return ApplyOutput{}, fmt.Errorf("convert api_server_cert_sans output: %w", err)
 	}
 
 	secretOutput, ok := tfState.Values.Outputs["initSecret"]
 	if !ok {
-		return CreateOutput{}, errors.New("no initSecret output found")
+		return ApplyOutput{}, errors.New("no initSecret output found")
 	}
 	secret, ok := secretOutput.Value.(string)
 	if !ok {
-		return CreateOutput{}, errors.New("invalid type in initSecret output: not a string")
+		return ApplyOutput{}, errors.New("invalid type in initSecret output: not a string")
 	}
 
 	uidOutput, ok := tfState.Values.Outputs["uid"]
 	if !ok {
-		return CreateOutput{}, errors.New("no uid output found")
+		return ApplyOutput{}, errors.New("no uid output found")
 	}
 	uid, ok := uidOutput.Value.(string)
 	if !ok {
-		return CreateOutput{}, errors.New("invalid type in uid output: not a string")
+		return ApplyOutput{}, errors.New("invalid type in uid output: not a string")
 	}
 
 	var attestationURL string
@@ -159,19 +172,22 @@ func (c *Client) CreateCluster(ctx context.Context, logLevel LogLevel) (CreateOu
 		}
 	}
 
-	return CreateOutput{
-		IP:             ip,
-		Secret:         secret,
-		UID:            uid,
-		AttestationURL: attestationURL,
+	return ApplyOutput{
+		IP:                ip,
+		APIServerCertSANs: apiServerCertSANs,
+		Secret:            secret,
+		UID:               uid,
+		AttestationURL:    attestationURL,
 	}, nil
 }
 
-// CreateOutput contains the Terraform output values of a cluster creation.
-type CreateOutput struct {
-	IP     string
-	Secret string
-	UID    string
+// ApplyOutput contains the Terraform output values of a cluster creation
+// or apply operation.
+type ApplyOutput struct {
+	IP                string
+	APIServerCertSANs []string
+	Secret            string
+	UID               string
 	// AttestationURL is the URL of the attestation provider.
 	// It is only set if the cluster is created on Azure.
 	AttestationURL string
@@ -447,6 +463,18 @@ type StateMigration struct {
 	Hook        func(ctx context.Context, tfClient TFMigrator) error
 }
 
+func toStringSlice(in []any) ([]string, error) {
+	out := make([]string, len(in))
+	for i, v := range in {
+		s, ok := v.(string)
+		if !ok {
+			return nil, fmt.Errorf("invalid type in list: item at index %v of list is not a string", i)
+		}
+		out[i] = s
+	}
+	return out, nil
+}
+
 type tfInterface interface {
 	Apply(context.Context, ...tfexec.ApplyOption) error
 	Destroy(context.Context, ...tfexec.DestroyOption) error

cli/internal/terraform/terraform/aws/main.tf

@@ -46,8 +46,13 @@ locals {
   zones = distinct(sort([
     for node_group in var.node_groups : node_group.zone
   ]))
+  // wildcard_lb_dns_name is the DNS name of the load balancer with a wildcard for the name.
+  // example: given "name-1234567890.region.elb.amazonaws.com" it will return "*.region.elb.amazonaws.com"
+  wildcard_lb_dns_name = replace(aws_lb.front_end.dns_name, "/^[^.]*\\./", "*.")
 
-  tags = { constellation-uid = local.uid }
+  tags = {
+    constellation-uid = local.uid,
+  }
 }
 
 resource "random_id" "uid" {
@@ -83,7 +88,7 @@ resource "aws_eip" "lb" {
   # control-plane.
   for_each = toset([var.zone])
   domain   = "vpc"
-  tags     = local.tags
+  tags     = merge(local.tags, { "constellation-ip-endpoint" = each.key == var.zone ? "legacy-primary-zone" : "additional-zone" })
 }
 
 resource "aws_lb" "front_end" {

cli/internal/terraform/terraform/aws/outputs.tf

@@ -2,6 +2,10 @@ output "ip" {
   value = aws_eip.lb[var.zone].public_ip
 }
 
+output "api_server_cert_sans" {
+  value = sort(concat([aws_eip.lb[var.zone].public_ip, local.wildcard_lb_dns_name], var.custom_endpoint == "" ? [] : [var.custom_endpoint]))
+}
+
 output "uid" {
   value = local.uid
 }

cli/internal/terraform/terraform/aws/variables.tf

@@ -63,3 +63,9 @@ variable "enable_snp" {
   default     = true
   description = "Enable AMD SEV SNP. Setting this to true sets the cpu-option AmdSevSnp to enable."
 }
+
+variable "custom_endpoint" {
+  type        = string
+  default     = ""
+  description = "Custom endpoint to use for the Kubernetes apiserver. If not set, the default endpoint will be used."
+}

cli/internal/terraform/terraform/azure/main.tf

@@ -20,10 +20,12 @@ provider "azurerm" {
 }
 
 locals {
-  uid                   = random_id.uid.hex
-  name                  = "${var.name}-${local.uid}"
-  initSecretHash        = random_password.initSecret.bcrypt_hash
-  tags                  = { constellation-uid = local.uid }
+  uid            = random_id.uid.hex
+  name           = "${var.name}-${local.uid}"
+  initSecretHash = random_password.initSecret.bcrypt_hash
+  tags = {
+    constellation-uid = local.uid,
+  }
   ports_node_range      = "30000-32767"
   ports_kubernetes      = "6443"
   ports_bootstrapper    = "9000"
@@ -33,6 +35,9 @@ locals {
   ports_debugd          = "4000"
   cidr_vpc_subnet_nodes = "192.168.178.0/24"
   cidr_vpc_subnet_pods  = "10.10.0.0/16"
+  // wildcard_lb_dns_name is the DNS name of the load balancer with a wildcard for the name.
+  // example: given "name-1234567890.location.cloudapp.azure.com" it will return "*.location.cloudapp.azure.com"
+  wildcard_lb_dns_name = replace(azurerm_public_ip.loadbalancer_ip.fqdn, "/^[^.]*\\./", "*.")
 }
 
 resource "random_id" "uid" {
@@ -72,6 +77,7 @@ resource "azurerm_application_insights" "insights" {
 
 resource "azurerm_public_ip" "loadbalancer_ip" {
   name                = "${local.name}-lb"
+  domain_name_label   = local.name
   resource_group_name = var.resource_group
   location            = var.location
   allocation_method   = "Static"

cli/internal/terraform/terraform/azure/outputs.tf

@@ -2,6 +2,10 @@ output "ip" {
   value = azurerm_public_ip.loadbalancer_ip.ip_address
 }
 
+output "api_server_cert_sans" {
+  value = sort(concat([azurerm_public_ip.loadbalancer_ip.ip_address, local.wildcard_lb_dns_name], var.custom_endpoint == "" ? [] : [var.custom_endpoint]))
+}
+
 output "uid" {
   value = local.uid
 }

cli/internal/terraform/terraform/azure/variables.tf

@@ -61,3 +61,9 @@ variable "user_assigned_identity" {
   type        = string
   description = "The name of the user assigned identity to attache to the nodes of the cluster."
 }
+
+variable "custom_endpoint" {
+  type        = string
+  default     = ""
+  description = "Custom endpoint to use for the Kubernetes apiserver. If not set, the default endpoint will be used."
+}

cli/internal/terraform/terraform/gcp/main.tf

@@ -30,10 +30,12 @@ provider "google-beta" {
 }
 
 locals {
-  uid                   = random_id.uid.hex
-  name                  = "${var.name}-${local.uid}"
-  initSecretHash        = random_password.initSecret.bcrypt_hash
-  labels                = { constellation-uid = local.uid }
+  uid            = random_id.uid.hex
+  name           = "${var.name}-${local.uid}"
+  initSecretHash = random_password.initSecret.bcrypt_hash
+  labels = {
+    constellation-uid = local.uid,
+  }
   ports_node_range      = "30000-32767"
   ports_kubernetes      = "6443"
   ports_bootstrapper    = "9000"
@@ -170,6 +172,7 @@ module "instance_group" {
   named_ports         = each.value.role == "control-plane" ? local.control_plane_named_ports : []
   labels              = local.labels
   init_secret_hash    = local.initSecretHash
+  custom_endpoint     = var.custom_endpoint
 }
 
 resource "google_compute_global_address" "loadbalancer_ip" {

cli/internal/terraform/terraform/gcp/modules/instance_group/variables.tf

@@ -94,3 +94,8 @@ variable "zone" {
   type        = string
   description = "Zone to deploy the instance group in."
 }
+
+variable "custom_endpoint" {
+  type        = string
+  description = "Custom endpoint to use for the Kubernetes apiserver. If not set, the default endpoint will be used."
+}

cli/internal/terraform/terraform/gcp/outputs.tf

@@ -2,6 +2,14 @@ output "ip" {
   value = google_compute_global_address.loadbalancer_ip.address
 }
 
+output "api_server_cert_sans" {
+  value = sort(concat([google_compute_global_address.loadbalancer_ip.address], var.custom_endpoint == "" ? [] : [var.custom_endpoint]))
+}
+
+output "fallback_endpoint" {
+  value = google_compute_global_address.loadbalancer_ip.address
+}
+
 output "uid" {
   value = local.uid
 }

cli/internal/terraform/terraform/gcp/variables.tf

@@ -45,3 +45,9 @@ variable "debug" {
   default     = false
   description = "Enable debug mode. This opens up a debugd port that can be used to deploy a custom bootstrapper."
 }
+
+variable "custom_endpoint" {
+  type        = string
+  default     = ""
+  description = "Custom endpoint to use for the Kubernetes apiserver. If not set, the default endpoint will be used."
+}

cli/internal/terraform/terraform/openstack/outputs.tf

@@ -2,6 +2,10 @@ output "ip" {
   value = openstack_networking_floatingip_v2.public_ip.address
 }
 
+output "api_server_cert_sans" {
+  value = sort(concat([openstack_networking_floatingip_v2.public_ip.address], var.custom_endpoint == "" ? [] : [var.custom_endpoint]))
+}
+
 output "uid" {
   value = local.uid
 }

cli/internal/terraform/terraform/openstack/variables.tf

@@ -67,3 +67,9 @@ variable "debug" {
   default     = false
   description = "Enable debug mode. This opens up a debugd port that can be used to deploy a custom bootstrapper."
 }
+
+variable "custom_endpoint" {
+  type        = string
+  default     = ""
+  description = "Custom endpoint to use for the Kubernetes apiserver. If not set, the default endpoint will be used."
+}

cli/internal/terraform/terraform/qemu/outputs.tf

@@ -2,6 +2,10 @@ output "ip" {
   value = module.node_group["control_plane_default"].instance_ips[0]
 }
 
+output "api_server_cert_sans" {
+  value = sort(concat([module.node_group["control_plane_default"].instance_ips[0]], var.custom_endpoint == "" ? [] : [var.custom_endpoint]))
+}
+
 output "uid" {
   value = "qemu" // placeholder
 }

cli/internal/terraform/terraform/qemu/variables.tf

@@ -96,3 +96,9 @@ variable "name" {
   default     = "constellation"
   description = "name prefix of the cluster VMs"
 }
+
+variable "custom_endpoint" {
+  type        = string
+  default     = ""
+  description = "Custom endpoint to use for the Kubernetes apiserver. If not set, the default endpoint will be used."
+}

cli/internal/terraform/terraform_test.go

@@ -221,6 +221,9 @@ func TestCreateCluster(t *testing.T) {
 					"uid": {
 						Value: "12345abc",
 					},
+					"api_server_cert_sans": {
+						Value: []any{"192.0.2.100"},
+					},
 				},
 			},
 		}
@@ -242,6 +245,9 @@ func TestCreateCluster(t *testing.T) {
 					"attestationURL": {
 						Value: "https://12345.neu.attest.azure.net",
 					},
+					"api_server_cert_sans": {
+						Value: []any{"192.0.2.100"},
+					},
 				},
 			},
 		}

cli/internal/terraform/variables.go

@@ -30,29 +30,6 @@ type ClusterVariables interface {
 	GetCreateMAA() bool
 }
 
-// CommonVariables is user configuration for creating a cluster with Terraform.
-type CommonVariables struct {
-	// Name of the cluster.
-	Name string
-	// CountControlPlanes is the number of control-plane nodes to create.
-	CountControlPlanes int
-	// CountWorkers is the number of worker nodes to create.
-	CountWorkers int
-	// StateDiskSizeGB is the size of the state disk to allocate to each node, in GB.
-	StateDiskSizeGB int
-}
-
-// String returns a string representation of the variables, formatted as Terraform variables.
-func (v *CommonVariables) String() string {
-	b := &strings.Builder{}
-	writeLinef(b, "name = %q", v.Name)
-	writeLinef(b, "control_plane_count = %d", v.CountControlPlanes)
-	writeLinef(b, "worker_count = %d", v.CountWorkers)
-	writeLinef(b, "state_disk_size = %d", v.StateDiskSizeGB)
-
-	return b.String()
-}
-
 // AWSClusterVariables is user configuration for creating a cluster with Terraform on AWS.
 type AWSClusterVariables struct {
 	// Name of the cluster.
@@ -73,6 +50,8 @@ type AWSClusterVariables struct {
 	EnableSNP bool `hcl:"enable_snp" cty:"enable_snp"`
 	// NodeGroups is a map of node groups to create.
 	NodeGroups map[string]AWSNodeGroup `hcl:"node_groups" cty:"node_groups"`
+	// CustomEndpoint is the (optional) custom dns hostname for the kubernetes api server.
+	CustomEndpoint string `hcl:"custom_endpoint" cty:"custom_endpoint"`
 }
 
 // GetCreateMAA gets the CreateMAA variable.
@@ -138,6 +117,8 @@ type GCPClusterVariables struct {
 	Debug bool `hcl:"debug" cty:"debug"`
 	// NodeGroups is a map of node groups to create.
 	NodeGroups map[string]GCPNodeGroup `hcl:"node_groups" cty:"node_groups"`
+	// CustomEndpoint is the (optional) custom dns hostname for the kubernetes api server.
+	CustomEndpoint string `hcl:"custom_endpoint" cty:"custom_endpoint"`
 }
 
 // GetCreateMAA gets the CreateMAA variable.
@@ -212,6 +193,8 @@ type AzureClusterVariables struct {
 	SecureBoot *bool `hcl:"secure_boot" cty:"secure_boot"`
 	// NodeGroups is a map of node groups to create.
 	NodeGroups map[string]AzureNodeGroup `hcl:"node_groups" cty:"node_groups"`
+	// CustomEndpoint is the (optional) custom dns hostname for the kubernetes api server.
+	CustomEndpoint string `hcl:"custom_endpoint" cty:"custom_endpoint"`
 }
 
 // GetCreateMAA gets the CreateMAA variable.
@@ -287,6 +270,8 @@ type OpenStackClusterVariables struct {
 	OpenstackPassword string `hcl:"openstack_password" cty:"openstack_password"`
 	// Debug is true if debug mode is enabled.
 	Debug bool `hcl:"debug" cty:"debug"`
+	// CustomEndpoint is the (optional) custom dns hostname for the kubernetes api server.
+	CustomEndpoint string `hcl:"custom_endpoint" cty:"custom_endpoint"`
 }
 
 // GetCreateMAA gets the CreateMAA variable.
@@ -354,6 +339,8 @@ type QEMUVariables struct {
 	InitrdPath *string `hcl:"constellation_initrd" cty:"constellation_initrd"`
 	// KernelCmdline is the kernel command line.
 	KernelCmdline *string `hcl:"constellation_cmdline" cty:"constellation_cmdline"`
+	// CustomEndpoint is the (optional) custom dns hostname for the kubernetes api server.
+	CustomEndpoint string `hcl:"custom_endpoint" cty:"custom_endpoint"`
 }
 
 // GetCreateMAA gets the CreateMAA variable.

cli/internal/terraform/variables_test.go

@@ -43,6 +43,7 @@ func TestAWSClusterVariables(t *testing.T) {
 		IAMProfileWorkerNodes:  "arn:aws:iam::123456789012:instance-profile/cluster-name-worker",
 		Debug:                  true,
 		EnableSNP:              true,
+		CustomEndpoint:         "example.com",
 	}
 
 	// test that the variables are correctly rendered
@@ -72,6 +73,7 @@ node_groups = {
     zone          = "eu-central-1c"
   }
 }
+custom_endpoint = "example.com"
 `
 	got := vars.String()
 	assert.Equal(t, want, got)
@@ -117,6 +119,7 @@ func TestGCPClusterVariables(t *testing.T) {
 				DiskType:        "pd-ssd",
 			},
 		},
+		CustomEndpoint: "example.com",
 	}
 
 	// test that the variables are correctly rendered
@@ -144,6 +147,7 @@ node_groups = {
     zone          = "eu-central-1b"
   }
 }
+custom_endpoint = "example.com"
 `
 	got := vars.String()
 	assert.Equal(t, want, got)
@@ -186,6 +190,7 @@ func TestAzureClusterVariables(t *testing.T) {
 		CreateMAA:            to.Ptr(true),
 		Debug:                to.Ptr(true),
 		Location:             "eu-central-1",
+		CustomEndpoint:       "example.com",
 	}
 
 	// test that the variables are correctly rendered
@@ -207,6 +212,7 @@ node_groups = {
     zones         = null
   }
 }
+custom_endpoint = "example.com"
 `
 	got := vars.String()
 	assert.Equal(t, want, got)
@@ -249,6 +255,7 @@ func TestOpenStackClusterVariables(t *testing.T) {
 				StateDiskSizeGB: 30,
 			},
 		},
+		CustomEndpoint: "example.com",
 	}
 
 	// test that the variables are correctly rendered
@@ -271,6 +278,7 @@ openstack_user_domain_name = "my-user-domain"
 openstack_username         = "my-username"
 openstack_password         = "my-password"
 debug                      = true
+custom_endpoint            = "example.com"
 `
 	got := vars.String()
 	assert.Equal(t, want, got)
@@ -299,6 +307,7 @@ func TestQEMUClusterVariables(t *testing.T) {
 		NVRAM:              "production",
 		InitrdPath:         toPtr("/var/lib/libvirt/images/cluster-name-initrd"),
 		KernelCmdline:      toPtr("console=ttyS0,115200n8"),
+		CustomEndpoint:     "example.com",
 	}
 
 	// test that the variables are correctly rendered
@@ -323,6 +332,7 @@ metadata_libvirt_uri    = "qemu:///system"
 nvram                   = "/usr/share/OVMF/constellation_vars.production.fd"
 constellation_initrd    = "/var/lib/libvirt/images/cluster-name-initrd"
 constellation_cmdline   = "console=ttyS0,115200n8"
+custom_endpoint         = "example.com"
 `
 	got := vars.String()
 	assert.Equal(t, want, got)