bootstrapper: add fallback endpoint and custom endpoint to SAN field (#2108)

terraform: collect apiserver cert SANs and support custom endpoint

constants: add new constants for cluster configuration and custom endpoint

cloud: support apiserver cert sans and prepare for endpoint migration on AWS

config: add customEndpoint field

bootstrapper: use per-CSP apiserver cert SANs

cli: route customEndpoint to terraform and add migration for apiserver cert SANs

bootstrapper: change interface of GetLoadBalancerEndpoint to return host and port separately

cli/internal/terraform/terraform/aws/main.tf

@@ -46,8 +46,13 @@ locals {
   zones = distinct(sort([
     for node_group in var.node_groups : node_group.zone
   ]))
+  // wildcard_lb_dns_name is the DNS name of the load balancer with a wildcard for the name.
+  // example: given "name-1234567890.region.elb.amazonaws.com" it will return "*.region.elb.amazonaws.com"
+  wildcard_lb_dns_name = replace(aws_lb.front_end.dns_name, "/^[^.]*\\./", "*.")
 
-  tags = { constellation-uid = local.uid }
+  tags = {
+    constellation-uid = local.uid,
+  }
 }
 
 resource "random_id" "uid" {
@@ -83,7 +88,7 @@ resource "aws_eip" "lb" {
   # control-plane.
   for_each = toset([var.zone])
   domain   = "vpc"
-  tags     = local.tags
+  tags     = merge(local.tags, { "constellation-ip-endpoint" = each.key == var.zone ? "legacy-primary-zone" : "additional-zone" })
 }
 
 resource "aws_lb" "front_end" {

cli/internal/terraform/terraform/aws/outputs.tf

@@ -2,6 +2,10 @@ output "ip" {
   value = aws_eip.lb[var.zone].public_ip
 }
 
+output "api_server_cert_sans" {
+  value = sort(concat([aws_eip.lb[var.zone].public_ip, local.wildcard_lb_dns_name], var.custom_endpoint == "" ? [] : [var.custom_endpoint]))
+}
+
 output "uid" {
   value = local.uid
 }

cli/internal/terraform/terraform/aws/variables.tf

@@ -63,3 +63,9 @@ variable "enable_snp" {
   default     = true
   description = "Enable AMD SEV SNP. Setting this to true sets the cpu-option AmdSevSnp to enable."
 }
+
+variable "custom_endpoint" {
+  type        = string
+  default     = ""
+  description = "Custom endpoint to use for the Kubernetes apiserver. If not set, the default endpoint will be used."
+}

cli/internal/terraform/terraform/azure/main.tf

@@ -20,10 +20,12 @@ provider "azurerm" {
 }
 
 locals {
-  uid                   = random_id.uid.hex
-  name                  = "${var.name}-${local.uid}"
-  initSecretHash        = random_password.initSecret.bcrypt_hash
-  tags                  = { constellation-uid = local.uid }
+  uid            = random_id.uid.hex
+  name           = "${var.name}-${local.uid}"
+  initSecretHash = random_password.initSecret.bcrypt_hash
+  tags = {
+    constellation-uid = local.uid,
+  }
   ports_node_range      = "30000-32767"
   ports_kubernetes      = "6443"
   ports_bootstrapper    = "9000"
@@ -33,6 +35,9 @@ locals {
   ports_debugd          = "4000"
   cidr_vpc_subnet_nodes = "192.168.178.0/24"
   cidr_vpc_subnet_pods  = "10.10.0.0/16"
+  // wildcard_lb_dns_name is the DNS name of the load balancer with a wildcard for the name.
+  // example: given "name-1234567890.location.cloudapp.azure.com" it will return "*.location.cloudapp.azure.com"
+  wildcard_lb_dns_name = replace(azurerm_public_ip.loadbalancer_ip.fqdn, "/^[^.]*\\./", "*.")
 }
 
 resource "random_id" "uid" {
@@ -72,6 +77,7 @@ resource "azurerm_application_insights" "insights" {
 
 resource "azurerm_public_ip" "loadbalancer_ip" {
   name                = "${local.name}-lb"
+  domain_name_label   = local.name
   resource_group_name = var.resource_group
   location            = var.location
   allocation_method   = "Static"

cli/internal/terraform/terraform/azure/outputs.tf

@@ -2,6 +2,10 @@ output "ip" {
   value = azurerm_public_ip.loadbalancer_ip.ip_address
 }
 
+output "api_server_cert_sans" {
+  value = sort(concat([azurerm_public_ip.loadbalancer_ip.ip_address, local.wildcard_lb_dns_name], var.custom_endpoint == "" ? [] : [var.custom_endpoint]))
+}
+
 output "uid" {
   value = local.uid
 }

cli/internal/terraform/terraform/azure/variables.tf

@@ -61,3 +61,9 @@ variable "user_assigned_identity" {
   type        = string
   description = "The name of the user assigned identity to attache to the nodes of the cluster."
 }
+
+variable "custom_endpoint" {
+  type        = string
+  default     = ""
+  description = "Custom endpoint to use for the Kubernetes apiserver. If not set, the default endpoint will be used."
+}

cli/internal/terraform/terraform/gcp/main.tf

@@ -30,10 +30,12 @@ provider "google-beta" {
 }
 
 locals {
-  uid                   = random_id.uid.hex
-  name                  = "${var.name}-${local.uid}"
-  initSecretHash        = random_password.initSecret.bcrypt_hash
-  labels                = { constellation-uid = local.uid }
+  uid            = random_id.uid.hex
+  name           = "${var.name}-${local.uid}"
+  initSecretHash = random_password.initSecret.bcrypt_hash
+  labels = {
+    constellation-uid = local.uid,
+  }
   ports_node_range      = "30000-32767"
   ports_kubernetes      = "6443"
   ports_bootstrapper    = "9000"
@@ -170,6 +172,7 @@ module "instance_group" {
   named_ports         = each.value.role == "control-plane" ? local.control_plane_named_ports : []
   labels              = local.labels
   init_secret_hash    = local.initSecretHash
+  custom_endpoint     = var.custom_endpoint
 }
 
 resource "google_compute_global_address" "loadbalancer_ip" {

cli/internal/terraform/terraform/gcp/modules/instance_group/variables.tf

@@ -94,3 +94,8 @@ variable "zone" {
   type        = string
   description = "Zone to deploy the instance group in."
 }
+
+variable "custom_endpoint" {
+  type        = string
+  description = "Custom endpoint to use for the Kubernetes apiserver. If not set, the default endpoint will be used."
+}

cli/internal/terraform/terraform/gcp/outputs.tf

@@ -2,6 +2,14 @@ output "ip" {
   value = google_compute_global_address.loadbalancer_ip.address
 }
 
+output "api_server_cert_sans" {
+  value = sort(concat([google_compute_global_address.loadbalancer_ip.address], var.custom_endpoint == "" ? [] : [var.custom_endpoint]))
+}
+
+output "fallback_endpoint" {
+  value = google_compute_global_address.loadbalancer_ip.address
+}
+
 output "uid" {
   value = local.uid
 }

cli/internal/terraform/terraform/gcp/variables.tf

@@ -45,3 +45,9 @@ variable "debug" {
   default     = false
   description = "Enable debug mode. This opens up a debugd port that can be used to deploy a custom bootstrapper."
 }
+
+variable "custom_endpoint" {
+  type        = string
+  default     = ""
+  description = "Custom endpoint to use for the Kubernetes apiserver. If not set, the default endpoint will be used."
+}

cli/internal/terraform/terraform/openstack/outputs.tf

@@ -2,6 +2,10 @@ output "ip" {
   value = openstack_networking_floatingip_v2.public_ip.address
 }
 
+output "api_server_cert_sans" {
+  value = sort(concat([openstack_networking_floatingip_v2.public_ip.address], var.custom_endpoint == "" ? [] : [var.custom_endpoint]))
+}
+
 output "uid" {
   value = local.uid
 }

cli/internal/terraform/terraform/openstack/variables.tf

@@ -67,3 +67,9 @@ variable "debug" {
   default     = false
   description = "Enable debug mode. This opens up a debugd port that can be used to deploy a custom bootstrapper."
 }
+
+variable "custom_endpoint" {
+  type        = string
+  default     = ""
+  description = "Custom endpoint to use for the Kubernetes apiserver. If not set, the default endpoint will be used."
+}

cli/internal/terraform/terraform/qemu/outputs.tf

@@ -2,6 +2,10 @@ output "ip" {
   value = module.node_group["control_plane_default"].instance_ips[0]
 }
 
+output "api_server_cert_sans" {
+  value = sort(concat([module.node_group["control_plane_default"].instance_ips[0]], var.custom_endpoint == "" ? [] : [var.custom_endpoint]))
+}
+
 output "uid" {
   value = "qemu" // placeholder
 }

cli/internal/terraform/terraform/qemu/variables.tf

@@ -96,3 +96,9 @@ variable "name" {
   default     = "constellation"
   description = "name prefix of the cluster VMs"
 }
+
+variable "custom_endpoint" {
+  type        = string
+  default     = ""
+  description = "Custom endpoint to use for the Kubernetes apiserver. If not set, the default endpoint will be used."
+}