gcp: unofficial support for internal LBs (#826)

hack/terraform/gcp/internal-loadbalancer/.terraform.lock.hcl

@@ -0,0 +1,79 @@
+# This file is maintained automatically by "terraform init".
+# Manual edits may be lost in future updates.
+
+provider "registry.terraform.io/hashicorp/google" {
+  version     = "4.46.0"
+  constraints = "4.46.0"
+  hashes = [
+    "h1:2DY8gHha2ayaTnFLMmjr11j3/mTyhM8FZr11GWPm1Rc=",
+    "h1:8stjYyQklFKl9vP2p/nBJ2r8uDYdRHAztAN4h4xBdbE=",
+    "h1:Co+X4DRbMUtaJXVb+4ZtzE/DcuzdhwJmsTyjfNOTCGQ=",
+    "h1:EWUedNKJvJNtiFbzkTx5u9PH9xGDpA9QLfKm+zvu4a8=",
+    "h1:HSbC3sovuqGBwzlKOAeJEeQl36SiwPkMrEshBkA7Crs=",
+    "h1:KDS0EbTOCdtJ0U0bfqI34jh82cyb9UQcYqIHB4T3Un4=",
+    "h1:Qd6DB82WvTULNghWDeX7RdHkEBWNEF9Bxf93cpK+jxg=",
+    "h1:Tyk+Po2wfXa+QHLW9AzL8pYIVWrbsmC929owx2PFxpU=",
+    "h1:c5SNofELMuqNisIlmIWqlLFUkf+sKDntPKtokLTs+BY=",
+    "h1:d8LpLqPxKAVlGUcXaOtbumVYu0vazovHO+I3JwxpyaQ=",
+    "h1:xS2gA1JFaHIYTDZC4CnZxcLbHVn4rkJcdplrBvRf9HM=",
+    "zh:06667e8c975734c0fbfe9cdbc6ee6789f3114e40bb97a8a721ed87f3e5c42dfc",
+    "zh:0e9829e326095d142ffa394f7189528001b074249b2ec8f98567ada049e35cbf",
+    "zh:1feceb13f5dfd658bcd94886c495b8064096f58bccb65f8386465803ae67e821",
+    "zh:404a642769271a290eb600c238f33ecaeb5c6d1354694d0ee6f83030941fde00",
+    "zh:46923a708123777a383aec56d0e834eae9095e440511f27c8ac5d0dfaf9b937b",
+    "zh:5f7b32404f70444dbe9e89d4ba0f9599d76f1072da2a52ca207a14e117ba165b",
+    "zh:a9d3d03c5e9cf904f08906ab13ad5ad8ff3f2271565fda75cd32c6ba528a4ea3",
+    "zh:aaef3d327fd6c8fbe33c454103ac0bad3e2043ff6bcb6c0aaa0d5974695dc8e4",
+    "zh:b49ea5676065543c2c28a0f5fef28058ad8fc96894782123fb9c013cd083be98",
+    "zh:f432cb018a4826b7a61e39f10239ca017a74313e0755b24858f082395e38bf9a",
+    "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c",
+    "zh:fe5dde2af422510ba2a1388209dc94723ee740b4282323aafa199366705387cc",
+  ]
+}
+
+provider "registry.terraform.io/hashicorp/google-beta" {
+  version = "4.46.0"
+  hashes = [
+    "h1:Aj3ywOJ6IqwOHTMPIZFXQt1x2bMK/D/FHnhr7ft63bY=",
+    "h1:GQgp57Wrf3oDVB/nDtBd/VL3/uMTFN5bLPhPLLADGB4=",
+    "h1:SGL8Ge/ofpY9jtx6bapYkcwDzXNnDydrXpwW5BlbVFk=",
+    "h1:ZbWF1DV/Zdjk9yMpu5iRwxxviwAL1liehBM3/EHOxMw=",
+    "h1:zGs24p4gAJJdVTx2nNxWDtE14Ihd1RT27qU7GwqHQ7Q=",
+    "zh:08aa990fd9944061194138ad4f136f5e6b45f331d110d882e4ddb566619eb9d8",
+    "zh:186b9c7b49ad93a2ab2d8d713429caa8b23dab8d90763c01244205c3455dd813",
+    "zh:221598948eab9c64e13a778c6be17dd1e9cb2e08a3217072d9759202986c3f09",
+    "zh:402d386ea907923bbf36568dc481becda2dd0522c5286602dcb716f364f73d91",
+    "zh:84d70da182503ce312148cc86e110482c88d57041223af00d2ead60fefe851ee",
+    "zh:92bd8e30f6334988d6e7fedff11b99c68fe0d21bfead6f1cbbfc73acb665c36d",
+    "zh:a0e5815460c2a1d720955a2abbd6ca6eef450da7a76c52c223203f5d03ca45d7",
+    "zh:c34c0124f70b86ceb4cf79b93539539286f47175e1a648e37cfb754200cc19b7",
+    "zh:d00769dbfd1bea46da5ac81a9e3384d0f954ed3f912859b21ed3ea2378c2cb1d",
+    "zh:dadd937ed05dcc3d521cd6ca04bc9681fd30acbb42265f748e69f5feb0f5b829",
+    "zh:e4c2d65cf138cc868ae52e3e2cf97e37fbbdbc9c510f1c669fdbcc736256e402",
+    "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c",
+  ]
+}
+
+provider "registry.terraform.io/hashicorp/random" {
+  version     = "3.4.3"
+  constraints = "3.4.3"
+  hashes = [
+    "h1:hV66lcagXXRwwCW3Y542bI1JgPo8z/taYKT7K+a2Z5U=",
+    "h1:hXUPrH8igYBhatzatkp80RCeeUJGu9lQFDyKemOlsTo=",
+    "h1:saZR+mhthL0OZl4SyHXZraxyaBNVMxiZzks78nWcZ2o=",
+    "h1:tL3katm68lX+4lAncjQA9AXL4GR/VM+RPwqYf4D2X8Q=",
+    "h1:xZGZf18JjMS06pFa4NErzANI98qi59SEcBsOcS2P2yQ=",
+    "zh:41c53ba47085d8261590990f8633c8906696fa0a3c4b384ff6a7ecbf84339752",
+    "zh:59d98081c4475f2ad77d881c4412c5129c56214892f490adf11c7e7a5a47de9b",
+    "zh:686ad1ee40b812b9e016317e7f34c0d63ef837e084dea4a1f578f64a6314ad53",
+    "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3",
+    "zh:84103eae7251384c0d995f5a257c72b0096605048f757b749b7b62107a5dccb3",
+    "zh:8ee974b110adb78c7cd18aae82b2729e5124d8f115d484215fd5199451053de5",
+    "zh:9dd4561e3c847e45de603f17fa0c01ae14cae8c4b7b4e6423c9ef3904b308dda",
+    "zh:bb07bb3c2c0296beba0beec629ebc6474c70732387477a65966483b5efabdbc6",
+    "zh:e891339e96c9e5a888727b45b2e1bb3fcbdfe0fd7c5b4396e4695459b38c8cb1",
+    "zh:ea4739860c24dfeaac6c100b2a2e357106a89d18751f7693f3c31ecf6a996f8d",
+    "zh:f0c76ac303fd0ab59146c39bc121c5d7d86f878e9a69294e29444d4c653786f8",
+    "zh:f143a9a5af42b38fed328a161279906759ff39ac428ebcfe55606e05e1518b93",
+  ]
+}

hack/terraform/gcp/internal-loadbalancer/main.tf

@@ -0,0 +1,284 @@
+terraform {
+  required_providers {
+    google = {
+      source  = "hashicorp/google"
+      version = "4.46.0"
+    }
+    random = {
+      source  = "hashicorp/random"
+      version = "3.4.3"
+    }
+  }
+}
+
+provider "google" {
+  project = var.project
+  region  = var.region
+  zone    = var.zone
+}
+
+provider "google-beta" {
+  project = var.project
+  region  = var.region
+  zone    = var.zone
+}
+
+locals {
+  uid                    = random_id.uid.hex
+  name                   = "${var.name}-${local.uid}"
+  initSecretHash         = random_password.initSecret.bcrypt_hash
+  labels                 = { constellation-uid = local.uid }
+  ports_node_range       = "30000-32767"
+  ports_kubernetes       = "6443"
+  ports_bootstrapper     = "9000"
+  ports_konnectivity     = "8132"
+  ports_verify           = "30081"
+  ports_recovery         = "9999"
+  ports_debugd           = "4000"
+  cidr_vpc_subnet_nodes  = "192.168.178.0/24"
+  cidr_vpc_subnet_pods   = "10.10.0.0/16"
+  cidr_vpc_subnet_public = "10.11.0.0/16"
+  cidr_vpc_subnet_ilb    = "10.12.0.0/16"
+  kube_env               = "AUTOSCALER_ENV_VARS: kube_reserved=cpu=1060m,memory=1019Mi,ephemeral-storage=41Gi;node_labels=;os=linux;os_distribution=cos;evictionHard="
+}
+
+resource "random_id" "uid" {
+  byte_length = 4
+}
+
+resource "random_password" "initSecret" {
+  length           = 32
+  special          = true
+  override_special = "_%@"
+}
+
+resource "google_compute_network" "vpc_network" {
+  name                    = local.name
+  auto_create_subnetworks = false
+}
+
+
+resource "google_compute_subnetwork" "vpc_subnetwork_backend" {
+  name          = "${local.name}-be"
+  description   = "Constellation backend VPC subnetwork"
+  network       = google_compute_network.vpc_network.id
+  ip_cidr_range = local.cidr_vpc_subnet_nodes
+  secondary_ip_range = [
+    {
+      range_name    = local.name,
+      ip_cidr_range = local.cidr_vpc_subnet_pods,
+    }
+  ]
+}
+
+# backed subnet
+resource "google_compute_subnetwork" "ilb_subnet" {
+  name          = "${local.name}-ilb"
+  ip_cidr_range = local.cidr_vpc_subnet_ilb
+  region        = var.region
+  purpose       = "REGIONAL_MANAGED_PROXY"
+  role          = "ACTIVE"
+  network       = google_compute_network.vpc_network.id
+}
+
+resource "google_compute_subnetwork" "vpc_subnetwork_public" {
+  name          = "${local.name}-pub"
+  description   = "Constellation public VPC subnetwork"
+  network       = google_compute_network.vpc_network.id
+  ip_cidr_range = local.cidr_vpc_subnet_public
+}
+
+resource "google_compute_router" "vpc_router" {
+  name        = local.name
+  description = "Constellation VPC router"
+  network     = google_compute_network.vpc_network.id
+}
+
+resource "google_compute_router_nat" "vpc_router_nat" {
+  name                               = local.name
+  router                             = google_compute_router.vpc_router.name
+  nat_ip_allocate_option             = "AUTO_ONLY"
+  source_subnetwork_ip_ranges_to_nat = "ALL_SUBNETWORKS_ALL_IP_RANGES"
+}
+
+resource "google_compute_firewall" "firewall_external" {
+  name          = local.name
+  description   = "Constellation VPC firewall"
+  network       = google_compute_network.vpc_network.id
+  source_ranges = ["0.0.0.0/0"]
+  direction     = "INGRESS"
+
+  allow {
+    protocol = "tcp"
+    ports = flatten([
+      local.ports_node_range,
+      local.ports_bootstrapper,
+      local.ports_kubernetes,
+      local.ports_konnectivity,
+      local.ports_recovery,
+      var.debug ? [local.ports_debugd] : [],
+    ])
+  }
+
+}
+
+resource "google_compute_firewall" "firewall_internal_nodes" {
+  name          = "${local.name}-nodes"
+  description   = "Constellation VPC firewall"
+  network       = google_compute_network.vpc_network.id
+  source_ranges = [local.cidr_vpc_subnet_nodes]
+  direction     = "INGRESS"
+
+  allow { protocol = "tcp" }
+  allow { protocol = "udp" }
+  allow { protocol = "icmp" }
+}
+
+resource "google_compute_firewall" "firewall_internal_pods" {
+  name          = "${local.name}-pods"
+  description   = "Constellation VPC firewall"
+  network       = google_compute_network.vpc_network.id
+  source_ranges = [local.cidr_vpc_subnet_pods]
+  direction     = "INGRESS"
+
+  allow { protocol = "tcp" }
+  allow { protocol = "udp" }
+  allow { protocol = "icmp" }
+}
+
+module "instance_group_control_plane" {
+  source         = "./modules/instance_group"
+  name           = local.name
+  role           = "ControlPlane"
+  uid            = local.uid
+  instance_type  = var.instance_type
+  instance_count = var.control_plane_count
+  image_id       = var.image_id
+  disk_size      = var.state_disk_size
+  disk_type      = var.state_disk_type
+  network        = google_compute_network.vpc_network.id
+  subnetwork     = google_compute_subnetwork.vpc_subnetwork_backend.id
+  kube_env       = local.kube_env
+  debug          = var.debug
+  named_ports = flatten([
+    { name = "kubernetes", port = local.ports_kubernetes },
+    { name = "bootstrapper", port = local.ports_bootstrapper },
+    { name = "verify", port = local.ports_verify },
+    { name = "konnectivity", port = local.ports_konnectivity },
+    { name = "recovery", port = local.ports_recovery },
+    var.debug ? [{ name = "debugd", port = local.ports_debugd }] : [],
+  ])
+  labels           = local.labels
+  init_secret_hash = local.initSecretHash
+}
+
+module "instance_group_worker" {
+  source           = "./modules/instance_group"
+  name             = local.name
+  role             = "Worker"
+  uid              = local.uid
+  instance_type    = var.instance_type
+  instance_count   = var.worker_count
+  image_id         = var.image_id
+  disk_size        = var.state_disk_size
+  disk_type        = var.state_disk_type
+  network          = google_compute_network.vpc_network.id
+  subnetwork       = google_compute_subnetwork.vpc_subnetwork_backend.id
+  kube_env         = local.kube_env
+  debug            = var.debug
+  labels           = local.labels
+  init_secret_hash = local.initSecretHash
+}
+
+resource "google_compute_address" "loadbalancer_ip" {
+  name         = local.name
+  region       = var.region
+  subnetwork   = google_compute_subnetwork.vpc_subnetwork_backend.id
+  purpose      = "SHARED_LOADBALANCER_VIP"
+  address_type = "INTERNAL"
+}
+
+module "loadbalancer_kube" {
+  source                 = "./modules/internal_loadbalancer"
+  name                   = local.name
+  port_name              = "kubernetes"
+  region                 = var.region
+  network                = google_compute_network.vpc_network.id
+  backend_subnet         = google_compute_subnetwork.vpc_subnetwork_backend.id
+  health_check           = "HTTPS"
+  backend_instance_group = module.instance_group_control_plane.instance_group
+  ip_address             = google_compute_address.loadbalancer_ip.self_link
+  port                   = local.ports_kubernetes
+  frontend_labels        = merge(local.labels, { constellation-use = "kubernetes" })
+}
+
+module "loadbalancer_boot" {
+  source                 = "./modules/internal_loadbalancer"
+  name                   = local.name
+  port_name              = "bootstrapper"
+  region                 = var.region
+  network                = google_compute_network.vpc_network.id
+  backend_subnet         = google_compute_subnetwork.vpc_subnetwork_backend.id
+  health_check           = "TCP"
+  backend_instance_group = module.instance_group_control_plane.instance_group
+  ip_address             = google_compute_address.loadbalancer_ip.self_link
+  port                   = local.ports_bootstrapper
+  frontend_labels        = merge(local.labels, { constellation-use = "bootstrapper" })
+}
+
+module "loadbalancer_verify" {
+  source                 = "./modules/internal_loadbalancer"
+  name                   = local.name
+  port_name              = "verify"
+  region                 = var.region
+  network                = google_compute_network.vpc_network.id
+  backend_subnet         = google_compute_subnetwork.vpc_subnetwork_backend.id
+  health_check           = "TCP"
+  backend_instance_group = module.instance_group_control_plane.instance_group
+  ip_address             = google_compute_address.loadbalancer_ip.self_link
+  port                   = local.ports_verify
+  frontend_labels        = merge(local.labels, { constellation-use = "verify" })
+}
+
+module "loadbalancer_konnectivity" {
+  source                 = "./modules/internal_loadbalancer"
+  name                   = local.name
+  port_name              = "konnectivity"
+  region                 = var.region
+  network                = google_compute_network.vpc_network.id
+  backend_subnet         = google_compute_subnetwork.vpc_subnetwork_backend.id
+  health_check           = "TCP"
+  backend_instance_group = module.instance_group_control_plane.instance_group
+  ip_address             = google_compute_address.loadbalancer_ip.self_link
+  port                   = local.ports_konnectivity
+  frontend_labels        = merge(local.labels, { constellation-use = "konnectivity" })
+}
+
+module "loadbalancer_recovery" {
+  source                 = "./modules/internal_loadbalancer"
+  name                   = local.name
+  port_name              = "recovery"
+  region                 = var.region
+  network                = google_compute_network.vpc_network.id
+  backend_subnet         = google_compute_subnetwork.vpc_subnetwork_backend.id
+  health_check           = "TCP"
+  backend_instance_group = module.instance_group_control_plane.instance_group
+  ip_address             = google_compute_address.loadbalancer_ip.self_link
+  port                   = local.ports_recovery
+  frontend_labels        = merge(local.labels, { constellation-use = "recovery" })
+}
+
+module "loadbalancer_debugd" {
+  count                  = var.debug ? 1 : 0 // only deploy debugd in debug mode
+  source                 = "./modules/internal_loadbalancer"
+  name                   = local.name
+  port_name              = "debugd"
+  region                 = var.region
+  network                = google_compute_network.vpc_network.id
+  backend_subnet         = google_compute_subnetwork.vpc_subnetwork_backend.id
+  health_check           = "TCP"
+  backend_instance_group = module.instance_group_control_plane.instance_group
+  ip_address             = google_compute_address.loadbalancer_ip.self_link
+  port                   = local.ports_debugd
+  frontend_labels        = merge(local.labels, { constellation-use = "debugd" })
+}

hack/terraform/gcp/internal-loadbalancer/modules/instance_group/main.tf

@@ -0,0 +1,106 @@
+terraform {
+  required_providers {
+    google = {
+      source  = "hashicorp/google"
+      version = "4.46.0"
+    }
+  }
+}
+
+locals {
+  role_dashed = var.role == "ControlPlane" ? "control-plane" : "worker"
+  name        = "${var.name}-${local.role_dashed}"
+}
+
+resource "google_compute_instance_template" "template" {
+  name         = local.name
+  machine_type = var.instance_type
+  tags         = ["constellation-${var.uid}"] // Note that this is also applied as a label 
+  labels       = merge(var.labels, { constellation-role = local.role_dashed })
+
+  confidential_instance_config {
+    enable_confidential_compute = true
+  }
+
+  disk {
+    disk_size_gb = 10
+    source_image = var.image_id
+    auto_delete  = true
+    boot         = true
+    mode         = "READ_WRITE"
+  }
+
+  disk {
+    disk_size_gb = var.disk_size
+    disk_type    = var.disk_type
+    auto_delete  = true
+    device_name  = "state-disk" // This name is used by disk mapper to find the disk
+    boot         = false
+    mode         = "READ_WRITE"
+    type         = "PERSISTENT"
+  }
+
+  metadata = {
+    kube-env                       = var.kube_env
+    constellation-init-secret-hash = var.init_secret_hash
+    serial-port-enable             = var.debug ? "TRUE" : "FALSE"
+  }
+
+  network_interface {
+    network    = var.network
+    subnetwork = var.subnetwork
+    alias_ip_range {
+      ip_cidr_range         = "/24"
+      subnetwork_range_name = var.name
+    }
+  }
+
+  scheduling {
+    on_host_maintenance = "TERMINATE"
+  }
+
+  service_account {
+    scopes = [
+      "https://www.googleapis.com/auth/compute",
+      "https://www.googleapis.com/auth/servicecontrol",
+      "https://www.googleapis.com/auth/service.management.readonly",
+      "https://www.googleapis.com/auth/devstorage.read_only",
+      "https://www.googleapis.com/auth/logging.write",
+      "https://www.googleapis.com/auth/monitoring.write",
+      "https://www.googleapis.com/auth/trace.append",
+      "https://www.googleapis.com/auth/cloud-platform",
+    ]
+  }
+
+  shielded_instance_config {
+    enable_secure_boot          = true
+    enable_vtpm                 = true
+    enable_integrity_monitoring = true
+  }
+}
+
+resource "google_compute_instance_group_manager" "instance_group_manager" {
+  name               = local.name
+  description        = "Instance group manager for Constellation"
+  base_instance_name = local.name
+  target_size        = var.instance_count
+
+  version {
+    instance_template = google_compute_instance_template.template.id
+  }
+
+  dynamic "named_port" {
+    for_each = toset(var.named_ports)
+    content {
+      name = named_port.value.name
+      port = named_port.value.port
+    }
+  }
+
+  lifecycle {
+    ignore_changes = [
+      target_size, # required. autoscaling modifies the instance count externally
+      version,     # required. update procedure modifies the instance template externally
+    ]
+  }
+}

hack/terraform/gcp/internal-loadbalancer/modules/instance_group/outputs.tf

@@ -0,0 +1,3 @@
+output "instance_group" {
+  value = google_compute_instance_group_manager.instance_group_manager.instance_group
+}

hack/terraform/gcp/internal-loadbalancer/modules/instance_group/variables.tf

@@ -0,0 +1,77 @@
+variable "name" {
+  type        = string
+  description = "Base name of the instance group."
+}
+
+variable "role" {
+  type        = string
+  description = "The role of the instance group. Has to be 'ControlPlane' or 'Worker'."
+}
+
+variable "uid" {
+  type        = string
+  description = "UID of the cluster. This is used for tags."
+}
+
+variable "labels" {
+  type        = map(string)
+  default     = {}
+  description = "Labels to apply to the instance group."
+}
+
+variable "instance_type" {
+  type        = string
+  description = "Instance type for the nodes."
+}
+
+variable "instance_count" {
+  type        = number
+  description = "Number of instances in the instance group."
+}
+
+variable "image_id" {
+  type        = string
+  description = "Image ID for the nodes."
+}
+
+variable "disk_size" {
+  type        = number
+  description = "Disk size for the nodes, in GB."
+}
+
+variable "disk_type" {
+  type        = string
+  description = "Disk type for the nodes. Has to be 'pd-standard' or 'pd-ssd'."
+}
+
+variable "network" {
+  type        = string
+  description = "Name of the network to use."
+}
+
+variable "subnetwork" {
+  type        = string
+  description = "Name of the subnetwork to use."
+}
+
+variable "kube_env" {
+  type        = string
+  description = "Kubernetes env."
+}
+
+variable "init_secret_hash" {
+  type        = string
+  description = "Hash of the init secret."
+}
+
+variable "named_ports" {
+  type        = list(object({ name = string, port = number }))
+  default     = []
+  description = "Named ports for the instance group."
+}
+
+variable "debug" {
+  type        = bool
+  default     = false
+  description = "Enable debug mode. This will enable serial port access on the instances."
+}

hack/terraform/gcp/internal-loadbalancer/modules/internal_loadbalancer/main.tf

@@ -0,0 +1,75 @@
+terraform {
+  required_providers {
+    google = {
+      source  = "hashicorp/google"
+      version = "4.46.0"
+    }
+  }
+}
+
+locals {
+  name = "${var.name}-${var.port_name}"
+}
+
+# forwarding rule
+resource "google_compute_forwarding_rule" "forwarding" {
+  name                  = local.name
+  network               = var.network
+  subnetwork            = var.backend_subnet
+  region                = var.region
+  ip_address            = var.ip_address
+  ip_protocol           = "TCP"
+  load_balancing_scheme = "INTERNAL_MANAGED"
+  port_range            = var.port
+  allow_global_access   = true
+  target                = google_compute_region_target_tcp_proxy.proxy.id
+  labels                = var.frontend_labels
+}
+
+resource "google_compute_region_backend_service" "backend" {
+  name                  = local.name
+  region                = var.region
+  port_name             = var.port_name
+  protocol              = "TCP"
+  load_balancing_scheme = "INTERNAL_MANAGED"
+
+  backend {
+    group           = var.backend_instance_group
+    balancing_mode  = "UTILIZATION"
+    capacity_scaler = 1.0
+  }
+
+
+  health_checks = [google_compute_region_health_check.health.id]
+  timeout_sec   = 240
+}
+
+resource "google_compute_region_target_tcp_proxy" "proxy" {
+  provider        = google-beta
+  name            = local.name
+  region          = var.region
+  backend_service = google_compute_region_backend_service.backend.id
+}
+
+resource "google_compute_region_health_check" "health" {
+  name               = local.name
+  region             = var.region
+  check_interval_sec = 1
+  timeout_sec        = 1
+
+  dynamic "tcp_health_check" {
+    for_each = var.health_check == "TCP" ? [1] : []
+    content {
+      port = var.port
+    }
+  }
+
+  dynamic "https_health_check" {
+    for_each = var.health_check == "HTTPS" ? [1] : []
+    content {
+      host         = ""
+      port         = var.port
+      request_path = "/readyz"
+    }
+  }
+}

hack/terraform/gcp/internal-loadbalancer/modules/internal_loadbalancer/variables.tf

@@ -0,0 +1,54 @@
+variable "name" {
+  type        = string
+  description = "Base name of the load balancer."
+}
+
+variable "region" {
+  type        = string
+  description = "The region where the load balancer will be created."
+}
+
+variable "network" {
+  type        = string
+  description = "The network to which all network resources will be attached."
+}
+
+variable "backend_subnet" {
+  type        = string
+  description = "The subnet to which all backend network resources will be attached."
+}
+
+variable "health_check" {
+  type        = string
+  description = "The type of the health check. 'HTTPS' or 'TCP'."
+  validation {
+    condition     = contains(["HTTPS", "TCP"], var.health_check)
+    error_message = "Health check must be either 'HTTPS' or 'TCP'."
+  }
+}
+
+variable "port" {
+  type        = string
+  description = "The port on which to listen for incoming traffic."
+}
+
+variable "port_name" {
+  type        = string
+  description = "Name of backend port. The same name should appear in the instance groups referenced by this service."
+}
+
+variable "backend_instance_group" {
+  type        = string
+  description = "The URL of the instance group resource from which the load balancer will direct traffic."
+}
+
+variable "ip_address" {
+  type        = string
+  description = "The IP address that this forwarding rule serves."
+}
+
+variable "frontend_labels" {
+  type        = map(string)
+  default     = {}
+  description = "Labels to apply to the forwarding rule."
+}

hack/terraform/gcp/internal-loadbalancer/outputs.tf

@@ -0,0 +1,8 @@
+output "ip" {
+  value = google_compute_address.loadbalancer_ip.address
+}
+
+output "initSecret" {
+  value     = random_password.initSecret.result
+  sensitive = true
+}

hack/terraform/gcp/internal-loadbalancer/variables.tf

@@ -0,0 +1,58 @@
+variable "name" {
+  type        = string
+  default     = "constell"
+  description = "Base name of the cluster."
+}
+
+variable "control_plane_count" {
+  type        = number
+  description = "The number of control plane nodes to deploy."
+}
+
+variable "worker_count" {
+  type        = number
+  description = "The number of worker nodes to deploy."
+}
+
+variable "state_disk_size" {
+  type        = number
+  default     = 30
+  description = "The size of the state disk in GB."
+}
+
+variable "project" {
+  type        = string
+  description = "The GCP project to deploy the cluster in."
+}
+
+variable "region" {
+  type        = string
+  description = "The GCP region to deploy the cluster in."
+}
+
+variable "zone" {
+  type        = string
+  description = "The GCP zone to deploy the cluster in."
+}
+
+variable "instance_type" {
+  type        = string
+  description = "The GCP instance type to deploy."
+}
+
+variable "state_disk_type" {
+  type        = string
+  default     = "pd-ssd"
+  description = "The type of the state disk."
+}
+
+variable "image_id" {
+  type        = string
+  description = "The GCP image to use for the cluster nodes."
+}
+
+variable "debug" {
+  type        = bool
+  default     = false
+  description = "Enable debug mode. This opens up a debugd port that can be used to deploy a custom bootstrapper."
+}

internal/cloud/gcp/cloud.go

@@ -31,14 +31,18 @@ const (
 	tagUsage = "constellation-use"
 )
 
-var zoneFromRegionRegex = regexp.MustCompile("([a-z]*-[a-z]*[0-9])")
+var (
+	zoneFromRegionRegex = regexp.MustCompile("([a-z]*-[a-z]*[0-9])")
+	errNoForwardingRule = errors.New("no forwarding rule found")
+)
 
 // Cloud provides GCP cloud metadata information and API access.
 type Cloud struct {
-	forwardingRulesAPI forwardingRulesAPI
+	globalForwardingRulesAPI   globalForwardingRulesAPI
-	imds               imdsAPI
+	regionalForwardingRulesAPI regionalForwardingRulesAPI
-	instanceAPI        instanceAPI
+	imds                       imdsAPI
-	subnetAPI          subnetAPI
+	instanceAPI                instanceAPI
+	subnetAPI                  subnetAPI
 
 	closers []func() error
 }
@@ -53,11 +57,19 @@ func New(ctx context.Context) (cloud *Cloud, err error) {
 		return nil, err
 	}
 	closers = append(closers, insAPI.Close)
-	forwardingRulesAPI, err := compute.NewGlobalForwardingRulesRESTClient(ctx)
+
+	globalForwardingRulesAPI, err := compute.NewGlobalForwardingRulesRESTClient(ctx)
 	if err != nil {
 		return nil, err
 	}
-	closers = append(closers, forwardingRulesAPI.Close)
+	closers = append(closers, globalForwardingRulesAPI.Close)
+
+	regionalForwardingRulesAPI, err := compute.NewForwardingRulesRESTClient(ctx)
+	if err != nil {
+		return nil, err
+	}
+	closers = append(closers, regionalForwardingRulesAPI.Close)
+
 	subnetAPI, err := compute.NewSubnetworksRESTClient(ctx)
 	if err != nil {
 		return nil, err
@@ -65,11 +77,12 @@ func New(ctx context.Context) (cloud *Cloud, err error) {
 	closers = append(closers, subnetAPI.Close)
 
 	return &Cloud{
-		imds:               imds.NewClient(nil),
+		imds:                       imds.NewClient(nil),
-		instanceAPI:        &instanceClient{insAPI},
+		instanceAPI:                &instanceClient{insAPI},
-		forwardingRulesAPI: &forwardingRulesClient{forwardingRulesAPI},
+		globalForwardingRulesAPI:   &globalForwardingRulesClient{globalForwardingRulesAPI},
-		subnetAPI:          subnetAPI,
+		regionalForwardingRulesAPI: &regionalForwardingRulesClient{regionalForwardingRulesAPI},
-		closers:            closers,
+		subnetAPI:                  subnetAPI,
+		closers:                    closers,
 	}, nil
 }
 
@@ -91,8 +104,36 @@ func (c *Cloud) GetLoadBalancerEndpoint(ctx context.Context) (string, error) {
 		return "", err
 	}
 
+	// First try to find a global forwarding rule.
+	endpoint, err := c.getGlobalForwardingRule(ctx, project, uid)
+	if err != nil && !errors.Is(err, errNoForwardingRule) {
+		return "", fmt.Errorf("getting global forwarding rule: %w", err)
+	} else if err == nil {
+		return endpoint, nil
+	}
+
+	// If no global forwarding rule was found, try to find a regional forwarding rule.
+	region := zoneFromRegionRegex.FindString(zone)
+	if region == "" {
+		return "", fmt.Errorf("invalid zone %s", zone)
+	}
+
+	endpoint, err = c.getRegionalForwardingRule(ctx, project, uid, region)
+	if err != nil && !errors.Is(err, errNoForwardingRule) {
+		return "", fmt.Errorf("getting regional forwarding rule: %w", err)
+	} else if err == nil {
+		return endpoint, nil
+	}
+
+	return "", fmt.Errorf("kubernetes load balancer with UID %s not found: %w", uid, err)
+}
+
+// getGlobalForwardingRule returns the endpoint of the load balancer if it is a global load balancer.
+// This functions returns ErrNoForwardingRule if no forwarding rule was found.
+func (c *Cloud) getGlobalForwardingRule(ctx context.Context, project, uid string) (string, error) {
 	var resp *computepb.ForwardingRule
-	iter := c.forwardingRulesAPI.List(ctx, &computepb.ListGlobalForwardingRulesRequest{
+	var err error
+	iter := c.globalForwardingRulesAPI.List(ctx, &computepb.ListGlobalForwardingRulesRequest{
 		Project: project,
 		Filter:  proto.String(fmt.Sprintf("(labels.%s:%s) AND (labels.%s:kubernetes)", cloud.TagUID, uid, tagUsage)),
 	})
@@ -106,8 +147,39 @@ func (c *Cloud) GetLoadBalancerEndpoint(ctx context.Context) (string, error) {
 		portRange := strings.Split(*resp.PortRange, "-")
 		return net.JoinHostPort(*resp.IPAddress, portRange[0]), nil
 	}
+	if err != iterator.Done {
+		return "", fmt.Errorf("error listing global forwarding rules with UID %s: %w", uid, err)
+	}
+	return "", errNoForwardingRule
+}
 
-	return "", fmt.Errorf("kubernetes load balancer with UID %s not found: %w", uid, err)
+// getRegionalForwardingRule returns the endpoint of the load balancer if it is a regional load balancer.
+// This functions returns ErrNoForwardingRule if no forwarding rule was found.
+func (c *Cloud) getRegionalForwardingRule(ctx context.Context, project, uid, region string) (string, error) {
+	var resp *computepb.ForwardingRule
+	var err error
+	iter := c.regionalForwardingRulesAPI.List(ctx, &computepb.ListForwardingRulesRequest{
+		Project: project,
+		Region:  region,
+		Filter:  proto.String(fmt.Sprintf("(labels.%s:%s) AND (labels.%s:kubernetes)", cloud.TagUID, uid, tagUsage)),
+	})
+	for resp, err = iter.Next(); err == nil; resp, err = iter.Next() {
+		if resp.PortRange == nil {
+			continue
+		}
+		if resp.IPAddress == nil {
+			continue
+		}
+		if resp.Region == nil {
+			continue
+		}
+		portRange := strings.Split(*resp.PortRange, "-")
+		return net.JoinHostPort(*resp.IPAddress, portRange[0]), nil
+	}
+	if err != iterator.Done {
+		return "", fmt.Errorf("error listing global forwarding rules with UID %s: %w", uid, err)
+	}
+	return "", errNoForwardingRule
 }
 
 // List retrieves all instances belonging to the current constellation.

internal/cloud/gcp/cloud_test.go

@@ -207,13 +207,14 @@ func TestGetLoadbalancerEndpoint(t *testing.T) {
 	}
 
 	testCases := map[string]struct {
-		imds               stubIMDS
+		imds                       stubIMDS
-		instanceAPI        stubInstanceAPI
+		instanceAPI                stubInstanceAPI
-		forwardingRulesAPI stubForwardingRulesAPI
+		globalForwardingRulesAPI   stubGlobalForwardingRulesAPI
-		wantEndpoint       string
+		regionalForwardingRulesAPI stubRegionalForwardingRulesAPI
-		wantErr            bool
+		wantEndpoint               string
+		wantErr                    bool
 	}{
-		"success": {
+		"success global forwarding rule": {
 			imds: stubIMDS{
 				projectID:    "someProject",
 				zone:         "someZone-west3-b",
@@ -222,7 +223,7 @@ func TestGetLoadbalancerEndpoint(t *testing.T) {
 			instanceAPI: stubInstanceAPI{
 				instance: goodInstance,
 			},
-			forwardingRulesAPI: stubForwardingRulesAPI{
+			globalForwardingRulesAPI: stubGlobalForwardingRulesAPI{
 				iterator: &stubForwardingRulesIterator{
 					forwardingRules: []*computepb.ForwardingRule{
 						{
@@ -232,18 +233,49 @@ func TestGetLoadbalancerEndpoint(t *testing.T) {
 					},
 				},
 			},
+			regionalForwardingRulesAPI: stubRegionalForwardingRulesAPI{
+				iterator: &stubForwardingRulesIterator{},
+			},
 			wantEndpoint: "192.0.2.255:6443",
 		},
-		"imds error": {
+		"success regional forwarding rule": {
 			imds: stubIMDS{
-				projectIDErr: someErr,
+				projectID:    "someProject",
 				zone:         "someZone-west3-b",
 				instanceName: "someInstance",
 			},
 			instanceAPI: stubInstanceAPI{
 				instance: goodInstance,
 			},
-			forwardingRulesAPI: stubForwardingRulesAPI{
+			globalForwardingRulesAPI: stubGlobalForwardingRulesAPI{
+				iterator: &stubForwardingRulesIterator{},
+			},
+			regionalForwardingRulesAPI: stubRegionalForwardingRulesAPI{
+				iterator: &stubForwardingRulesIterator{
+					forwardingRules: []*computepb.ForwardingRule{
+						{
+							PortRange: proto.String("6443"),
+							IPAddress: proto.String("192.0.2.255"),
+							Region:    proto.String("someRegion"),
+						},
+					},
+				},
+			},
+			wantEndpoint: "192.0.2.255:6443",
+		},
+		"regional forwarding rule has no region": {
+			imds: stubIMDS{
+				projectID:    "someProject",
+				zone:         "someZone-west3-b",
+				instanceName: "someInstance",
+			},
+			instanceAPI: stubInstanceAPI{
+				instance: goodInstance,
+			},
+			globalForwardingRulesAPI: stubGlobalForwardingRulesAPI{
+				iterator: &stubForwardingRulesIterator{},
+			},
+			regionalForwardingRulesAPI: stubRegionalForwardingRulesAPI{
 				iterator: &stubForwardingRulesIterator{
 					forwardingRules: []*computepb.ForwardingRule{
 						{
@@ -255,7 +287,31 @@ func TestGetLoadbalancerEndpoint(t *testing.T) {
 			},
 			wantErr: true,
 		},
-		"iterator error": {
+		"imds error": {
+			imds: stubIMDS{
+				projectIDErr: someErr,
+				zone:         "someZone-west3-b",
+				instanceName: "someInstance",
+			},
+			instanceAPI: stubInstanceAPI{
+				instance: goodInstance,
+			},
+			globalForwardingRulesAPI: stubGlobalForwardingRulesAPI{
+				iterator: &stubForwardingRulesIterator{
+					forwardingRules: []*computepb.ForwardingRule{
+						{
+							PortRange: proto.String("6443"),
+							IPAddress: proto.String("192.0.2.255"),
+						},
+					},
+				},
+			},
+			regionalForwardingRulesAPI: stubRegionalForwardingRulesAPI{
+				iterator: &stubForwardingRulesIterator{},
+			},
+			wantErr: true,
+		},
+		"global forwarding rule iterator error": {
 			imds: stubIMDS{
 				projectID:    "someProject",
 				zone:         "someZone-west3-b",
@@ -264,7 +320,29 @@ func TestGetLoadbalancerEndpoint(t *testing.T) {
 			instanceAPI: stubInstanceAPI{
 				instance: goodInstance,
 			},
-			forwardingRulesAPI: stubForwardingRulesAPI{
+			globalForwardingRulesAPI: stubGlobalForwardingRulesAPI{
+				iterator: &stubForwardingRulesIterator{
+					err: someErr,
+				},
+			},
+			regionalForwardingRulesAPI: stubRegionalForwardingRulesAPI{
+				iterator: &stubForwardingRulesIterator{},
+			},
+			wantErr: true,
+		},
+		"regional forwarding rule iterator error": {
+			imds: stubIMDS{
+				projectID:    "someProject",
+				zone:         "someZone-west3-b",
+				instanceName: "someInstance",
+			},
+			instanceAPI: stubInstanceAPI{
+				instance: goodInstance,
+			},
+			globalForwardingRulesAPI: stubGlobalForwardingRulesAPI{
+				iterator: &stubForwardingRulesIterator{},
+			},
+			regionalForwardingRulesAPI: stubRegionalForwardingRulesAPI{
 				iterator: &stubForwardingRulesIterator{
 					err: someErr,
 				},
@@ -280,7 +358,10 @@ func TestGetLoadbalancerEndpoint(t *testing.T) {
 			instanceAPI: stubInstanceAPI{
 				instance: goodInstance,
 			},
-			forwardingRulesAPI: stubForwardingRulesAPI{
+			globalForwardingRulesAPI: stubGlobalForwardingRulesAPI{
+				iterator: &stubForwardingRulesIterator{},
+			},
+			regionalForwardingRulesAPI: stubRegionalForwardingRulesAPI{
 				iterator: &stubForwardingRulesIterator{},
 			},
 			wantErr: true,
@@ -294,7 +375,7 @@ func TestGetLoadbalancerEndpoint(t *testing.T) {
 			instanceAPI: stubInstanceAPI{
 				instance: goodInstance,
 			},
-			forwardingRulesAPI: stubForwardingRulesAPI{
+			globalForwardingRulesAPI: stubGlobalForwardingRulesAPI{
 				iterator: &stubForwardingRulesIterator{
 					forwardingRules: []*computepb.ForwardingRule{
 						{
@@ -303,6 +384,9 @@ func TestGetLoadbalancerEndpoint(t *testing.T) {
 					},
 				},
 			},
+			regionalForwardingRulesAPI: stubRegionalForwardingRulesAPI{
+				iterator: &stubForwardingRulesIterator{},
+			},
 			wantErr: true,
 		},
 		"missing IP address": {
@@ -314,7 +398,7 @@ func TestGetLoadbalancerEndpoint(t *testing.T) {
 			instanceAPI: stubInstanceAPI{
 				instance: goodInstance,
 			},
-			forwardingRulesAPI: stubForwardingRulesAPI{
+			globalForwardingRulesAPI: stubGlobalForwardingRulesAPI{
 				iterator: &stubForwardingRulesIterator{
 					forwardingRules: []*computepb.ForwardingRule{
 						{
@@ -323,6 +407,9 @@ func TestGetLoadbalancerEndpoint(t *testing.T) {
 					},
 				},
 			},
+			regionalForwardingRulesAPI: stubRegionalForwardingRulesAPI{
+				iterator: &stubForwardingRulesIterator{},
+			},
 			wantErr: true,
 		},
 		"get instance error": {
@@ -334,7 +421,7 @@ func TestGetLoadbalancerEndpoint(t *testing.T) {
 			instanceAPI: stubInstanceAPI{
 				instanceErr: someErr,
 			},
-			forwardingRulesAPI: stubForwardingRulesAPI{
+			globalForwardingRulesAPI: stubGlobalForwardingRulesAPI{
 				iterator: &stubForwardingRulesIterator{
 					forwardingRules: []*computepb.ForwardingRule{
 						{
@@ -344,6 +431,9 @@ func TestGetLoadbalancerEndpoint(t *testing.T) {
 					},
 				},
 			},
+			regionalForwardingRulesAPI: stubRegionalForwardingRulesAPI{
+				iterator: &stubForwardingRulesIterator{},
+			},
 			wantErr: true,
 		},
 		"invalid instance": {
@@ -355,7 +445,7 @@ func TestGetLoadbalancerEndpoint(t *testing.T) {
 			instanceAPI: stubInstanceAPI{
 				instance: nil,
 			},
-			forwardingRulesAPI: stubForwardingRulesAPI{
+			globalForwardingRulesAPI: stubGlobalForwardingRulesAPI{
 				iterator: &stubForwardingRulesIterator{
 					forwardingRules: []*computepb.ForwardingRule{
 						{
@@ -365,6 +455,9 @@ func TestGetLoadbalancerEndpoint(t *testing.T) {
 					},
 				},
 			},
+			regionalForwardingRulesAPI: stubRegionalForwardingRulesAPI{
+				iterator: &stubForwardingRulesIterator{},
+			},
 			wantErr: true,
 		},
 	}
@@ -374,9 +467,10 @@ func TestGetLoadbalancerEndpoint(t *testing.T) {
 			assert := assert.New(t)
 
 			cloud := &Cloud{
-				imds:               &tc.imds,
+				imds:                       &tc.imds,
-				instanceAPI:        &tc.instanceAPI,
+				instanceAPI:                &tc.instanceAPI,
-				forwardingRulesAPI: &tc.forwardingRulesAPI,
+				globalForwardingRulesAPI:   &tc.globalForwardingRulesAPI,
+				regionalForwardingRulesAPI: &tc.regionalForwardingRulesAPI,
 			}
 
 			endpoint, err := cloud.GetLoadBalancerEndpoint(context.Background())
@@ -861,17 +955,29 @@ func TestInitSecretHash(t *testing.T) {
 	}
 }
 
-type stubForwardingRulesAPI struct {
+type stubGlobalForwardingRulesAPI struct {
 	iterator forwardingRuleIterator
 }
 
-func (s *stubForwardingRulesAPI) List(
+func (s *stubGlobalForwardingRulesAPI) List(
 	ctx context.Context, req *computepb.ListGlobalForwardingRulesRequest, opts ...gax.CallOption,
 ) forwardingRuleIterator {
 	return s.iterator
 }
 
-func (s *stubForwardingRulesAPI) Close() error { return nil }
+func (s *stubGlobalForwardingRulesAPI) Close() error { return nil }
+
+type stubRegionalForwardingRulesAPI struct {
+	iterator forwardingRuleIterator
+}
+
+func (s *stubRegionalForwardingRulesAPI) List(
+	ctx context.Context, req *computepb.ListForwardingRulesRequest, opts ...gax.CallOption,
+) forwardingRuleIterator {
+	return s.iterator
+}
+
+func (s *stubRegionalForwardingRulesAPI) Close() error { return nil }
 
 type stubForwardingRulesIterator struct {
 	ctr             int

internal/cloud/gcp/interface.go

@@ -13,11 +13,16 @@ import (
 	"github.com/googleapis/gax-go/v2"
 )
 
-type forwardingRulesAPI interface {
+type globalForwardingRulesAPI interface {
 	List(ctx context.Context, req *computepb.ListGlobalForwardingRulesRequest, opts ...gax.CallOption) forwardingRuleIterator
 	Close() error
 }
 
+type regionalForwardingRulesAPI interface {
+	List(ctx context.Context, req *computepb.ListForwardingRulesRequest, opts ...gax.CallOption) forwardingRuleIterator
+	Close() error
+}
+
 type imdsAPI interface {
 	InstanceID() (string, error)
 	ProjectID() (string, error)

internal/cloud/gcp/wrappers.go

@@ -22,20 +22,34 @@ type instanceIterator interface {
 	Next() (*computepb.Instance, error)
 }
 
-type forwardingRulesClient struct {
+type globalForwardingRulesClient struct {
 	*compute.GlobalForwardingRulesClient
 }
 
-func (c *forwardingRulesClient) Close() error {
+func (c *globalForwardingRulesClient) Close() error {
 	return c.GlobalForwardingRulesClient.Close()
 }
 
-func (c *forwardingRulesClient) List(ctx context.Context, req *computepb.ListGlobalForwardingRulesRequest,
+func (c *globalForwardingRulesClient) List(ctx context.Context, req *computepb.ListGlobalForwardingRulesRequest,
 	opts ...gax.CallOption,
 ) forwardingRuleIterator {
 	return c.GlobalForwardingRulesClient.List(ctx, req)
 }
 
+type regionalForwardingRulesClient struct {
+	*compute.ForwardingRulesClient
+}
+
+func (c *regionalForwardingRulesClient) Close() error {
+	return c.ForwardingRulesClient.Close()
+}
+
+func (c *regionalForwardingRulesClient) List(ctx context.Context, req *computepb.ListForwardingRulesRequest,
+	opts ...gax.CallOption,
+) forwardingRuleIterator {
+	return c.ForwardingRulesClient.List(ctx, req)
+}
+
 type instanceClient struct {
 	*compute.InstancesClient
 }