Create Terraform module for GCP

terraform/gcp/main.tf

@@ -0,0 +1,210 @@
+terraform {
+  required_providers {
+    google = {
+      source  = "hashicorp/google"
+      version = "4.34.0"
+    }
+    random = {
+      source  = "hashicorp/random"
+      version = "3.4.1"
+    }
+  }
+}
+
+provider "google" {
+  credentials = file(var.credentials_file)
+
+  project = var.project
+  region  = var.region
+  zone    = var.zone
+}
+
+locals {
+  uid                   = random_id.uid.hex
+  name                  = "${var.name}-${local.uid}"
+  tag                   = "constellation-${local.uid}"
+  ports_node_range      = "30000-32767"
+  ports_kubernetes      = "6443"
+  ports_bootstrapper    = "9000"
+  ports_konnectivity    = "8132"
+  ports_verify          = "30081"
+  ports_recovery        = "9999"
+  ports_debugd          = "4000"
+  cidr_vpc_subnet_nodes = "192.168.178.0/24"
+  cidr_vpc_subnet_pods  = "10.10.0.0/16"
+  kube_env              = "AUTOSCALER_ENV_VARS: kube_reserved=cpu=1060m,memory=1019Mi,ephemeral-storage=41Gi;node_labels=;os=linux;os_distribution=cos;evictionHard="
+}
+
+resource "random_id" "uid" {
+  byte_length = 4
+}
+
+resource "google_compute_network" "vpc_network" {
+  name                    = local.name
+  description             = "Constellation VPC network"
+  auto_create_subnetworks = false
+}
+
+resource "google_compute_subnetwork" "vpc_subnetwork" {
+  name          = local.name
+  description   = "Constellation VPC subnetwork"
+  network       = google_compute_network.vpc_network.id
+  ip_cidr_range = local.cidr_vpc_subnet_nodes
+  secondary_ip_range = [
+    {
+      range_name    = local.name,
+      ip_cidr_range = local.cidr_vpc_subnet_pods,
+    }
+  ]
+}
+
+resource "google_compute_firewall" "firewall_external" {
+  name          = local.name
+  description   = "Constellation VPC firewall"
+  network       = google_compute_network.vpc_network.id
+  source_ranges = ["0.0.0.0/0"]
+  direction     = "INGRESS"
+
+  allow {
+    protocol = "tcp"
+    ports = flatten([
+      local.ports_node_range,
+      local.ports_bootstrapper,
+      local.ports_kubernetes,
+      local.ports_konnectivity,
+      local.ports_recovery,
+      var.debug ? [local.ports_debugd] : [],
+    ])
+  }
+
+}
+
+resource "google_compute_firewall" "firewall_internal_nodes" {
+  name          = "${local.name}-nodes"
+  description   = "Constellation VPC firewall"
+  network       = google_compute_network.vpc_network.id
+  source_ranges = [local.cidr_vpc_subnet_nodes]
+  direction     = "INGRESS"
+
+  allow { protocol = "tcp" }
+  allow { protocol = "udp" }
+  allow { protocol = "icmp" }
+}
+
+resource "google_compute_firewall" "firewall_internal_pods" {
+  name          = "${local.name}-pods"
+  description   = "Constellation VPC firewall"
+  network       = google_compute_network.vpc_network.id
+  source_ranges = [local.cidr_vpc_subnet_pods]
+  direction     = "INGRESS"
+
+  allow { protocol = "tcp" }
+  allow { protocol = "udp" }
+  allow { protocol = "icmp" }
+}
+
+module "instance_group_control_plane" {
+  source         = "./modules/instance_group"
+  name           = local.name
+  role           = "ControlPlane"
+  uid            = local.uid
+  instance_type  = var.instance_type
+  instance_count = var.control_plane_count
+  image_id       = var.image_id
+  disk_size      = var.state_disk_size
+  disk_type      = var.state_disk_type
+  network        = google_compute_network.vpc_network.id
+  subnetwork     = google_compute_subnetwork.vpc_subnetwork.id
+  kube_env       = local.kube_env
+  named_ports = flatten([
+    { name = "kubernetes", port = local.ports_kubernetes },
+    { name = "bootstrapper", port = local.ports_bootstrapper },
+    { name = "verify", port = local.ports_verify },
+    { name = "konnectivity", port = local.ports_konnectivity },
+    { name = "recovery", port = local.ports_recovery },
+    var.debug ? [{ name = "debugd", port = local.ports_debugd }] : [],
+  ])
+}
+
+module "instance_group_worker" {
+  source         = "./modules/instance_group"
+  name           = local.name
+  role           = "Worker"
+  uid            = local.uid
+  instance_type  = var.instance_type
+  instance_count = var.worker_count
+  image_id       = var.image_id
+  disk_size      = var.state_disk_size
+  disk_type      = var.state_disk_type
+  network        = google_compute_network.vpc_network.id
+  subnetwork     = google_compute_subnetwork.vpc_subnetwork.id
+  kube_env       = local.kube_env
+}
+
+resource "google_compute_global_address" "loadbalancer_ip" {
+  name = local.name
+}
+
+module "loadbalancer_kube" {
+  source                 = "./modules/loadbalancer"
+  name                   = local.name
+  health_check           = "HTTPS"
+  backend_port_name      = "kubernetes"
+  backend_instance_group = module.instance_group_control_plane.instance_group
+  ip_address             = google_compute_global_address.loadbalancer_ip.self_link
+  port                   = local.ports_kubernetes
+  frontend_labels = {
+    constellation-uid = local.uid
+  }
+}
+
+module "loadbalancer_boot" {
+  source                 = "./modules/loadbalancer"
+  name                   = local.name
+  health_check           = "TCP"
+  backend_port_name      = "bootstrapper"
+  backend_instance_group = module.instance_group_control_plane.instance_group
+  ip_address             = google_compute_global_address.loadbalancer_ip.self_link
+  port                   = local.ports_bootstrapper
+}
+
+module "loadbalancer_verify" {
+  source                 = "./modules/loadbalancer"
+  name                   = local.name
+  health_check           = "TCP"
+  backend_port_name      = "verify"
+  backend_instance_group = module.instance_group_control_plane.instance_group
+  ip_address             = google_compute_global_address.loadbalancer_ip.self_link
+  port                   = local.ports_verify
+}
+
+module "loadbalancer_konnectivity" {
+  source                 = "./modules/loadbalancer"
+  name                   = local.name
+  health_check           = "TCP"
+  backend_port_name      = "konnectivity"
+  backend_instance_group = module.instance_group_control_plane.instance_group
+  ip_address             = google_compute_global_address.loadbalancer_ip.self_link
+  port                   = local.ports_konnectivity
+}
+
+module "loadbalancer_recovery" {
+  source                 = "./modules/loadbalancer"
+  name                   = local.name
+  health_check           = "TCP"
+  backend_port_name      = "recovery"
+  backend_instance_group = module.instance_group_control_plane.instance_group
+  ip_address             = google_compute_global_address.loadbalancer_ip.self_link
+  port                   = local.ports_recovery
+}
+
+module "loadbalancer_debugd" {
+  count                  = var.debug ? 1 : 0 // only deploy debugd in debug mode
+  source                 = "./modules/loadbalancer"
+  name                   = local.name
+  health_check           = "TCP"
+  backend_port_name      = "debugd"
+  backend_instance_group = module.instance_group_control_plane.instance_group
+  ip_address             = google_compute_global_address.loadbalancer_ip.self_link
+  port                   = local.ports_debugd
+}

terraform/gcp/modules/instance_group/main.tf

@@ -0,0 +1,97 @@
+terraform {
+  required_providers {
+    google = {
+      source  = "hashicorp/google"
+      version = "4.34.0"
+    }
+  }
+}
+
+locals {
+  role_dashed = var.role == "ControlPlane" ? "control-plane" : "worker"
+  name        = "${var.name}-${local.role_dashed}"
+}
+
+resource "google_compute_instance_template" "template" {
+  name         = local.name
+  machine_type = var.instance_type
+  tags         = ["constellation-${var.uid}"]
+
+  confidential_instance_config {
+    enable_confidential_compute = true
+  }
+
+  disk {
+    disk_size_gb = 10
+    source_image = var.image_id
+    auto_delete  = true
+    boot         = true
+    mode         = "READ_WRITE"
+  }
+
+  disk {
+    disk_size_gb = var.disk_size
+    disk_type    = var.disk_type
+    auto_delete  = true
+    device_name  = "state-disk" // This name is used by disk mapper to find the disk
+    boot         = false
+    mode         = "READ_WRITE"
+    type         = "PERSISTENT"
+  }
+
+  metadata = {
+    kube-env           = var.kube_env
+    constellation-uid  = var.uid
+    constellation-role = var.role
+  }
+
+  network_interface {
+    network    = var.network
+    subnetwork = var.subnetwork
+    access_config {}
+    alias_ip_range {
+      ip_cidr_range         = "/24"
+      subnetwork_range_name = var.name
+    }
+  }
+
+  scheduling {
+    on_host_maintenance = "TERMINATE"
+  }
+
+  service_account {
+    scopes = [
+      "https://www.googleapis.com/auth/compute",
+      "https://www.googleapis.com/auth/servicecontrol",
+      "https://www.googleapis.com/auth/service.management.readonly",
+      "https://www.googleapis.com/auth/devstorage.read_only",
+      "https://www.googleapis.com/auth/logging.write",
+      "https://www.googleapis.com/auth/monitoring.write",
+      "https://www.googleapis.com/auth/trace.append",
+    ]
+  }
+
+  shielded_instance_config {
+    enable_secure_boot          = true
+    enable_vtpm                 = true
+    enable_integrity_monitoring = true
+  }
+}
+
+resource "google_compute_instance_group_manager" "instance_group_manager" {
+  name               = local.name
+  base_instance_name = local.name
+  target_size        = var.instance_count
+
+  version {
+    instance_template = google_compute_instance_template.template.id
+  }
+
+  dynamic "named_port" {
+    for_each = toset(var.named_ports)
+    content {
+      name = named_port.value.name
+      port = named_port.value.port
+    }
+  }
+}

terraform/gcp/modules/instance_group/outputs.tf

@@ -0,0 +1,3 @@
+output "instance_group" {
+  value = google_compute_instance_group_manager.instance_group_manager.instance_group
+}

terraform/gcp/modules/instance_group/variables.tf

@@ -0,0 +1,60 @@
+variable "name" {
+  type        = string
+  description = "Base name of the instance group."
+}
+
+variable "role" {
+  type        = string
+  description = "The role of the instance group. Has to be 'ControlPlane' or 'Worker'."
+}
+
+variable "uid" {
+  type        = string
+  description = "UID of the cluster. This is used for tags."
+}
+
+variable "instance_type" {
+  type        = string
+  description = "Instance type for the nodes."
+}
+
+variable "instance_count" {
+  type        = number
+  description = "Number of instances in the instance group."
+}
+
+variable "image_id" {
+  type        = string
+  description = "Image ID for the nodes."
+}
+
+variable "disk_size" {
+  type        = number
+  description = "Disk size for the nodes, in GB."
+}
+
+variable "disk_type" {
+  type        = string
+  description = "Disk type for the nodes. Has to be 'pd-standard' or 'pd-ssd'."
+}
+
+variable "network" {
+  type        = string
+  description = "Name of the network to use."
+}
+
+variable "subnetwork" {
+  type        = string
+  description = "Name of the subnetwork to use."
+}
+
+variable "kube_env" {
+  type        = string
+  description = "Kubernetes env."
+}
+
+variable "named_ports" {
+  type        = list(object({ name = string, port = number }))
+  default     = []
+  description = "Named ports for the instance group."
+}

terraform/gcp/modules/loadbalancer/main.tf

@@ -0,0 +1,62 @@
+terraform {
+  required_providers {
+    google = {
+      source  = "hashicorp/google"
+      version = "4.34.0"
+    }
+  }
+}
+
+locals {
+  name = "${var.name}-${var.backend_port_name}"
+}
+
+resource "google_compute_health_check" "health" {
+  name               = local.name
+  check_interval_sec = 1
+  timeout_sec        = 1
+
+  dynamic "tcp_health_check" {
+    for_each = var.health_check == "TCP" ? [1] : []
+    content {
+      port = var.port
+    }
+  }
+
+  dynamic "https_health_check" {
+    for_each = var.health_check == "HTTPS" ? [1] : []
+    content {
+      host         = ""
+      port         = var.port
+      request_path = "/readyz"
+    }
+  }
+}
+
+resource "google_compute_backend_service" "backend" {
+  name                  = local.name
+  protocol              = "TCP"
+  load_balancing_scheme = "EXTERNAL"
+  health_checks         = [google_compute_health_check.health.self_link]
+  port_name             = var.backend_port_name
+
+  backend {
+    group          = var.backend_instance_group
+    balancing_mode = "UTILIZATION"
+  }
+}
+
+resource "google_compute_target_tcp_proxy" "proxy" {
+  name            = local.name
+  backend_service = google_compute_backend_service.backend.self_link
+}
+
+resource "google_compute_global_forwarding_rule" "forwarding" {
+  name                  = local.name
+  ip_address            = var.ip_address
+  ip_protocol           = "TCP"
+  load_balancing_scheme = "EXTERNAL"
+  port_range            = var.port
+  target                = google_compute_target_tcp_proxy.proxy.self_link
+  labels                = var.frontend_labels
+}

terraform/gcp/modules/loadbalancer/variables.tf

@@ -0,0 +1,35 @@
+variable "name" {
+  type        = string
+  description = "Base name of the load balancer."
+}
+
+variable "health_check" {
+  type        = string
+  description = "The type of the health check. 'HTTPS' or 'TCP'."
+}
+
+variable "backend_port_name" {
+  type        = string
+  description = "Name of backend port. The same name should appear in the instance groups referenced by this service."
+}
+
+variable "backend_instance_group" {
+  type        = string
+  description = "The URL of the instance group resource from which the load balancer will direct traffic."
+}
+
+variable "ip_address" {
+  type        = string
+  description = "The IP address that this forwarding rule serves. An address can be specified either by a literal IP address or a reference to an existing Address resource."
+}
+
+variable "port" {
+  type        = number
+  description = "The port on which to listen for incoming traffic."
+}
+
+variable "frontend_labels" {
+  type        = map(string)
+  default     = {}
+  description = "Labels to apply to the forwarding rule."
+}

terraform/gcp/variables.tf

@@ -0,0 +1,63 @@
+variable "name" {
+  type        = string
+  default     = "constell"
+  description = "Base name of the cluster."
+}
+
+variable "project" {
+  type        = string
+  description = "The GCP project to deploy the cluster in."
+}
+
+variable "region" {
+  type        = string
+  description = "The GCP region to deploy the cluster in."
+}
+
+variable "zone" {
+  type        = string
+  description = "The GCP zone to deploy the cluster in."
+}
+
+variable "credentials_file" {
+  type        = string
+  description = "The path to the GCP credentials file."
+}
+
+variable "control_plane_count" {
+  type        = number
+  description = "The number of control plane nodes to deploy."
+}
+
+variable "worker_count" {
+  type        = number
+  description = "The number of worker nodes to deploy."
+}
+
+variable "instance_type" {
+  type        = string
+  description = "The GCP instance type to deploy."
+}
+
+variable "state_disk_size" {
+  type        = number
+  default     = 30
+  description = "The size of the state disk in GB."
+}
+
+variable "state_disk_type" {
+  type        = string
+  default     = "pd-ssd"
+  description = "The type of the state disk."
+}
+
+variable "image_id" {
+  type        = string
+  description = "The GCP image to use for the cluster nodes."
+}
+
+variable "debug" {
+  type        = bool
+  default     = false
+  description = "Enable debug mode. This opens up a debugd port that can be used to deploy a custom bootstrapper."
+}