diff --git a/terraform/gcp/main.tf b/terraform/gcp/main.tf new file mode 100644 index 000000000..a5c0581d5 --- /dev/null +++ b/terraform/gcp/main.tf @@ -0,0 +1,210 @@ +terraform { + required_providers { + google = { + source = "hashicorp/google" + version = "4.34.0" + } + random = { + source = "hashicorp/random" + version = "3.4.1" + } + } +} + +provider "google" { + credentials = file(var.credentials_file) + + project = var.project + region = var.region + zone = var.zone +} + +locals { + uid = random_id.uid.hex + name = "${var.name}-${local.uid}" + tag = "constellation-${local.uid}" + ports_node_range = "30000-32767" + ports_kubernetes = "6443" + ports_bootstrapper = "9000" + ports_konnectivity = "8132" + ports_verify = "30081" + ports_recovery = "9999" + ports_debugd = "4000" + cidr_vpc_subnet_nodes = "192.168.178.0/24" + cidr_vpc_subnet_pods = "10.10.0.0/16" + kube_env = "AUTOSCALER_ENV_VARS: kube_reserved=cpu=1060m,memory=1019Mi,ephemeral-storage=41Gi;node_labels=;os=linux;os_distribution=cos;evictionHard=" +} + +resource "random_id" "uid" { + byte_length = 4 +} + +resource "google_compute_network" "vpc_network" { + name = local.name + description = "Constellation VPC network" + auto_create_subnetworks = false +} + +resource "google_compute_subnetwork" "vpc_subnetwork" { + name = local.name + description = "Constellation VPC subnetwork" + network = google_compute_network.vpc_network.id + ip_cidr_range = local.cidr_vpc_subnet_nodes + secondary_ip_range = [ + { + range_name = local.name, + ip_cidr_range = local.cidr_vpc_subnet_pods, + } + ] +} + +resource "google_compute_firewall" "firewall_external" { + name = local.name + description = "Constellation VPC firewall" + network = google_compute_network.vpc_network.id + source_ranges = ["0.0.0.0/0"] + direction = "INGRESS" + + allow { + protocol = "tcp" + ports = flatten([ + local.ports_node_range, + local.ports_bootstrapper, + local.ports_kubernetes, + local.ports_konnectivity, + local.ports_recovery, + var.debug ? [local.ports_debugd] : [], + ]) + } + +} + +resource "google_compute_firewall" "firewall_internal_nodes" { + name = "${local.name}-nodes" + description = "Constellation VPC firewall" + network = google_compute_network.vpc_network.id + source_ranges = [local.cidr_vpc_subnet_nodes] + direction = "INGRESS" + + allow { protocol = "tcp" } + allow { protocol = "udp" } + allow { protocol = "icmp" } +} + +resource "google_compute_firewall" "firewall_internal_pods" { + name = "${local.name}-pods" + description = "Constellation VPC firewall" + network = google_compute_network.vpc_network.id + source_ranges = [local.cidr_vpc_subnet_pods] + direction = "INGRESS" + + allow { protocol = "tcp" } + allow { protocol = "udp" } + allow { protocol = "icmp" } +} + +module "instance_group_control_plane" { + source = "./modules/instance_group" + name = local.name + role = "ControlPlane" + uid = local.uid + instance_type = var.instance_type + instance_count = var.control_plane_count + image_id = var.image_id + disk_size = var.state_disk_size + disk_type = var.state_disk_type + network = google_compute_network.vpc_network.id + subnetwork = google_compute_subnetwork.vpc_subnetwork.id + kube_env = local.kube_env + named_ports = flatten([ + { name = "kubernetes", port = local.ports_kubernetes }, + { name = "bootstrapper", port = local.ports_bootstrapper }, + { name = "verify", port = local.ports_verify }, + { name = "konnectivity", port = local.ports_konnectivity }, + { name = "recovery", port = local.ports_recovery }, + var.debug ? [{ name = "debugd", port = local.ports_debugd }] : [], + ]) +} + +module "instance_group_worker" { + source = "./modules/instance_group" + name = local.name + role = "Worker" + uid = local.uid + instance_type = var.instance_type + instance_count = var.worker_count + image_id = var.image_id + disk_size = var.state_disk_size + disk_type = var.state_disk_type + network = google_compute_network.vpc_network.id + subnetwork = google_compute_subnetwork.vpc_subnetwork.id + kube_env = local.kube_env +} + +resource "google_compute_global_address" "loadbalancer_ip" { + name = local.name +} + +module "loadbalancer_kube" { + source = "./modules/loadbalancer" + name = local.name + health_check = "HTTPS" + backend_port_name = "kubernetes" + backend_instance_group = module.instance_group_control_plane.instance_group + ip_address = google_compute_global_address.loadbalancer_ip.self_link + port = local.ports_kubernetes + frontend_labels = { + constellation-uid = local.uid + } +} + +module "loadbalancer_boot" { + source = "./modules/loadbalancer" + name = local.name + health_check = "TCP" + backend_port_name = "bootstrapper" + backend_instance_group = module.instance_group_control_plane.instance_group + ip_address = google_compute_global_address.loadbalancer_ip.self_link + port = local.ports_bootstrapper +} + +module "loadbalancer_verify" { + source = "./modules/loadbalancer" + name = local.name + health_check = "TCP" + backend_port_name = "verify" + backend_instance_group = module.instance_group_control_plane.instance_group + ip_address = google_compute_global_address.loadbalancer_ip.self_link + port = local.ports_verify +} + +module "loadbalancer_konnectivity" { + source = "./modules/loadbalancer" + name = local.name + health_check = "TCP" + backend_port_name = "konnectivity" + backend_instance_group = module.instance_group_control_plane.instance_group + ip_address = google_compute_global_address.loadbalancer_ip.self_link + port = local.ports_konnectivity +} + +module "loadbalancer_recovery" { + source = "./modules/loadbalancer" + name = local.name + health_check = "TCP" + backend_port_name = "recovery" + backend_instance_group = module.instance_group_control_plane.instance_group + ip_address = google_compute_global_address.loadbalancer_ip.self_link + port = local.ports_recovery +} + +module "loadbalancer_debugd" { + count = var.debug ? 1 : 0 // only deploy debugd in debug mode + source = "./modules/loadbalancer" + name = local.name + health_check = "TCP" + backend_port_name = "debugd" + backend_instance_group = module.instance_group_control_plane.instance_group + ip_address = google_compute_global_address.loadbalancer_ip.self_link + port = local.ports_debugd +} diff --git a/terraform/gcp/modules/instance_group/main.tf b/terraform/gcp/modules/instance_group/main.tf new file mode 100644 index 000000000..1a44414c8 --- /dev/null +++ b/terraform/gcp/modules/instance_group/main.tf @@ -0,0 +1,97 @@ +terraform { + required_providers { + google = { + source = "hashicorp/google" + version = "4.34.0" + } + } +} + +locals { + role_dashed = var.role == "ControlPlane" ? "control-plane" : "worker" + name = "${var.name}-${local.role_dashed}" +} + +resource "google_compute_instance_template" "template" { + name = local.name + machine_type = var.instance_type + tags = ["constellation-${var.uid}"] + + confidential_instance_config { + enable_confidential_compute = true + } + + disk { + disk_size_gb = 10 + source_image = var.image_id + auto_delete = true + boot = true + mode = "READ_WRITE" + } + + disk { + disk_size_gb = var.disk_size + disk_type = var.disk_type + auto_delete = true + device_name = "state-disk" // This name is used by disk mapper to find the disk + boot = false + mode = "READ_WRITE" + type = "PERSISTENT" + } + + metadata = { + kube-env = var.kube_env + constellation-uid = var.uid + constellation-role = var.role + } + + network_interface { + network = var.network + subnetwork = var.subnetwork + access_config {} + alias_ip_range { + ip_cidr_range = "/24" + subnetwork_range_name = var.name + } + } + + scheduling { + on_host_maintenance = "TERMINATE" + } + + service_account { + scopes = [ + "https://www.googleapis.com/auth/compute", + "https://www.googleapis.com/auth/servicecontrol", + "https://www.googleapis.com/auth/service.management.readonly", + "https://www.googleapis.com/auth/devstorage.read_only", + "https://www.googleapis.com/auth/logging.write", + "https://www.googleapis.com/auth/monitoring.write", + "https://www.googleapis.com/auth/trace.append", + ] + } + + shielded_instance_config { + enable_secure_boot = true + enable_vtpm = true + enable_integrity_monitoring = true + } +} + +resource "google_compute_instance_group_manager" "instance_group_manager" { + name = local.name + base_instance_name = local.name + target_size = var.instance_count + + version { + instance_template = google_compute_instance_template.template.id + } + + dynamic "named_port" { + for_each = toset(var.named_ports) + content { + name = named_port.value.name + port = named_port.value.port + } + } +} diff --git a/terraform/gcp/modules/instance_group/outputs.tf b/terraform/gcp/modules/instance_group/outputs.tf new file mode 100644 index 000000000..66f793fb0 --- /dev/null +++ b/terraform/gcp/modules/instance_group/outputs.tf @@ -0,0 +1,3 @@ +output "instance_group" { + value = google_compute_instance_group_manager.instance_group_manager.instance_group +} diff --git a/terraform/gcp/modules/instance_group/variables.tf b/terraform/gcp/modules/instance_group/variables.tf new file mode 100644 index 000000000..b091d0b47 --- /dev/null +++ b/terraform/gcp/modules/instance_group/variables.tf @@ -0,0 +1,60 @@ +variable "name" { + type = string + description = "Base name of the instance group." +} + +variable "role" { + type = string + description = "The role of the instance group. Has to be 'ControlPlane' or 'Worker'." +} + +variable "uid" { + type = string + description = "UID of the cluster. This is used for tags." +} + +variable "instance_type" { + type = string + description = "Instance type for the nodes." +} + +variable "instance_count" { + type = number + description = "Number of instances in the instance group." +} + +variable "image_id" { + type = string + description = "Image ID for the nodes." +} + +variable "disk_size" { + type = number + description = "Disk size for the nodes, in GB." +} + +variable "disk_type" { + type = string + description = "Disk type for the nodes. Has to be 'pd-standard' or 'pd-ssd'." +} + +variable "network" { + type = string + description = "Name of the network to use." +} + +variable "subnetwork" { + type = string + description = "Name of the subnetwork to use." +} + +variable "kube_env" { + type = string + description = "Kubernetes env." +} + +variable "named_ports" { + type = list(object({ name = string, port = number })) + default = [] + description = "Named ports for the instance group." +} diff --git a/terraform/gcp/modules/loadbalancer/main.tf b/terraform/gcp/modules/loadbalancer/main.tf new file mode 100644 index 000000000..38a1e5d5b --- /dev/null +++ b/terraform/gcp/modules/loadbalancer/main.tf @@ -0,0 +1,62 @@ +terraform { + required_providers { + google = { + source = "hashicorp/google" + version = "4.34.0" + } + } +} + +locals { + name = "${var.name}-${var.backend_port_name}" +} + +resource "google_compute_health_check" "health" { + name = local.name + check_interval_sec = 1 + timeout_sec = 1 + + dynamic "tcp_health_check" { + for_each = var.health_check == "TCP" ? [1] : [] + content { + port = var.port + } + } + + dynamic "https_health_check" { + for_each = var.health_check == "HTTPS" ? [1] : [] + content { + host = "" + port = var.port + request_path = "/readyz" + } + } +} + +resource "google_compute_backend_service" "backend" { + name = local.name + protocol = "TCP" + load_balancing_scheme = "EXTERNAL" + health_checks = [google_compute_health_check.health.self_link] + port_name = var.backend_port_name + + backend { + group = var.backend_instance_group + balancing_mode = "UTILIZATION" + } +} + +resource "google_compute_target_tcp_proxy" "proxy" { + name = local.name + backend_service = google_compute_backend_service.backend.self_link +} + +resource "google_compute_global_forwarding_rule" "forwarding" { + name = local.name + ip_address = var.ip_address + ip_protocol = "TCP" + load_balancing_scheme = "EXTERNAL" + port_range = var.port + target = google_compute_target_tcp_proxy.proxy.self_link + labels = var.frontend_labels +} diff --git a/terraform/gcp/modules/loadbalancer/variables.tf b/terraform/gcp/modules/loadbalancer/variables.tf new file mode 100644 index 000000000..18c290148 --- /dev/null +++ b/terraform/gcp/modules/loadbalancer/variables.tf @@ -0,0 +1,35 @@ +variable "name" { + type = string + description = "Base name of the load balancer." +} + +variable "health_check" { + type = string + description = "The type of the health check. 'HTTPS' or 'TCP'." +} + +variable "backend_port_name" { + type = string + description = "Name of backend port. The same name should appear in the instance groups referenced by this service." +} + +variable "backend_instance_group" { + type = string + description = "The URL of the instance group resource from which the load balancer will direct traffic." +} + +variable "ip_address" { + type = string + description = "The IP address that this forwarding rule serves. An address can be specified either by a literal IP address or a reference to an existing Address resource." +} + +variable "port" { + type = number + description = "The port on which to listen for incoming traffic." +} + +variable "frontend_labels" { + type = map(string) + default = {} + description = "Labels to apply to the forwarding rule." +} diff --git a/terraform/gcp/variables.tf b/terraform/gcp/variables.tf new file mode 100644 index 000000000..b78723a22 --- /dev/null +++ b/terraform/gcp/variables.tf @@ -0,0 +1,63 @@ +variable "name" { + type = string + default = "constell" + description = "Base name of the cluster." +} + +variable "project" { + type = string + description = "The GCP project to deploy the cluster in." +} + +variable "region" { + type = string + description = "The GCP region to deploy the cluster in." +} + +variable "zone" { + type = string + description = "The GCP zone to deploy the cluster in." +} + +variable "credentials_file" { + type = string + description = "The path to the GCP credentials file." +} + +variable "control_plane_count" { + type = number + description = "The number of control plane nodes to deploy." +} + +variable "worker_count" { + type = number + description = "The number of worker nodes to deploy." +} + +variable "instance_type" { + type = string + description = "The GCP instance type to deploy." +} + +variable "state_disk_size" { + type = number + default = 30 + description = "The size of the state disk in GB." +} + +variable "state_disk_type" { + type = string + default = "pd-ssd" + description = "The type of the state disk." +} + +variable "image_id" { + type = string + description = "The GCP image to use for the cluster nodes." +} + +variable "debug" { + type = bool + default = false + description = "Enable debug mode. This opens up a debugd port that can be used to deploy a custom bootstrapper." +}