terraform: Terraform module for AWS (#2503)

terraform/infrastructure/aws/modules/instance_group/main.tf

@@ -0,0 +1,115 @@
+terraform {
+  required_providers {
+    aws = {
+      source  = "hashicorp/aws"
+      version = "5.17.0"
+    }
+    random = {
+      source  = "hashicorp/random"
+      version = "3.5.1"
+    }
+  }
+}
+
+locals {
+  group_uid = random_id.uid.hex
+  name      = "${var.base_name}-${lower(var.role)}-${local.group_uid}"
+}
+
+resource "random_id" "uid" {
+  byte_length = 4
+}
+
+resource "aws_launch_template" "launch_template" {
+  name_prefix   = local.name
+  image_id      = var.image_id
+  instance_type = var.instance_type
+  iam_instance_profile {
+    name = var.iam_instance_profile
+  }
+  vpc_security_group_ids = var.security_groups
+  metadata_options {
+    http_endpoint               = "enabled"
+    http_tokens                 = "required"
+    instance_metadata_tags      = "disabled"
+    http_put_response_hop_limit = 2
+  }
+
+  block_device_mappings {
+    device_name = "/dev/sdb"
+    ebs {
+      volume_size           = var.state_disk_size
+      volume_type           = var.state_disk_type
+      encrypted             = true
+      delete_on_termination = true
+    }
+  }
+
+  # See: https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/launch_template#cpu-options
+  cpu_options {
+    # use "enabled" to enable SEV-SNP
+    # use "disabled" to disable SEV-SNP (but still require SNP-capable hardware)
+    # use null to leave the setting unset (allows non-SNP-capable hardware to be used)
+    amd_sev_snp = var.enable_snp ? "enabled" : null
+    # Disable SMT. We are already disabling it inside the image.
+    # Disabling SMT only in the image, not in the Hypervisor creates problems.
+    # Thus, also disable it in the Hypervisor.
+    # TODO(derpsteb): reenable once AWS confirms it's safe to do so.
+    # threads_per_core = 1
+    # When setting threads_per_core we also have to set core_count.
+    # For the currently supported SNP instance families (C6a, M6a, R6a) default_cores
+    # equals the maximum number of available cores.
+    # core_count = data.aws_ec2_instance_type.instance_data.default_cores
+  }
+
+  lifecycle {
+    create_before_destroy = true
+    ignore_changes = [
+      cpu_options,     # required. we cannot change the CPU options of a launch template
+      name_prefix,     # required. Allow legacy scale sets to keep their old names
+      default_version, # required. update procedure creates new versions of the launch template
+      image_id,        # required. update procedure modifies the image id externally
+    ]
+  }
+}
+
+resource "aws_autoscaling_group" "autoscaling_group" {
+  name = local.name
+  launch_template {
+    id = aws_launch_template.launch_template.id
+  }
+  min_size            = 1
+  max_size            = 10
+  desired_capacity    = var.initial_count
+  vpc_zone_identifier = [var.subnetwork]
+  target_group_arns   = var.target_group_arns
+
+  # TODO(msanft): Remove this (to have the 10m default) once AWS SEV-SNP boot problems are resolved.
+  # Set a higher timeout for the ASG to fulfill the desired healthy capcity. Temporary workaround to
+  # long boot times on SEV-SNP machines on AWS.
+  wait_for_capacity_timeout = var.enable_snp ? "20m" : "10m"
+
+  dynamic "tag" {
+    for_each = var.tags
+    content {
+      key                 = tag.key
+      value               = tag.value
+      propagate_at_launch = true
+    }
+  }
+
+  lifecycle {
+    create_before_destroy = true
+    ignore_changes = [
+      name,                      # required. Allow legacy scale sets to keep their old names
+      launch_template.0.version, # required. update procedure creates new versions of the launch template
+      min_size,                  # required. autoscaling modifies the instance count externally
+      max_size,                  # required. autoscaling modifies the instance count externally
+      desired_capacity,          # required. autoscaling modifies the instance count externally
+    ]
+  }
+}
+
+data "aws_ec2_instance_type" "instance_data" {
+  instance_type = var.instance_type
+}

terraform/infrastructure/aws/modules/instance_group/variables.tf

@@ -0,0 +1,84 @@
+variable "base_name" {
+  type        = string
+  description = "Base name of the instance group."
+}
+
+variable "node_group_name" {
+  type        = string
+  description = "Constellation name for the node group (used for configuration and CSP-independent naming)."
+}
+
+variable "role" {
+  type        = string
+  description = "The role of the instance group."
+  validation {
+    condition     = contains(["control-plane", "worker"], var.role)
+    error_message = "The role has to be 'control-plane' or 'worker'."
+  }
+}
+
+variable "uid" {
+  type        = string
+  description = "UID of the cluster. This is used for tags."
+}
+
+variable "instance_type" {
+  type        = string
+  description = "Instance type for the nodes."
+}
+
+variable "initial_count" {
+  type        = number
+  description = "Number of instances in the instance group."
+}
+
+variable "image_id" {
+  type        = string
+  description = "Image ID for the nodes."
+}
+
+variable "state_disk_type" {
+  type        = string
+  description = "EBS disk type for the state disk of the nodes."
+}
+
+variable "state_disk_size" {
+  type        = number
+  description = "Disk size for the state disk of the nodes [GB]."
+}
+
+variable "target_group_arns" {
+  type        = list(string)
+  description = "ARN of the target group."
+}
+
+variable "subnetwork" {
+  type        = string
+  description = "Name of the subnetwork to use."
+}
+
+variable "iam_instance_profile" {
+  type        = string
+  description = "IAM instance profile for the nodes."
+}
+
+variable "security_groups" {
+  type        = list(string)
+  description = "List of IDs of the security groups for an instance."
+}
+
+variable "tags" {
+  type        = map(string)
+  description = "The tags to add to the instance group."
+}
+
+variable "enable_snp" {
+  type        = bool
+  default     = true
+  description = "Enable AMD SEV SNP. Setting this to true sets the cpu-option AmdSevSnp to enable."
+}
+
+variable "zone" {
+  type        = string
+  description = "Zone to deploy the instance group in."
+}

terraform/infrastructure/aws/modules/jump_host/main.tf

@@ -0,0 +1,59 @@
+terraform {
+  required_providers {
+    aws = {
+      source  = "hashicorp/aws"
+      version = "5.17.0"
+    }
+  }
+}
+
+
+data "aws_ami" "ubuntu" {
+  most_recent = true
+  owners      = ["099720109477"] # Canonical
+
+  filter {
+    name   = "name"
+    values = ["ubuntu/images/hvm-ssd/ubuntu-jammy-22.04-amd64-server-*"]
+  }
+}
+
+resource "aws_instance" "jump_host" {
+  ami                         = data.aws_ami.ubuntu.id
+  instance_type               = "c5a.large"
+  associate_public_ip_address = true
+
+  iam_instance_profile = var.iam_instance_profile
+  subnet_id            = var.subnet_id
+  security_groups      = [var.security_group_id]
+
+  tags = {
+    "Name" = "${var.base_name}-jump-host"
+  }
+
+  user_data = <<EOF
+#!/bin/bash
+set -x
+
+# Uncomment to create user with password
+# useradd -m user
+# usermod -aG sudo user
+# usermod --shell /bin/bash user
+# sh -c "echo \"user:pass\" | chpasswd"
+
+sysctl -w net.ipv4.ip_forward=1
+sysctl -p
+
+internal_ip=$(ip route get 8.8.8.8 | grep -oP 'src \K[^ ]+')
+
+lb_ip=${var.lb_internal_ip}
+if [[ ! $${lb_ip} =~ ^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
+  lb_ip=$(dig +short ${var.lb_internal_ip})
+fi
+%{for port in var.ports~}
+iptables -t nat -A PREROUTING -p tcp --dport ${port} -j DNAT --to-destination $${lb_ip}:${port}
+iptables -t nat -A POSTROUTING -p tcp -d $${lb_ip} --dport ${port} -j SNAT --to-source $${internal_ip}
+%{endfor~}
+EOF
+
+}

terraform/infrastructure/aws/modules/jump_host/output.tf

@@ -0,0 +1,3 @@
+output "ip" {
+  value = aws_instance.jump_host.public_ip
+}

terraform/infrastructure/aws/modules/jump_host/variables.tf

@@ -0,0 +1,29 @@
+variable "base_name" {
+  description = "Base name of the jump host"
+  type        = string
+}
+
+variable "subnet_id" {
+  description = "Subnet ID to deploy the jump host into"
+  type        = string
+}
+
+variable "lb_internal_ip" {
+  description = "Internal IP of the load balancer"
+  type        = string
+}
+
+variable "iam_instance_profile" {
+  description = "IAM instance profile to attach to the jump host"
+  type        = string
+}
+
+variable "ports" {
+  description = "Ports to forward to the load balancer"
+  type        = list(number)
+}
+
+variable "security_group_id" {
+  description = "Security group to attach to the jump host"
+  type        = string
+}

terraform/infrastructure/aws/modules/load_balancer_target/main.tf

@@ -0,0 +1,42 @@
+terraform {
+  required_providers {
+    aws = {
+      source  = "hashicorp/aws"
+      version = "5.17.0"
+    }
+  }
+}
+
+resource "aws_lb_target_group" "front_end" {
+  name               = var.name
+  port               = var.port
+  protocol           = "TCP"
+  vpc_id             = var.vpc_id
+  tags               = var.tags
+  preserve_client_ip = "false"
+
+  health_check {
+    port                = var.port
+    protocol            = var.healthcheck_protocol
+    path                = var.healthcheck_protocol == "HTTPS" ? var.healthcheck_path : null
+    interval            = 10
+    healthy_threshold   = 2
+    unhealthy_threshold = 2
+  }
+
+  lifecycle {
+    create_before_destroy = true
+  }
+}
+
+resource "aws_lb_listener" "front_end" {
+  load_balancer_arn = var.lb_arn
+  port              = var.port
+  protocol          = "TCP"
+  tags              = var.tags
+
+  default_action {
+    type             = "forward"
+    target_group_arn = aws_lb_target_group.front_end.arn
+  }
+}

terraform/infrastructure/aws/modules/load_balancer_target/output.tf

@@ -0,0 +1,3 @@
+output "target_group_arn" {
+  value = aws_lb_target_group.front_end.arn
+}

terraform/infrastructure/aws/modules/load_balancer_target/variables.tf

@@ -0,0 +1,36 @@
+variable "name" {
+  type        = string
+  description = "Name of the load balancer target."
+}
+
+variable "port" {
+  type        = string
+  description = "Port of the load balancer target."
+}
+
+variable "vpc_id" {
+  type        = string
+  description = "ID of the VPC."
+}
+
+variable "lb_arn" {
+  type        = string
+  description = "ARN of the load balancer."
+}
+
+variable "healthcheck_protocol" {
+  type        = string
+  default     = "TCP"
+  description = "Type of the load balancer target."
+}
+
+variable "healthcheck_path" {
+  type        = string
+  default     = ""
+  description = "Path for health check."
+}
+
+variable "tags" {
+  type        = map(string)
+  description = "The tags to add to the loadbalancer."
+}

terraform/infrastructure/aws/modules/public_private_subnet/main.tf

@@ -0,0 +1,120 @@
+terraform {
+  required_providers {
+    aws = {
+      source  = "hashicorp/aws"
+      version = "5.17.0"
+    }
+  }
+}
+
+locals {
+  # az_number is a stable mapping of az suffix to a number used for calculating the subnet cidr
+  az_number = {
+    # we start counting at 2 to have the legacy subnet before the first newly created networks
+    # the legacy subnet did not start at a /20 boundary
+    # 0 => 192.168.176.0/24 (unused private subnet cidr)
+    # 1 => 192.168.177.0/24 (unused private subnet cidr)
+    legacy = 2 # => 192.168.178.0/24 (legacy private subnet)
+    a      = 3 # => 192.168.179.0/24 (first newly created zonal private subnet)
+    b      = 4
+    c      = 5
+    d      = 6
+    e      = 7
+    f      = 8
+    g      = 9
+    h      = 10
+    i      = 11
+    j      = 12
+    k      = 13
+    l      = 14
+    m      = 15 # => 192.168.191.0/24 (last reserved zonal private subnet cidr). In reality, AWS doesn't have that many zones in a region.
+  }
+}
+
+data "aws_availability_zones" "available" {
+  state = "available"
+}
+
+data "aws_availability_zone" "all" {
+  for_each = toset(data.aws_availability_zones.available.names)
+
+  name = each.key
+}
+
+resource "aws_eip" "nat" {
+  for_each = toset(var.zones)
+  domain   = "vpc"
+  tags     = var.tags
+}
+
+resource "aws_subnet" "private" {
+  for_each          = data.aws_availability_zone.all
+  vpc_id            = var.vpc_id
+  cidr_block        = cidrsubnet(var.cidr_vpc_subnet_nodes, 4, local.az_number[each.value.name_suffix])
+  availability_zone = each.key
+  tags              = merge(var.tags, { Name = "${var.name}-subnet-nodes" }, { "kubernetes.io/role/internal-elb" = 1 }) # aws-load-balancer-controller needs role annotation
+  lifecycle {
+    ignore_changes = [
+      cidr_block, # required. Legacy subnets used fixed cidr blocks for the single zone that don't match the new scheme.
+    ]
+  }
+}
+
+resource "aws_subnet" "public" {
+  for_each          = data.aws_availability_zone.all
+  vpc_id            = var.vpc_id
+  cidr_block        = cidrsubnet(var.cidr_vpc_subnet_internet, 4, local.az_number[each.value.name_suffix])
+  availability_zone = each.key
+  tags              = merge(var.tags, { Name = "${var.name}-subnet-internet" }, { "kubernetes.io/role/elb" = 1 }) # aws-load-balancer-controller needs role annotation
+  lifecycle {
+    ignore_changes = [
+      cidr_block, # required. Legacy subnets used fixed cidr blocks for the single zone that don't match the new scheme.
+    ]
+  }
+}
+
+resource "aws_internet_gateway" "gw" {
+  vpc_id = var.vpc_id
+  tags   = merge(var.tags, { Name = "${var.name}-internet-gateway" })
+}
+
+resource "aws_nat_gateway" "gw" {
+  for_each      = toset(var.zones)
+  subnet_id     = aws_subnet.public[each.key].id
+  allocation_id = aws_eip.nat[each.key].id
+  tags          = merge(var.tags, { Name = "${var.name}-nat-gateway" })
+}
+
+resource "aws_route_table" "private_nat" {
+  for_each = toset(var.zones)
+  vpc_id   = var.vpc_id
+  tags     = merge(var.tags, { Name = "${var.name}-private-nat" })
+
+  route {
+    cidr_block     = "0.0.0.0/0"
+    nat_gateway_id = aws_nat_gateway.gw[each.key].id
+  }
+}
+
+resource "aws_route_table" "public_igw" {
+  for_each = toset(var.zones)
+  vpc_id   = var.vpc_id
+  tags     = merge(var.tags, { Name = "${var.name}-public-igw" })
+
+  route {
+    cidr_block = "0.0.0.0/0"
+    gateway_id = aws_internet_gateway.gw.id
+  }
+}
+
+resource "aws_route_table_association" "private_nat" {
+  for_each       = toset(var.zones)
+  subnet_id      = aws_subnet.private[each.key].id
+  route_table_id = aws_route_table.private_nat[each.key].id
+}
+
+resource "aws_route_table_association" "route_to_internet" {
+  for_each       = toset(var.zones)
+  subnet_id      = aws_subnet.public[each.key].id
+  route_table_id = aws_route_table.public_igw[each.key].id
+}

terraform/infrastructure/aws/modules/public_private_subnet/output.tf

@@ -0,0 +1,19 @@
+output "private_subnet_id" {
+  value = {
+    for az in data.aws_availability_zone.all :
+    az.name => aws_subnet.private[az.name].id
+  }
+}
+
+output "public_subnet_id" {
+  value = {
+    for az in data.aws_availability_zone.all :
+    az.name => aws_subnet.public[az.name].id
+  }
+}
+
+# all_zones is a list of all availability zones in the region
+# it also contains zones that are not currently used by node groups (but might be in the future)
+output "all_zones" {
+  value = distinct(sort([for az in data.aws_availability_zone.all : az.name]))
+}

terraform/infrastructure/aws/modules/public_private_subnet/variables.tf

@@ -0,0 +1,34 @@
+variable "name" {
+  type        = string
+  description = "Name of your Constellation, which is used as a prefix for tags."
+}
+
+variable "vpc_id" {
+  type        = string
+  description = "ID of the VPC."
+}
+
+variable "zone" {
+  type        = string
+  description = "Main availability zone. Only used for legacy reasons."
+}
+
+variable "zones" {
+  type        = list(string)
+  description = "Availability zones."
+}
+
+variable "cidr_vpc_subnet_nodes" {
+  type        = string
+  description = "CIDR block for the subnet that will contain the nodes."
+}
+
+variable "cidr_vpc_subnet_internet" {
+  type        = string
+  description = "CIDR block for the subnet that contains resources reachable from the Internet."
+}
+
+variable "tags" {
+  type        = map(string)
+  description = "The tags to add to the resource."
+}