AB#2436: Initial support for create/terminate AWS NitroTPM instances

* Add .DS_Store to .gitignore

* Add AWS to config / supported instance types

* Move AWS terraform skeleton to cli/internal/terraform

* Move currently unused IAM to hack/terraform/aws

* Print supported AWS instance types when AWS dev flag is set

* Block everything aTLS related (e.g. init, verify) until AWS attestation is available

* Create/Terminate AWS dev cluster when dev flag is set

* Restrict Nitro instances to NitroTPM supported specifically

* Pin zone for subnets

This is not great for HA, but for now we need to avoid the two subnets
ending up in different zones, causing the load balancer to not be able
to connect to the targets.

Should be replaced later with a better implementation that just uses
multiple subnets within the same region dynamically
based on # of nodes or similar.

* Add AWS/GCP to Terraform TestLoader unit test

* Add uid tag and create log group

Co-authored-by: Daniel Weiße <dw@edgeless.systems>
Co-authored-by: Malte Poll <mp@edgeless.systems>

cli/internal/terraform/terraform/aws/main.tf

@@ -0,0 +1,240 @@
+terraform {
+  required_providers {
+    aws = {
+      source  = "hashicorp/aws"
+      version = "~> 4.0"
+    }
+    random = {
+      source  = "hashicorp/random"
+      version = "3.4.3"
+    }
+  }
+}
+
+# Configure the AWS Provider
+provider "aws" {
+  region = var.region
+}
+
+locals {
+  uid                = random_id.uid.hex
+  name               = "${var.name}-${local.uid}"
+  ports_node_range   = "30000-32767"
+  ports_ssh          = "22"
+  ports_kubernetes   = "6443"
+  ports_bootstrapper = "9000"
+  ports_konnectivity = "8132"
+  ports_verify       = "30081"
+  ports_debugd       = "4000"
+
+  tags = { constellation-uid = local.uid }
+}
+
+resource "random_id" "uid" {
+  byte_length = 4
+}
+
+resource "aws_vpc" "vpc" {
+  cidr_block = "192.168.0.0/16"
+  tags       = merge(local.tags, { Name = "${local.name}-vpc" })
+}
+
+module "public_private_subnet" {
+  source                   = "./modules/public_private_subnet"
+  name                     = local.name
+  vpc_id                   = aws_vpc.vpc.id
+  cidr_vpc_subnet_nodes    = "192.168.178.0/24"
+  cidr_vpc_subnet_internet = "192.168.0.0/24"
+  zone                     = var.zone
+  tags                     = local.tags
+}
+
+resource "aws_eip" "lb" {
+  vpc  = true
+  tags = local.tags
+}
+
+resource "aws_lb" "front_end" {
+  name               = "${local.name}-loadbalancer"
+  internal           = false
+  load_balancer_type = "network"
+  tags               = local.tags
+
+  subnet_mapping {
+    subnet_id     = module.public_private_subnet.public_subnet_id
+    allocation_id = aws_eip.lb.id
+  }
+  enable_cross_zone_load_balancing = true
+}
+
+resource "aws_security_group" "security_group" {
+  name        = local.name
+  vpc_id      = aws_vpc.vpc.id
+  description = "Security group for ${local.name}"
+  tags        = local.tags
+
+  egress {
+    from_port   = 0
+    to_port     = 0
+    protocol    = "-1"
+    cidr_blocks = ["0.0.0.0/0"]
+    description = "Allow all outbound traffic"
+  }
+
+  ingress {
+    from_port   = split("-", local.ports_node_range)[0]
+    to_port     = split("-", local.ports_node_range)[1]
+    protocol    = "tcp"
+    cidr_blocks = ["0.0.0.0/0"]
+    description = "K8s node ports"
+  }
+
+  # TODO: Remove when development is more advanced
+  dynamic "ingress" {
+    for_each = var.debug ? [1] : []
+    content {
+      from_port   = local.ports_ssh
+      to_port     = local.ports_ssh
+      protocol    = "tcp"
+      cidr_blocks = ["0.0.0.0/0"]
+      description = "SSH"
+    }
+  }
+
+  ingress {
+    from_port   = local.ports_bootstrapper
+    to_port     = local.ports_bootstrapper
+    protocol    = "tcp"
+    cidr_blocks = ["0.0.0.0/0"]
+    description = "bootstrapper"
+  }
+
+  ingress {
+    from_port   = local.ports_kubernetes
+    to_port     = local.ports_kubernetes
+    protocol    = "tcp"
+    cidr_blocks = ["0.0.0.0/0"]
+    description = "kubernetes"
+  }
+
+  ingress {
+    from_port   = local.ports_konnectivity
+    to_port     = local.ports_konnectivity
+    protocol    = "tcp"
+    cidr_blocks = ["0.0.0.0/0"]
+    description = "konnectivity"
+  }
+
+  dynamic "ingress" {
+    for_each = var.debug ? [1] : []
+    content {
+      from_port   = local.ports_debugd
+      to_port     = local.ports_debugd
+      protocol    = "tcp"
+      cidr_blocks = ["0.0.0.0/0"]
+      description = "debugd"
+    }
+  }
+}
+
+resource "aws_cloudwatch_log_group" "log_group" {
+  name              = local.name
+  retention_in_days = 30
+  tags              = local.tags
+}
+
+module "load_balancer_target_bootstrapper" {
+  source = "./modules/load_balancer_target"
+  name   = "${local.name}-bootstrapper"
+  vpc_id = aws_vpc.vpc.id
+  lb_arn = aws_lb.front_end.arn
+  port   = local.ports_bootstrapper
+  tags   = local.tags
+}
+
+module "load_balancer_target_kubernetes" {
+  source = "./modules/load_balancer_target"
+  name   = "${local.name}-kubernetes"
+  vpc_id = aws_vpc.vpc.id
+  lb_arn = aws_lb.front_end.arn
+  port   = local.ports_kubernetes
+  tags   = local.tags
+}
+
+module "load_balancer_target_verify" {
+  source = "./modules/load_balancer_target"
+  name   = "${local.name}-verify"
+  vpc_id = aws_vpc.vpc.id
+  lb_arn = aws_lb.front_end.arn
+  port   = local.ports_verify
+  tags   = local.tags
+}
+
+module "load_balancer_target_debugd" {
+  count  = var.debug ? 1 : 0 // only deploy debugd in debug mode
+  source = "./modules/load_balancer_target"
+  name   = "${local.name}-debugd"
+  vpc_id = aws_vpc.vpc.id
+  lb_arn = aws_lb.front_end.arn
+  port   = local.ports_debugd
+  tags   = local.tags
+}
+
+module "load_balancer_target_konnectivity" {
+  source = "./modules/load_balancer_target"
+  name   = "${local.name}-konnectivity"
+  vpc_id = aws_vpc.vpc.id
+  lb_arn = aws_lb.front_end.arn
+  port   = local.ports_konnectivity
+  tags   = local.tags
+}
+
+# TODO: Remove when development is more advanced
+module "load_balancer_target_ssh" {
+  count  = var.debug ? 1 : 0 // only deploy SSH in debug mode
+  source = "./modules/load_balancer_target"
+  name   = "${local.name}-ssh"
+  vpc_id = aws_vpc.vpc.id
+  lb_arn = aws_lb.front_end.arn
+  port   = local.ports_ssh
+  tags   = local.tags
+}
+
+module "instance_group_control_plane" {
+  source          = "./modules/instance_group"
+  name            = local.name
+  role            = "control-plane"
+  uid             = local.uid
+  instance_type   = var.instance_type
+  instance_count  = var.control_plane_count
+  image_id        = var.ami
+  state_disk_type = var.state_disk_type
+  state_disk_size = var.state_disk_size
+  target_group_arns = flatten([
+    module.load_balancer_target_bootstrapper.target_group_arn,
+    module.load_balancer_target_kubernetes.target_group_arn,
+    module.load_balancer_target_verify.target_group_arn,
+    module.load_balancer_target_konnectivity.target_group_arn,
+    var.debug ? [module.load_balancer_target_debugd[0].target_group_arn,
+    module.load_balancer_target_ssh[0].target_group_arn] : [],
+  ])
+  security_groups      = [aws_security_group.security_group.id]
+  subnetwork           = module.public_private_subnet.private_subnet_id
+  iam_instance_profile = var.iam_instance_profile_control_plane
+}
+
+module "instance_group_worker_nodes" {
+  source               = "./modules/instance_group"
+  name                 = local.name
+  role                 = "worker"
+  uid                  = local.uid
+  instance_type        = var.instance_type
+  instance_count       = var.worker_count
+  image_id             = var.ami
+  state_disk_type      = var.state_disk_type
+  state_disk_size      = var.state_disk_size
+  subnetwork           = module.public_private_subnet.private_subnet_id
+  target_group_arns    = []
+  security_groups      = []
+  iam_instance_profile = var.iam_instance_profile_worker_nodes
+}

cli/internal/terraform/terraform/aws/modules/instance_group/main.tf

@@ -0,0 +1,70 @@
+terraform {
+  required_providers {
+    aws = {
+      source  = "hashicorp/aws"
+      version = "~> 4.0"
+    }
+  }
+}
+
+locals {
+  name = "${var.name}-${lower(var.role)}"
+}
+
+
+resource "aws_launch_configuration" "control_plane_launch_config" {
+  name_prefix          = local.name
+  image_id             = var.image_id
+  instance_type        = var.instance_type
+  iam_instance_profile = var.iam_instance_profile
+  security_groups      = var.security_groups
+  metadata_options {
+    http_tokens = "required"
+  }
+
+  root_block_device {
+    encrypted = true
+  }
+
+  ebs_block_device {
+    device_name           = "/dev/sdb" # Note: AWS may adjust this to /dev/xvdb, /dev/hdb or /dev/nvme1n1 depending on the disk type. See: https://docs.aws.amazon.com/en_us/AWSEC2/latest/UserGuide/device_naming.html
+    volume_size           = var.state_disk_size
+    volume_type           = var.state_disk_type
+    encrypted             = true
+    delete_on_termination = true
+  }
+
+  lifecycle {
+    create_before_destroy = true
+  }
+}
+
+resource "aws_autoscaling_group" "control_plane_autoscaling_group" {
+  name                 = local.name
+  launch_configuration = aws_launch_configuration.control_plane_launch_config.name
+  min_size             = 1
+  max_size             = 10
+  desired_capacity     = var.instance_count
+  vpc_zone_identifier  = [var.subnetwork]
+  target_group_arns    = var.target_group_arns
+
+  lifecycle {
+    create_before_destroy = true
+  }
+
+  tag {
+    key                 = "Name"
+    value               = local.name
+    propagate_at_launch = true
+  }
+  tag {
+    key                 = "constellation-role"
+    value               = var.role
+    propagate_at_launch = true
+  }
+  tag {
+    key                 = "constellation-uid"
+    value               = var.uid
+    propagate_at_launch = true
+  }
+}

cli/internal/terraform/terraform/aws/modules/instance_group/variables.tf

@@ -0,0 +1,59 @@
+variable "name" {
+  type        = string
+  description = "Base name of the instance group."
+}
+
+variable "role" {
+  type        = string
+  description = "The role of the instance group. Has to be 'ControlPlane' or 'Worker'."
+}
+
+variable "uid" {
+  type        = string
+  description = "UID of the cluster. This is used for tags."
+}
+
+variable "instance_type" {
+  type        = string
+  description = "Instance type for the nodes."
+}
+
+variable "instance_count" {
+  type        = number
+  description = "Number of instances in the instance group."
+}
+
+variable "image_id" {
+  type        = string
+  description = "Image ID for the nodes."
+}
+
+variable "state_disk_type" {
+  type        = string
+  description = "EBS disk type for the state disk of the nodes."
+}
+
+variable "state_disk_size" {
+  type        = number
+  description = "Disk size for the state disk of the nodes [GB]."
+}
+
+variable "target_group_arns" {
+  type        = list(string)
+  description = "ARN of the target group."
+}
+
+variable "subnetwork" {
+  type        = string
+  description = "Name of the subnetwork to use."
+}
+
+variable "iam_instance_profile" {
+  type        = string
+  description = "IAM instance profile for the nodes."
+}
+
+variable "security_groups" {
+  type        = list(string)
+  description = "List of IDs of the security groups for an instance."
+}

cli/internal/terraform/terraform/aws/modules/load_balancer_target/main.tf

@@ -0,0 +1,37 @@
+terraform {
+  required_providers {
+    aws = {
+      source  = "hashicorp/aws"
+      version = "~> 4.0"
+    }
+  }
+}
+
+resource "aws_lb_target_group" "front_end" {
+  name     = var.name
+  port     = var.port
+  protocol = "TCP"
+  vpc_id   = var.vpc_id
+  tags     = var.tags
+
+  health_check {
+    port     = var.port
+    protocol = "TCP"
+  }
+
+  lifecycle {
+    create_before_destroy = true
+  }
+}
+
+resource "aws_lb_listener" "front_end" {
+  load_balancer_arn = var.lb_arn
+  port              = var.port
+  protocol          = "TCP"
+  tags              = var.tags
+
+  default_action {
+    type             = "forward"
+    target_group_arn = aws_lb_target_group.front_end.arn
+  }
+}

cli/internal/terraform/terraform/aws/modules/load_balancer_target/output.tf

@@ -0,0 +1,3 @@
+output "target_group_arn" {
+  value = aws_lb_target_group.front_end.arn
+}

cli/internal/terraform/terraform/aws/modules/load_balancer_target/variables.tf

@@ -0,0 +1,24 @@
+variable "name" {
+  type        = string
+  description = "Name of the load balancer target."
+}
+
+variable "port" {
+  type        = string
+  description = "Port of the load balancer target."
+}
+
+variable "vpc_id" {
+  type        = string
+  description = "ID of the VPC."
+}
+
+variable "lb_arn" {
+  type        = string
+  description = "ARN of the load balancer."
+}
+
+variable "tags" {
+  type        = map(string)
+  description = "The tags to add to the loadbalancer."
+}

cli/internal/terraform/terraform/aws/modules/public_private_subnet/main.tf

@@ -0,0 +1,68 @@
+terraform {
+  required_providers {
+    aws = {
+      source  = "hashicorp/aws"
+      version = "~> 4.0"
+    }
+  }
+}
+
+resource "aws_eip" "nat" {
+  vpc  = true
+  tags = var.tags
+}
+
+resource "aws_subnet" "private" {
+  vpc_id            = var.vpc_id
+  cidr_block        = var.cidr_vpc_subnet_nodes
+  availability_zone = var.zone
+  tags              = merge(var.tags, { Name = "${var.name}-subnet-nodes" })
+}
+
+resource "aws_subnet" "public" {
+  vpc_id            = var.vpc_id
+  cidr_block        = var.cidr_vpc_subnet_internet
+  availability_zone = var.zone
+  tags              = merge(var.tags, { Name = "${var.name}-subnet-internet" })
+}
+
+resource "aws_internet_gateway" "gw" {
+  vpc_id = var.vpc_id
+  tags   = merge(var.tags, { Name = "${var.name}-internet-gateway" })
+}
+
+resource "aws_nat_gateway" "gw" {
+  subnet_id     = aws_subnet.public.id
+  allocation_id = aws_eip.nat.id
+  tags          = merge(var.tags, { Name = "${var.name}-nat-gateway" })
+}
+
+resource "aws_route_table" "private_nat" {
+  vpc_id = var.vpc_id
+  tags   = merge(var.tags, { Name = "${var.name}-private-nat" })
+
+  route {
+    cidr_block = "0.0.0.0/0"
+    gateway_id = aws_nat_gateway.gw.id
+  }
+}
+
+resource "aws_route_table" "public_igw" {
+  vpc_id = var.vpc_id
+  tags   = merge(var.tags, { Name = "${var.name}-public-igw" })
+
+  route {
+    cidr_block = "0.0.0.0/0"
+    gateway_id = aws_internet_gateway.gw.id
+  }
+}
+
+resource "aws_route_table_association" "private-nat" {
+  subnet_id      = aws_subnet.private.id
+  route_table_id = aws_route_table.private_nat.id
+}
+
+resource "aws_route_table_association" "route_to_internet" {
+  subnet_id      = aws_subnet.public.id
+  route_table_id = aws_route_table.public_igw.id
+}

cli/internal/terraform/terraform/aws/modules/public_private_subnet/output.tf

@@ -0,0 +1,7 @@
+output "private_subnet_id" {
+  value = aws_subnet.private.id
+}
+
+output "public_subnet_id" {
+  value = aws_subnet.public.id
+}

cli/internal/terraform/terraform/aws/modules/public_private_subnet/variables.tf

@@ -0,0 +1,29 @@
+variable "name" {
+  type        = string
+  description = "Name of your Constellation, which is used as a prefix for tags."
+}
+
+variable "vpc_id" {
+  type        = string
+  description = "ID of the VPC."
+}
+
+variable "zone" {
+  type        = string
+  description = "Availability zone."
+}
+
+variable "cidr_vpc_subnet_nodes" {
+  type        = string
+  description = "CIDR block for the subnet that will contain the nodes."
+}
+
+variable "cidr_vpc_subnet_internet" {
+  type        = string
+  description = "CIDR block for the subnet that contains resources reachable from the Internet."
+}
+
+variable "tags" {
+  type        = map(string)
+  description = "The tags to add to the resource."
+}

cli/internal/terraform/terraform/aws/outputs.tf

@@ -0,0 +1,3 @@
+output "ip" {
+  value = aws_eip.lb.public_ip
+}

cli/internal/terraform/terraform/aws/variables.tf

@@ -0,0 +1,59 @@
+variable "name" {
+  type        = string
+  description = "Name of your Constellation"
+}
+
+variable "iam_instance_profile_worker_nodes" {
+  type        = string
+  description = "Name of the IAM instance profile for worker nodes"
+}
+
+variable "iam_instance_profile_control_plane" {
+  type        = string
+  description = "Name of the IAM instance profile for control plane nodes"
+}
+
+variable "instance_type" {
+  type        = string
+  description = "Instance type for worker nodes"
+}
+
+variable "state_disk_type" {
+  type        = string
+  description = "EBS disk type for the state disk of the nodes"
+}
+
+variable "state_disk_size" {
+  type        = number
+  description = "Disk size for the state disk of the nodes [GB]"
+}
+
+variable "control_plane_count" {
+  type        = number
+  description = "Number of control plane nodes"
+}
+
+variable "worker_count" {
+  type        = number
+  description = "Number of worker nodes"
+}
+
+variable "ami" {
+  type        = string
+  description = "AMI ID"
+}
+
+variable "region" {
+  type        = string
+  description = "The AWS region to create the cluster in"
+}
+
+variable "zone" {
+  type        = string
+  description = "The AWS availability zone name to create the cluster in"
+}
+
+variable "debug" {
+  type        = bool
+  description = "Enable debug mode. This opens up a debugd port that can be used to deploy a custom bootstrapper."
+}