AB#2436: Initial support for create/terminate AWS NitroTPM instances

* Add .DS_Store to .gitignore

* Add AWS to config / supported instance types

* Move AWS terraform skeleton to cli/internal/terraform

* Move currently unused IAM to hack/terraform/aws

* Print supported AWS instance types when AWS dev flag is set

* Block everything aTLS related (e.g. init, verify) until AWS attestation is available

* Create/Terminate AWS dev cluster when dev flag is set

* Restrict Nitro instances to NitroTPM supported specifically

* Pin zone for subnets

This is not great for HA, but for now we need to avoid the two subnets
ending up in different zones, causing the load balancer to not be able
to connect to the targets.

Should be replaced later with a better implementation that just uses
multiple subnets within the same region dynamically
based on # of nodes or similar.

* Add AWS/GCP to Terraform TestLoader unit test

* Add uid tag and create log group

Co-authored-by: Daniel Weiße <dw@edgeless.systems>
Co-authored-by: Malte Poll <mp@edgeless.systems>

cli/internal/terraform/loader_test.go

@@ -22,6 +22,24 @@ func TestLoader(t *testing.T) {
 		provider cloudprovider.Provider
 		fileList []string
 	}{
+		"aws": {
+			provider: cloudprovider.AWS,
+			fileList: []string{
+				"main.tf",
+				"variables.tf",
+				"outputs.tf",
+				"modules",
+			},
+		},
+		"gcp": {
+			provider: cloudprovider.GCP,
+			fileList: []string{
+				"main.tf",
+				"variables.tf",
+				"outputs.tf",
+				"modules",
+			},
+		},
 		"qemu": {
 			provider: cloudprovider.QEMU,
 			fileList: []string{

cli/internal/terraform/terraform/aws/main.tf

@@ -0,0 +1,240 @@
+terraform {
+  required_providers {
+    aws = {
+      source  = "hashicorp/aws"
+      version = "~> 4.0"
+    }
+    random = {
+      source  = "hashicorp/random"
+      version = "3.4.3"
+    }
+  }
+}
+
+# Configure the AWS Provider
+provider "aws" {
+  region = var.region
+}
+
+locals {
+  uid                = random_id.uid.hex
+  name               = "${var.name}-${local.uid}"
+  ports_node_range   = "30000-32767"
+  ports_ssh          = "22"
+  ports_kubernetes   = "6443"
+  ports_bootstrapper = "9000"
+  ports_konnectivity = "8132"
+  ports_verify       = "30081"
+  ports_debugd       = "4000"
+
+  tags = { constellation-uid = local.uid }
+}
+
+resource "random_id" "uid" {
+  byte_length = 4
+}
+
+resource "aws_vpc" "vpc" {
+  cidr_block = "192.168.0.0/16"
+  tags       = merge(local.tags, { Name = "${local.name}-vpc" })
+}
+
+module "public_private_subnet" {
+  source                   = "./modules/public_private_subnet"
+  name                     = local.name
+  vpc_id                   = aws_vpc.vpc.id
+  cidr_vpc_subnet_nodes    = "192.168.178.0/24"
+  cidr_vpc_subnet_internet = "192.168.0.0/24"
+  zone                     = var.zone
+  tags                     = local.tags
+}
+
+resource "aws_eip" "lb" {
+  vpc  = true
+  tags = local.tags
+}
+
+resource "aws_lb" "front_end" {
+  name               = "${local.name}-loadbalancer"
+  internal           = false
+  load_balancer_type = "network"
+  tags               = local.tags
+
+  subnet_mapping {
+    subnet_id     = module.public_private_subnet.public_subnet_id
+    allocation_id = aws_eip.lb.id
+  }
+  enable_cross_zone_load_balancing = true
+}
+
+resource "aws_security_group" "security_group" {
+  name        = local.name
+  vpc_id      = aws_vpc.vpc.id
+  description = "Security group for ${local.name}"
+  tags        = local.tags
+
+  egress {
+    from_port   = 0
+    to_port     = 0
+    protocol    = "-1"
+    cidr_blocks = ["0.0.0.0/0"]
+    description = "Allow all outbound traffic"
+  }
+
+  ingress {
+    from_port   = split("-", local.ports_node_range)[0]
+    to_port     = split("-", local.ports_node_range)[1]
+    protocol    = "tcp"
+    cidr_blocks = ["0.0.0.0/0"]
+    description = "K8s node ports"
+  }
+
+  # TODO: Remove when development is more advanced
+  dynamic "ingress" {
+    for_each = var.debug ? [1] : []
+    content {
+      from_port   = local.ports_ssh
+      to_port     = local.ports_ssh
+      protocol    = "tcp"
+      cidr_blocks = ["0.0.0.0/0"]
+      description = "SSH"
+    }
+  }
+
+  ingress {
+    from_port   = local.ports_bootstrapper
+    to_port     = local.ports_bootstrapper
+    protocol    = "tcp"
+    cidr_blocks = ["0.0.0.0/0"]
+    description = "bootstrapper"
+  }
+
+  ingress {
+    from_port   = local.ports_kubernetes
+    to_port     = local.ports_kubernetes
+    protocol    = "tcp"
+    cidr_blocks = ["0.0.0.0/0"]
+    description = "kubernetes"
+  }
+
+  ingress {
+    from_port   = local.ports_konnectivity
+    to_port     = local.ports_konnectivity
+    protocol    = "tcp"
+    cidr_blocks = ["0.0.0.0/0"]
+    description = "konnectivity"
+  }
+
+  dynamic "ingress" {
+    for_each = var.debug ? [1] : []
+    content {
+      from_port   = local.ports_debugd
+      to_port     = local.ports_debugd
+      protocol    = "tcp"
+      cidr_blocks = ["0.0.0.0/0"]
+      description = "debugd"
+    }
+  }
+}
+
+resource "aws_cloudwatch_log_group" "log_group" {
+  name              = local.name
+  retention_in_days = 30
+  tags              = local.tags
+}
+
+module "load_balancer_target_bootstrapper" {
+  source = "./modules/load_balancer_target"
+  name   = "${local.name}-bootstrapper"
+  vpc_id = aws_vpc.vpc.id
+  lb_arn = aws_lb.front_end.arn
+  port   = local.ports_bootstrapper
+  tags   = local.tags
+}
+
+module "load_balancer_target_kubernetes" {
+  source = "./modules/load_balancer_target"
+  name   = "${local.name}-kubernetes"
+  vpc_id = aws_vpc.vpc.id
+  lb_arn = aws_lb.front_end.arn
+  port   = local.ports_kubernetes
+  tags   = local.tags
+}
+
+module "load_balancer_target_verify" {
+  source = "./modules/load_balancer_target"
+  name   = "${local.name}-verify"
+  vpc_id = aws_vpc.vpc.id
+  lb_arn = aws_lb.front_end.arn
+  port   = local.ports_verify
+  tags   = local.tags
+}
+
+module "load_balancer_target_debugd" {
+  count  = var.debug ? 1 : 0 // only deploy debugd in debug mode
+  source = "./modules/load_balancer_target"
+  name   = "${local.name}-debugd"
+  vpc_id = aws_vpc.vpc.id
+  lb_arn = aws_lb.front_end.arn
+  port   = local.ports_debugd
+  tags   = local.tags
+}
+
+module "load_balancer_target_konnectivity" {
+  source = "./modules/load_balancer_target"
+  name   = "${local.name}-konnectivity"
+  vpc_id = aws_vpc.vpc.id
+  lb_arn = aws_lb.front_end.arn
+  port   = local.ports_konnectivity
+  tags   = local.tags
+}
+
+# TODO: Remove when development is more advanced
+module "load_balancer_target_ssh" {
+  count  = var.debug ? 1 : 0 // only deploy SSH in debug mode
+  source = "./modules/load_balancer_target"
+  name   = "${local.name}-ssh"
+  vpc_id = aws_vpc.vpc.id
+  lb_arn = aws_lb.front_end.arn
+  port   = local.ports_ssh
+  tags   = local.tags
+}
+
+module "instance_group_control_plane" {
+  source          = "./modules/instance_group"
+  name            = local.name
+  role            = "control-plane"
+  uid             = local.uid
+  instance_type   = var.instance_type
+  instance_count  = var.control_plane_count
+  image_id        = var.ami
+  state_disk_type = var.state_disk_type
+  state_disk_size = var.state_disk_size
+  target_group_arns = flatten([
+    module.load_balancer_target_bootstrapper.target_group_arn,
+    module.load_balancer_target_kubernetes.target_group_arn,
+    module.load_balancer_target_verify.target_group_arn,
+    module.load_balancer_target_konnectivity.target_group_arn,
+    var.debug ? [module.load_balancer_target_debugd[0].target_group_arn,
+    module.load_balancer_target_ssh[0].target_group_arn] : [],
+  ])
+  security_groups      = [aws_security_group.security_group.id]
+  subnetwork           = module.public_private_subnet.private_subnet_id
+  iam_instance_profile = var.iam_instance_profile_control_plane
+}
+
+module "instance_group_worker_nodes" {
+  source               = "./modules/instance_group"
+  name                 = local.name
+  role                 = "worker"
+  uid                  = local.uid
+  instance_type        = var.instance_type
+  instance_count       = var.worker_count
+  image_id             = var.ami
+  state_disk_type      = var.state_disk_type
+  state_disk_size      = var.state_disk_size
+  subnetwork           = module.public_private_subnet.private_subnet_id
+  target_group_arns    = []
+  security_groups      = []
+  iam_instance_profile = var.iam_instance_profile_worker_nodes
+}

cli/internal/terraform/terraform/aws/modules/instance_group/main.tf

@@ -0,0 +1,70 @@
+terraform {
+  required_providers {
+    aws = {
+      source  = "hashicorp/aws"
+      version = "~> 4.0"
+    }
+  }
+}
+
+locals {
+  name = "${var.name}-${lower(var.role)}"
+}
+
+
+resource "aws_launch_configuration" "control_plane_launch_config" {
+  name_prefix          = local.name
+  image_id             = var.image_id
+  instance_type        = var.instance_type
+  iam_instance_profile = var.iam_instance_profile
+  security_groups      = var.security_groups
+  metadata_options {
+    http_tokens = "required"
+  }
+
+  root_block_device {
+    encrypted = true
+  }
+
+  ebs_block_device {
+    device_name           = "/dev/sdb" # Note: AWS may adjust this to /dev/xvdb, /dev/hdb or /dev/nvme1n1 depending on the disk type. See: https://docs.aws.amazon.com/en_us/AWSEC2/latest/UserGuide/device_naming.html
+    volume_size           = var.state_disk_size
+    volume_type           = var.state_disk_type
+    encrypted             = true
+    delete_on_termination = true
+  }
+
+  lifecycle {
+    create_before_destroy = true
+  }
+}
+
+resource "aws_autoscaling_group" "control_plane_autoscaling_group" {
+  name                 = local.name
+  launch_configuration = aws_launch_configuration.control_plane_launch_config.name
+  min_size             = 1
+  max_size             = 10
+  desired_capacity     = var.instance_count
+  vpc_zone_identifier  = [var.subnetwork]
+  target_group_arns    = var.target_group_arns
+
+  lifecycle {
+    create_before_destroy = true
+  }
+
+  tag {
+    key                 = "Name"
+    value               = local.name
+    propagate_at_launch = true
+  }
+  tag {
+    key                 = "constellation-role"
+    value               = var.role
+    propagate_at_launch = true
+  }
+  tag {
+    key                 = "constellation-uid"
+    value               = var.uid
+    propagate_at_launch = true
+  }
+}

cli/internal/terraform/terraform/aws/modules/instance_group/variables.tf

@@ -0,0 +1,59 @@
+variable "name" {
+  type        = string
+  description = "Base name of the instance group."
+}
+
+variable "role" {
+  type        = string
+  description = "The role of the instance group. Has to be 'ControlPlane' or 'Worker'."
+}
+
+variable "uid" {
+  type        = string
+  description = "UID of the cluster. This is used for tags."
+}
+
+variable "instance_type" {
+  type        = string
+  description = "Instance type for the nodes."
+}
+
+variable "instance_count" {
+  type        = number
+  description = "Number of instances in the instance group."
+}
+
+variable "image_id" {
+  type        = string
+  description = "Image ID for the nodes."
+}
+
+variable "state_disk_type" {
+  type        = string
+  description = "EBS disk type for the state disk of the nodes."
+}
+
+variable "state_disk_size" {
+  type        = number
+  description = "Disk size for the state disk of the nodes [GB]."
+}
+
+variable "target_group_arns" {
+  type        = list(string)
+  description = "ARN of the target group."
+}
+
+variable "subnetwork" {
+  type        = string
+  description = "Name of the subnetwork to use."
+}
+
+variable "iam_instance_profile" {
+  type        = string
+  description = "IAM instance profile for the nodes."
+}
+
+variable "security_groups" {
+  type        = list(string)
+  description = "List of IDs of the security groups for an instance."
+}

cli/internal/terraform/terraform/aws/modules/load_balancer_target/main.tf

@@ -0,0 +1,37 @@
+terraform {
+  required_providers {
+    aws = {
+      source  = "hashicorp/aws"
+      version = "~> 4.0"
+    }
+  }
+}
+
+resource "aws_lb_target_group" "front_end" {
+  name     = var.name
+  port     = var.port
+  protocol = "TCP"
+  vpc_id   = var.vpc_id
+  tags     = var.tags
+
+  health_check {
+    port     = var.port
+    protocol = "TCP"
+  }
+
+  lifecycle {
+    create_before_destroy = true
+  }
+}
+
+resource "aws_lb_listener" "front_end" {
+  load_balancer_arn = var.lb_arn
+  port              = var.port
+  protocol          = "TCP"
+  tags              = var.tags
+
+  default_action {
+    type             = "forward"
+    target_group_arn = aws_lb_target_group.front_end.arn
+  }
+}

cli/internal/terraform/terraform/aws/modules/load_balancer_target/output.tf

@@ -0,0 +1,3 @@
+output "target_group_arn" {
+  value = aws_lb_target_group.front_end.arn
+}

cli/internal/terraform/terraform/aws/modules/load_balancer_target/variables.tf

@@ -0,0 +1,24 @@
+variable "name" {
+  type        = string
+  description = "Name of the load balancer target."
+}
+
+variable "port" {
+  type        = string
+  description = "Port of the load balancer target."
+}
+
+variable "vpc_id" {
+  type        = string
+  description = "ID of the VPC."
+}
+
+variable "lb_arn" {
+  type        = string
+  description = "ARN of the load balancer."
+}
+
+variable "tags" {
+  type        = map(string)
+  description = "The tags to add to the loadbalancer."
+}

cli/internal/terraform/terraform/aws/modules/public_private_subnet/main.tf

@@ -0,0 +1,68 @@
+terraform {
+  required_providers {
+    aws = {
+      source  = "hashicorp/aws"
+      version = "~> 4.0"
+    }
+  }
+}
+
+resource "aws_eip" "nat" {
+  vpc  = true
+  tags = var.tags
+}
+
+resource "aws_subnet" "private" {
+  vpc_id            = var.vpc_id
+  cidr_block        = var.cidr_vpc_subnet_nodes
+  availability_zone = var.zone
+  tags              = merge(var.tags, { Name = "${var.name}-subnet-nodes" })
+}
+
+resource "aws_subnet" "public" {
+  vpc_id            = var.vpc_id
+  cidr_block        = var.cidr_vpc_subnet_internet
+  availability_zone = var.zone
+  tags              = merge(var.tags, { Name = "${var.name}-subnet-internet" })
+}
+
+resource "aws_internet_gateway" "gw" {
+  vpc_id = var.vpc_id
+  tags   = merge(var.tags, { Name = "${var.name}-internet-gateway" })
+}
+
+resource "aws_nat_gateway" "gw" {
+  subnet_id     = aws_subnet.public.id
+  allocation_id = aws_eip.nat.id
+  tags          = merge(var.tags, { Name = "${var.name}-nat-gateway" })
+}
+
+resource "aws_route_table" "private_nat" {
+  vpc_id = var.vpc_id
+  tags   = merge(var.tags, { Name = "${var.name}-private-nat" })
+
+  route {
+    cidr_block = "0.0.0.0/0"
+    gateway_id = aws_nat_gateway.gw.id
+  }
+}
+
+resource "aws_route_table" "public_igw" {
+  vpc_id = var.vpc_id
+  tags   = merge(var.tags, { Name = "${var.name}-public-igw" })
+
+  route {
+    cidr_block = "0.0.0.0/0"
+    gateway_id = aws_internet_gateway.gw.id
+  }
+}
+
+resource "aws_route_table_association" "private-nat" {
+  subnet_id      = aws_subnet.private.id
+  route_table_id = aws_route_table.private_nat.id
+}
+
+resource "aws_route_table_association" "route_to_internet" {
+  subnet_id      = aws_subnet.public.id
+  route_table_id = aws_route_table.public_igw.id
+}

cli/internal/terraform/terraform/aws/modules/public_private_subnet/output.tf

@@ -0,0 +1,7 @@
+output "private_subnet_id" {
+  value = aws_subnet.private.id
+}
+
+output "public_subnet_id" {
+  value = aws_subnet.public.id
+}

cli/internal/terraform/terraform/aws/modules/public_private_subnet/variables.tf

@@ -0,0 +1,29 @@
+variable "name" {
+  type        = string
+  description = "Name of your Constellation, which is used as a prefix for tags."
+}
+
+variable "vpc_id" {
+  type        = string
+  description = "ID of the VPC."
+}
+
+variable "zone" {
+  type        = string
+  description = "Availability zone."
+}
+
+variable "cidr_vpc_subnet_nodes" {
+  type        = string
+  description = "CIDR block for the subnet that will contain the nodes."
+}
+
+variable "cidr_vpc_subnet_internet" {
+  type        = string
+  description = "CIDR block for the subnet that contains resources reachable from the Internet."
+}
+
+variable "tags" {
+  type        = map(string)
+  description = "The tags to add to the resource."
+}

cli/internal/terraform/terraform/aws/outputs.tf

@@ -0,0 +1,3 @@
+output "ip" {
+  value = aws_eip.lb.public_ip
+}

cli/internal/terraform/terraform/aws/variables.tf

@@ -0,0 +1,59 @@
+variable "name" {
+  type        = string
+  description = "Name of your Constellation"
+}
+
+variable "iam_instance_profile_worker_nodes" {
+  type        = string
+  description = "Name of the IAM instance profile for worker nodes"
+}
+
+variable "iam_instance_profile_control_plane" {
+  type        = string
+  description = "Name of the IAM instance profile for control plane nodes"
+}
+
+variable "instance_type" {
+  type        = string
+  description = "Instance type for worker nodes"
+}
+
+variable "state_disk_type" {
+  type        = string
+  description = "EBS disk type for the state disk of the nodes"
+}
+
+variable "state_disk_size" {
+  type        = number
+  description = "Disk size for the state disk of the nodes [GB]"
+}
+
+variable "control_plane_count" {
+  type        = number
+  description = "Number of control plane nodes"
+}
+
+variable "worker_count" {
+  type        = number
+  description = "Number of worker nodes"
+}
+
+variable "ami" {
+  type        = string
+  description = "AMI ID"
+}
+
+variable "region" {
+  type        = string
+  description = "The AWS region to create the cluster in"
+}
+
+variable "zone" {
+  type        = string
+  description = "The AWS availability zone name to create the cluster in"
+}
+
+variable "debug" {
+  type        = bool
+  description = "Enable debug mode. This opens up a debugd port that can be used to deploy a custom bootstrapper."
+}

cli/internal/terraform/variables.go

@@ -39,6 +39,28 @@ func (v *CommonVariables) String() string {
 	return b.String()
 }
 
+// GCPVariables is user configuration for creating a cluster with Terraform on GCP.
+type AWSVariables struct {
+	// CommonVariables contains common variables.
+	CommonVariables
+	// Region is the AWS region to use.
+	Region string
+	// Zone is the AWS zone to use in the given region.
+	Zone string
+	// AMIImageID is the ID of the AMI image to use.
+	AMIImageID string
+	// InstanceType is the type of the EC2 instance to use.
+	InstanceType string
+	// StateDiskType is the EBS disk type to use for the state disk.
+	StateDiskType string
+	// IAMGroupControlPlane is the IAM group to use for the control-plane nodes.
+	IAMProfileControlPlane string
+	// IAMGroupWorkerNodes is the IAM group to use for the worker nodes.
+	IAMProfileWorkerNodes string
+	// Debug is true if debug mode is enabled.
+	Debug bool
+}
+
 // GCPVariables is user configuration for creating a cluster with Terraform on GCP.
 type GCPVariables struct {
 	// CommonVariables contains common variables.
@@ -62,6 +84,21 @@ type GCPVariables struct {
 	Debug bool
 }
 
+func (v *AWSVariables) String() string {
+	b := &strings.Builder{}
+	b.WriteString(v.CommonVariables.String())
+	writeLinef(b, "region = %q", v.Region)
+	writeLinef(b, "zone = %q", v.Zone)
+	writeLinef(b, "ami = %q", v.AMIImageID)
+	writeLinef(b, "instance_type = %q", v.InstanceType)
+	writeLinef(b, "state_disk_type = %q", v.StateDiskType)
+	writeLinef(b, "iam_instance_profile_control_plane = %q", v.IAMProfileControlPlane)
+	writeLinef(b, "iam_instance_profile_worker_nodes = %q", v.IAMProfileWorkerNodes)
+	writeLinef(b, "debug = %t", v.Debug)
+
+	return b.String()
+}
+
 // String returns a string representation of the variables, formatted as Terraform variables.
 func (v *GCPVariables) String() string {
 	b := &strings.Builder{}