Create Terraform module for Azure

Co-authored-by: Benedict Schlueter <bs@edgeless.systems>

cli/internal/terraform/terraform/azure/main.tf

@@ -0,0 +1,195 @@
+terraform {
+  required_providers {
+    azurerm = {
+      source  = "hashicorp/azurerm"
+      version = "3.23.0"
+    }
+    random = {
+      source  = "hashicorp/random"
+      version = "3.4.1"
+    }
+  }
+}
+
+provider "azurerm" {
+  features {}
+}
+
+locals {
+  uid                   = random_id.uid.hex
+  name                  = "${var.name}-${local.uid}"
+  tags                  = { constellation-uid = local.uid }
+  ports_node_range      = "30000-32767"
+  ports_kubernetes      = "6443"
+  ports_bootstrapper    = "9000"
+  ports_konnectivity    = "8132"
+  ports_verify          = "30081"
+  ports_recovery        = "9999"
+  ports_debugd          = "4000"
+  cidr_vpc_subnet_nodes = "192.168.178.0/24"
+  cidr_vpc_subnet_pods  = "10.10.0.0/16"
+}
+
+resource "random_id" "uid" {
+  byte_length = 4
+}
+
+resource "azurerm_application_insights" "insights" {
+  name                = local.name
+  location            = var.location
+  resource_group_name = var.resource_group
+  application_type    = "other"
+  tags                = local.tags
+}
+
+resource "azurerm_public_ip" "loadbalancer_ip" {
+  name                = local.name
+  resource_group_name = var.resource_group
+  location            = var.location
+  allocation_method   = "Static"
+  sku                 = "Standard"
+  tags                = local.tags
+}
+
+resource "azurerm_lb" "loadbalancer" {
+  name                = local.name
+  location            = var.location
+  resource_group_name = var.resource_group
+  sku                 = "Standard"
+  tags                = local.tags
+
+  frontend_ip_configuration {
+    name                 = "PublicIPAddress"
+    public_ip_address_id = azurerm_public_ip.loadbalancer_ip.id
+  }
+}
+
+module "loadbalancer_backend_control_plane" {
+  source = "./modules/load_balancer_backend"
+
+  name            = "${local.name}-control-plane"
+  loadbalancer_id = azurerm_lb.loadbalancer.id
+  ports = flatten([
+    { name = "bootstrapper", port = local.ports_bootstrapper },
+    { name = "kubernetes", port = local.ports_kubernetes },
+    { name = "konnectivity", port = local.ports_konnectivity },
+    { name = "verify", port = local.ports_verify },
+    { name = "recovery", port = local.ports_recovery },
+    var.debug ? [{ name = "debugd", port = local.ports_debugd }] : [],
+  ])
+}
+
+module "loadbalancer_backend_worker" {
+  source = "./modules/load_balancer_backend"
+
+  name            = "${local.name}-worker"
+  loadbalancer_id = azurerm_lb.loadbalancer.id
+  ports           = []
+}
+
+resource "azurerm_lb_backend_address_pool" "all" {
+  loadbalancer_id = azurerm_lb.loadbalancer.id
+  name            = "${var.name}-all"
+}
+
+resource "azurerm_lb_outbound_rule" "outbound" {
+  name                    = "${var.name}-outbound"
+  loadbalancer_id         = azurerm_lb.loadbalancer.id
+  protocol                = "All"
+  backend_address_pool_id = azurerm_lb_backend_address_pool.all.id
+
+  frontend_ip_configuration { name = "PublicIPAddress" }
+}
+
+resource "azurerm_virtual_network" "network" {
+  name                = local.name
+  resource_group_name = var.resource_group
+  location            = var.location
+  address_space       = ["10.0.0.0/8"]
+  tags                = local.tags
+}
+
+resource "azurerm_subnet" "node_subnet" {
+  name                 = "${local.name}-node"
+  resource_group_name  = var.resource_group
+  virtual_network_name = azurerm_virtual_network.network.name
+  address_prefixes     = ["10.9.0.0/16"]
+}
+
+resource "azurerm_subnet" "pod_subnet" {
+  name                 = "${local.name}-pod"
+  resource_group_name  = var.resource_group
+  virtual_network_name = azurerm_virtual_network.network.name
+  address_prefixes     = ["10.10.0.0/16"]
+}
+
+resource "azurerm_network_security_group" "security_group" {
+  name                = local.name
+  location            = var.location
+  resource_group_name = var.resource_group
+  tags                = local.tags
+
+  dynamic "security_rule" {
+    for_each = flatten([
+      { name = "noderange", priority = 100, dest_port_range = local.ports_node_range },
+      { name = "kubernetes", priority = 101, dest_port_range = local.ports_kubernetes },
+      { name = "bootstrapper", priority = 102, dest_port_range = local.ports_bootstrapper },
+      { name = "konnectivity", priority = 103, dest_port_range = local.ports_konnectivity },
+      { name = "recovery", priority = 104, dest_port_range = local.ports_recovery },
+      var.debug ? [{ name = "debugd", priority = 105, dest_port_range = local.ports_debugd }] : [],
+    ])
+    content {
+      name                       = security_rule.value.name
+      priority                   = security_rule.value.priority
+      direction                  = "Inbound"
+      access                     = "Allow"
+      protocol                   = "Tcp"
+      source_port_range          = "*"
+      destination_port_range     = security_rule.value.dest_port_range
+      source_address_prefix      = "*"
+      destination_address_prefix = "*"
+    }
+  }
+}
+
+module "scale_set_control_plane" {
+  source = "./modules/scale_set"
+
+  name                      = "${local.name}-control-plane"
+  instance_count            = var.control_plane_count
+  state_disk_size           = var.state_disk_size
+  state_disk_type           = var.state_disk_type
+  resource_group            = var.resource_group
+  location                  = var.location
+  instance_type             = var.instance_type
+  tags                      = merge(local.tags, { role = "control-plane" })
+  image_id                  = var.image_id
+  user_assigned_identity    = var.user_assigned_identity
+  network_security_group_id = azurerm_network_security_group.security_group.id
+  subnet_id                 = azurerm_subnet.node_subnet.id
+  backend_address_pool_ids = [
+    azurerm_lb_backend_address_pool.all.id,
+    module.loadbalancer_backend_control_plane.backendpool_id
+  ]
+}
+
+module "scale_set_worker" {
+  source = "./modules/scale_set"
+
+  name                      = "${local.name}-worker"
+  instance_count            = var.control_plane_count
+  state_disk_size           = var.state_disk_size
+  state_disk_type           = var.state_disk_type
+  resource_group            = var.resource_group
+  location                  = var.location
+  instance_type             = var.instance_type
+  tags                      = merge(local.tags, { role = "worker" })
+  image_id                  = var.image_id
+  user_assigned_identity    = var.user_assigned_identity
+  network_security_group_id = azurerm_network_security_group.security_group.id
+  subnet_id                 = azurerm_subnet.node_subnet.id
+  backend_address_pool_ids = [
+    azurerm_lb_backend_address_pool.all.id,
+    module.loadbalancer_backend_worker.backendpool_id,
+  ]
+}

cli/internal/terraform/terraform/azure/modules/load_balancer_backend/main.tf

@@ -0,0 +1,37 @@
+terraform {
+  required_providers {
+    azurerm = {
+      source  = "hashicorp/azurerm"
+      version = "3.23.0"
+    }
+  }
+}
+
+resource "azurerm_lb_backend_address_pool" "backend_pool" {
+  loadbalancer_id = var.loadbalancer_id
+  name            = var.name
+}
+
+resource "azurerm_lb_probe" "health_probes" {
+  for_each = { for port in var.ports : port.name => port }
+
+  loadbalancer_id     = var.loadbalancer_id
+  name                = each.value.name
+  port                = each.value.port
+  protocol            = "Tcp"
+  interval_in_seconds = 5
+}
+
+resource "azurerm_lb_rule" "rules" {
+  for_each = azurerm_lb_probe.health_probes
+
+  loadbalancer_id                = var.loadbalancer_id
+  name                           = each.value.name
+  protocol                       = each.value.protocol
+  frontend_port                  = each.value.port
+  backend_port                   = each.value.port
+  frontend_ip_configuration_name = "PublicIPAddress"
+  backend_address_pool_ids       = [azurerm_lb_backend_address_pool.backend_pool.id]
+  probe_id                       = each.value.id
+  disable_outbound_snat          = true
+}

cli/internal/terraform/terraform/azure/modules/load_balancer_backend/outputs.tf

@@ -0,0 +1,4 @@
+output "backendpool_id" {
+  value       = azurerm_lb_backend_address_pool.backend_pool.id
+  description = "The ID of the created backend pool."
+}

cli/internal/terraform/terraform/azure/modules/load_balancer_backend/variables.tf

@@ -0,0 +1,18 @@
+variable "name" {
+  type        = string
+  default     = "constell"
+  description = "Base name of the cluster."
+}
+
+variable "loadbalancer_id" {
+  type        = string
+  description = "The ID of the load balancer to add the backend to."
+}
+
+variable "ports" {
+  type = list(object({
+    name = string
+    port = number
+  }))
+  description = "The ports to add to the backend."
+}

cli/internal/terraform/terraform/azure/modules/scale_set/main.tf

@@ -0,0 +1,76 @@
+terraform {
+  required_providers {
+    azurerm = {
+      source  = "hashicorp/azurerm"
+      version = "3.23.0"
+    }
+    random = {
+      source  = "hashicorp/random"
+      version = "3.4.1"
+    }
+  }
+}
+
+resource "random_password" "password" {
+  length = 16
+}
+
+resource "azurerm_linux_virtual_machine_scale_set" "scale_set" {
+  name                            = var.name
+  resource_group_name             = var.resource_group
+  location                        = var.location
+  sku                             = var.instance_type
+  instances                       = var.instance_count
+  admin_username                  = "adminuser"
+  admin_password                  = random_password.password.result
+  overprovision                   = false
+  vtpm_enabled                    = true
+  disable_password_authentication = false
+  upgrade_mode                    = "Manual"
+  secure_boot_enabled             = true
+  source_image_id                 = var.image_id
+  tags                            = var.tags
+
+  identity {
+    type         = "UserAssigned"
+    identity_ids = [var.user_assigned_identity]
+  }
+
+  boot_diagnostics {}
+
+  dynamic "os_disk" {
+    for_each = var.confidential_vm ? [1] : [] # if confidential_vm is true
+    content {
+      security_encryption_type = "VMGuestStateOnly"
+      caching                  = "ReadWrite"
+      storage_account_type     = "Premium_LRS"
+    }
+  }
+  dynamic "os_disk" {
+    for_each = var.confidential_vm ? [] : [1] # else
+    content {
+      caching              = "ReadWrite"
+      storage_account_type = "Premium_LRS"
+    }
+  }
+
+  data_disk {
+    storage_account_type = var.state_disk_type
+    disk_size_gb         = var.state_disk_size
+    caching              = "ReadWrite"
+    lun                  = 0
+  }
+
+  network_interface {
+    name                      = "node-network"
+    primary                   = true
+    network_security_group_id = var.network_security_group_id
+
+    ip_configuration {
+      name                                   = "node-network"
+      primary                                = true
+      subnet_id                              = var.subnet_id
+      load_balancer_backend_address_pool_ids = var.backend_address_pool_ids
+    }
+  }
+}

cli/internal/terraform/terraform/azure/modules/scale_set/variables.tf

@@ -0,0 +1,73 @@
+variable "name" {
+  type        = string
+  default     = "constell"
+  description = "Base name of the cluster."
+}
+
+variable "instance_count" {
+  type        = number
+  description = "The number of instances in this scale set."
+}
+
+variable "instance_type" {
+  type        = string
+  description = "The Azure instance type to deploy."
+}
+
+variable "state_disk_size" {
+  type        = number
+  default     = 30
+  description = "The size of the state disk in GB."
+}
+
+variable "resource_group" {
+  type        = string
+  description = "The name of the Azure resource group to create the Constellation cluster in."
+}
+
+variable "location" {
+  type        = string
+  description = "The Azure location to deploy the cluster in."
+}
+
+variable "image_id" {
+  type        = string
+  description = "The image to use for the cluster nodes."
+}
+
+variable "user_assigned_identity" {
+  type        = string
+  description = "The name of the user assigned identity to attache to the nodes of the cluster."
+}
+
+variable "state_disk_type" {
+  type        = string
+  default     = "Premium_LRS"
+  description = "The type of the state disk."
+}
+
+variable "network_security_group_id" {
+  type        = string
+  description = "The ID of the network security group to use for the scale set."
+}
+
+variable "backend_address_pool_ids" {
+  type        = list(string)
+  description = "The IDs of the backend address pools to use for the scale set."
+}
+
+variable "subnet_id" {
+  type        = string
+  description = "The ID of the subnet to use for the scale set."
+}
+
+variable "tags" {
+  type        = map(string)
+  description = "The tags to add to the scale set."
+}
+
+variable "confidential_vm" {
+  type        = bool
+  default     = true
+  description = "Whether to deploy the cluster nodes as confidential VMs."
+}

cli/internal/terraform/terraform/azure/outputs.tf

@@ -0,0 +1,3 @@
+output "ip" {
+  value = azurerm_public_ip.loadbalancer_ip.ip_address
+}

cli/internal/terraform/terraform/azure/variables.tf

@@ -0,0 +1,64 @@
+variable "name" {
+  type        = string
+  default     = "constell"
+  description = "Base name of the cluster."
+}
+
+variable "control_plane_count" {
+  type        = number
+  description = "The number of control plane nodes to deploy."
+}
+
+variable "worker_count" {
+  type        = number
+  description = "The number of worker nodes to deploy."
+}
+
+variable "state_disk_size" {
+  type        = number
+  default     = 30
+  description = "The size of the state disk in GB."
+}
+
+variable "resource_group" {
+  type        = string
+  description = "The name of the Azure resource group to create the Constellation cluster in."
+}
+
+variable "location" {
+  type        = string
+  description = "The Azure location to deploy the cluster in."
+}
+
+variable "user_assigned_identity" {
+  type        = string
+  description = "The name of the user assigned identity to attache to the nodes of the cluster."
+}
+
+variable "instance_type" {
+  type        = string
+  description = "The Azure instance type to deploy."
+}
+
+variable "state_disk_type" {
+  type        = string
+  default     = "Premium_LRS"
+  description = "The type of the state disk."
+}
+
+variable "image_id" {
+  type        = string
+  description = "The image to use for the cluster nodes."
+}
+
+variable "confidential_vm" {
+  type        = bool
+  default     = true
+  description = "Whether to deploy the cluster nodes as confidential VMs."
+}
+
+variable "debug" {
+  type        = bool
+  default     = false
+  description = "Enable debug mode. This opens up a debugd port that can be used to deploy a custom bootstrapper."
+}