Use Terraform for create on GCP

This commit is contained in:
katexochen 2022-09-27 09:22:29 +02:00 committed by Paul Meyer
parent f990c4d692
commit d973740b03
25 changed files with 341 additions and 607 deletions

View file

@ -1,35 +0,0 @@
/*
Copyright (c) Edgeless Systems GmbH
SPDX-License-Identifier: AGPL-3.0-only
*/
package terraform
// CreateClusterInput is user configuration for creating a cluster with Terraform.
type CreateClusterInput struct {
// CountControlPlanes is the number of control-plane nodes to create.
CountControlPlanes int
// CountWorkers is the number of worker nodes to create.
CountWorkers int
// QEMU is the configuration for QEMU clusters.
QEMU QEMUInput
}
// QEMUInput is user configuration for creating a QEMU cluster with Terraform.
type QEMUInput struct {
// CPUCount is the number of CPUs to allocate to each node.
CPUCount int
// MemorySizeMiB is the amount of memory to allocate to each node, in MiB.
MemorySizeMiB int
// StateDiskSizeGB is the size of the state disk to allocate to each node, in GB.
StateDiskSizeGB int
// IPRangeStart is the first IP address in the IP range to allocate to the cluster.
IPRangeStart int
// ImagePath is the path to the image to use for the nodes.
ImagePath string
// ImageFormat is the format of the image from ImagePath.
ImageFormat string
// MetadataAPIImage is the container image to use for the metadata API.
MetadataAPIImage string
}

View file

@ -9,7 +9,6 @@ package terraform
import (
"context"
"errors"
"fmt"
"github.com/edgelesssys/constellation/v2/internal/cloud/cloudprovider"
"github.com/edgelesssys/constellation/v2/internal/file"
@ -59,7 +58,7 @@ func New(ctx context.Context, provider cloudprovider.Provider) (*Client, error)
}
// CreateCluster creates a Constellation cluster using Terraform.
func (c *Client) CreateCluster(ctx context.Context, name string, input CreateClusterInput) error {
func (c *Client) CreateCluster(ctx context.Context, name string, vars Variables) error {
if err := prepareWorkspace(c.file, c.provider); err != nil {
return err
}
@ -68,7 +67,7 @@ func (c *Client) CreateCluster(ctx context.Context, name string, input CreateClu
return err
}
if err := writeUserConfig(c.file, c.provider, name, input); err != nil {
if err := c.file.Write(terraformVarsFile, []byte(vars.String())); err != nil {
return err
}
@ -137,35 +136,6 @@ func (c *Client) GetState() state.ConstellationState {
return c.state
}
// writeUserConfig writes the user config file for Terraform.
func writeUserConfig(file file.Handler, provider cloudprovider.Provider, name string, input CreateClusterInput) error {
var userConfig string
switch provider {
case cloudprovider.QEMU:
userConfig = fmt.Sprintf(`
constellation_coreos_image = "%s"
image_format = "%s"
control_plane_count = %d
worker_count = %d
vcpus = %d
memory = %d
state_disk_size = %d
ip_range_start = %d
metadata_api_image = "%s"
name = "%s"
`,
input.QEMU.ImagePath, input.QEMU.ImageFormat,
input.CountControlPlanes, input.CountWorkers,
input.QEMU.CPUCount, input.QEMU.MemorySizeMiB, input.QEMU.StateDiskSizeGB,
input.QEMU.IPRangeStart,
input.QEMU.MetadataAPIImage,
name,
)
}
return file.Write(terraformVarsFile, []byte(userConfig))
}
// GetExecutable returns a Terraform executable either from the local filesystem,
// or downloads the latest version fulfilling the version constraint.
func GetExecutable(ctx context.Context, workingDir string) (terraform *tfexec.Terraform, remove func(), err error) {

View file

@ -0,0 +1,210 @@
terraform {
required_providers {
google = {
source = "hashicorp/google"
version = "4.34.0"
}
random = {
source = "hashicorp/random"
version = "3.4.1"
}
}
}
provider "google" {
credentials = file(var.credentials_file)
project = var.project
region = var.region
zone = var.zone
}
locals {
uid = random_id.uid.hex
name = "${var.name}-${local.uid}"
tag = "constellation-${local.uid}"
ports_node_range = "30000-32767"
ports_kubernetes = "6443"
ports_bootstrapper = "9000"
ports_konnectivity = "8132"
ports_verify = "30081"
ports_recovery = "9999"
ports_debugd = "4000"
cidr_vpc_subnet_nodes = "192.168.178.0/24"
cidr_vpc_subnet_pods = "10.10.0.0/16"
kube_env = "AUTOSCALER_ENV_VARS: kube_reserved=cpu=1060m,memory=1019Mi,ephemeral-storage=41Gi;node_labels=;os=linux;os_distribution=cos;evictionHard="
}
resource "random_id" "uid" {
byte_length = 4
}
resource "google_compute_network" "vpc_network" {
name = local.name
description = "Constellation VPC network"
auto_create_subnetworks = false
}
resource "google_compute_subnetwork" "vpc_subnetwork" {
name = local.name
description = "Constellation VPC subnetwork"
network = google_compute_network.vpc_network.id
ip_cidr_range = local.cidr_vpc_subnet_nodes
secondary_ip_range = [
{
range_name = local.name,
ip_cidr_range = local.cidr_vpc_subnet_pods,
}
]
}
resource "google_compute_firewall" "firewall_external" {
name = local.name
description = "Constellation VPC firewall"
network = google_compute_network.vpc_network.id
source_ranges = ["0.0.0.0/0"]
direction = "INGRESS"
allow {
protocol = "tcp"
ports = flatten([
local.ports_node_range,
local.ports_bootstrapper,
local.ports_kubernetes,
local.ports_konnectivity,
local.ports_recovery,
var.debug ? [local.ports_debugd] : [],
])
}
}
resource "google_compute_firewall" "firewall_internal_nodes" {
name = "${local.name}-nodes"
description = "Constellation VPC firewall"
network = google_compute_network.vpc_network.id
source_ranges = [local.cidr_vpc_subnet_nodes]
direction = "INGRESS"
allow { protocol = "tcp" }
allow { protocol = "udp" }
allow { protocol = "icmp" }
}
resource "google_compute_firewall" "firewall_internal_pods" {
name = "${local.name}-pods"
description = "Constellation VPC firewall"
network = google_compute_network.vpc_network.id
source_ranges = [local.cidr_vpc_subnet_pods]
direction = "INGRESS"
allow { protocol = "tcp" }
allow { protocol = "udp" }
allow { protocol = "icmp" }
}
module "instance_group_control_plane" {
source = "./modules/instance_group"
name = local.name
role = "ControlPlane"
uid = local.uid
instance_type = var.instance_type
instance_count = var.control_plane_count
image_id = var.image_id
disk_size = var.state_disk_size
disk_type = var.state_disk_type
network = google_compute_network.vpc_network.id
subnetwork = google_compute_subnetwork.vpc_subnetwork.id
kube_env = local.kube_env
named_ports = flatten([
{ name = "kubernetes", port = local.ports_kubernetes },
{ name = "bootstrapper", port = local.ports_bootstrapper },
{ name = "verify", port = local.ports_verify },
{ name = "konnectivity", port = local.ports_konnectivity },
{ name = "recovery", port = local.ports_recovery },
var.debug ? [{ name = "debugd", port = local.ports_debugd }] : [],
])
}
module "instance_group_worker" {
source = "./modules/instance_group"
name = local.name
role = "Worker"
uid = local.uid
instance_type = var.instance_type
instance_count = var.worker_count
image_id = var.image_id
disk_size = var.state_disk_size
disk_type = var.state_disk_type
network = google_compute_network.vpc_network.id
subnetwork = google_compute_subnetwork.vpc_subnetwork.id
kube_env = local.kube_env
}
resource "google_compute_global_address" "loadbalancer_ip" {
name = local.name
}
module "loadbalancer_kube" {
source = "./modules/loadbalancer"
name = local.name
health_check = "HTTPS"
backend_port_name = "kubernetes"
backend_instance_group = module.instance_group_control_plane.instance_group
ip_address = google_compute_global_address.loadbalancer_ip.self_link
port = local.ports_kubernetes
frontend_labels = {
constellation-uid = local.uid
}
}
module "loadbalancer_boot" {
source = "./modules/loadbalancer"
name = local.name
health_check = "TCP"
backend_port_name = "bootstrapper"
backend_instance_group = module.instance_group_control_plane.instance_group
ip_address = google_compute_global_address.loadbalancer_ip.self_link
port = local.ports_bootstrapper
}
module "loadbalancer_verify" {
source = "./modules/loadbalancer"
name = local.name
health_check = "TCP"
backend_port_name = "verify"
backend_instance_group = module.instance_group_control_plane.instance_group
ip_address = google_compute_global_address.loadbalancer_ip.self_link
port = local.ports_verify
}
module "loadbalancer_konnectivity" {
source = "./modules/loadbalancer"
name = local.name
health_check = "TCP"
backend_port_name = "konnectivity"
backend_instance_group = module.instance_group_control_plane.instance_group
ip_address = google_compute_global_address.loadbalancer_ip.self_link
port = local.ports_konnectivity
}
module "loadbalancer_recovery" {
source = "./modules/loadbalancer"
name = local.name
health_check = "TCP"
backend_port_name = "recovery"
backend_instance_group = module.instance_group_control_plane.instance_group
ip_address = google_compute_global_address.loadbalancer_ip.self_link
port = local.ports_recovery
}
module "loadbalancer_debugd" {
count = var.debug ? 1 : 0 // only deploy debugd in debug mode
source = "./modules/loadbalancer"
name = local.name
health_check = "TCP"
backend_port_name = "debugd"
backend_instance_group = module.instance_group_control_plane.instance_group
ip_address = google_compute_global_address.loadbalancer_ip.self_link
port = local.ports_debugd
}

View file

@ -0,0 +1,97 @@
terraform {
required_providers {
google = {
source = "hashicorp/google"
version = "4.34.0"
}
}
}
locals {
role_dashed = var.role == "ControlPlane" ? "control-plane" : "worker"
name = "${var.name}-${local.role_dashed}"
}
resource "google_compute_instance_template" "template" {
name = local.name
machine_type = var.instance_type
tags = ["constellation-${var.uid}"]
confidential_instance_config {
enable_confidential_compute = true
}
disk {
disk_size_gb = 10
source_image = var.image_id
auto_delete = true
boot = true
mode = "READ_WRITE"
}
disk {
disk_size_gb = var.disk_size
disk_type = var.disk_type
auto_delete = true
device_name = "state-disk" // This name is used by disk mapper to find the disk
boot = false
mode = "READ_WRITE"
type = "PERSISTENT"
}
metadata = {
kube-env = var.kube_env
constellation-uid = var.uid
constellation-role = var.role
}
network_interface {
network = var.network
subnetwork = var.subnetwork
access_config {}
alias_ip_range {
ip_cidr_range = "/24"
subnetwork_range_name = var.name
}
}
scheduling {
on_host_maintenance = "TERMINATE"
}
service_account {
scopes = [
"https://www.googleapis.com/auth/compute",
"https://www.googleapis.com/auth/servicecontrol",
"https://www.googleapis.com/auth/service.management.readonly",
"https://www.googleapis.com/auth/devstorage.read_only",
"https://www.googleapis.com/auth/logging.write",
"https://www.googleapis.com/auth/monitoring.write",
"https://www.googleapis.com/auth/trace.append",
]
}
shielded_instance_config {
enable_secure_boot = true
enable_vtpm = true
enable_integrity_monitoring = true
}
}
resource "google_compute_instance_group_manager" "instance_group_manager" {
name = local.name
base_instance_name = local.name
target_size = var.instance_count
version {
instance_template = google_compute_instance_template.template.id
}
dynamic "named_port" {
for_each = toset(var.named_ports)
content {
name = named_port.value.name
port = named_port.value.port
}
}
}

View file

@ -0,0 +1,3 @@
output "instance_group" {
value = google_compute_instance_group_manager.instance_group_manager.instance_group
}

View file

@ -0,0 +1,60 @@
variable "name" {
type = string
description = "Base name of the instance group."
}
variable "role" {
type = string
description = "The role of the instance group. Has to be 'ControlPlane' or 'Worker'."
}
variable "uid" {
type = string
description = "UID of the cluster. This is used for tags."
}
variable "instance_type" {
type = string
description = "Instance type for the nodes."
}
variable "instance_count" {
type = number
description = "Number of instances in the instance group."
}
variable "image_id" {
type = string
description = "Image ID for the nodes."
}
variable "disk_size" {
type = number
description = "Disk size for the nodes, in GB."
}
variable "disk_type" {
type = string
description = "Disk type for the nodes. Has to be 'pd-standard' or 'pd-ssd'."
}
variable "network" {
type = string
description = "Name of the network to use."
}
variable "subnetwork" {
type = string
description = "Name of the subnetwork to use."
}
variable "kube_env" {
type = string
description = "Kubernetes env."
}
variable "named_ports" {
type = list(object({ name = string, port = number }))
default = []
description = "Named ports for the instance group."
}

View file

@ -0,0 +1,62 @@
terraform {
required_providers {
google = {
source = "hashicorp/google"
version = "4.34.0"
}
}
}
locals {
name = "${var.name}-${var.backend_port_name}"
}
resource "google_compute_health_check" "health" {
name = local.name
check_interval_sec = 1
timeout_sec = 1
dynamic "tcp_health_check" {
for_each = var.health_check == "TCP" ? [1] : []
content {
port = var.port
}
}
dynamic "https_health_check" {
for_each = var.health_check == "HTTPS" ? [1] : []
content {
host = ""
port = var.port
request_path = "/readyz"
}
}
}
resource "google_compute_backend_service" "backend" {
name = local.name
protocol = "TCP"
load_balancing_scheme = "EXTERNAL"
health_checks = [google_compute_health_check.health.self_link]
port_name = var.backend_port_name
backend {
group = var.backend_instance_group
balancing_mode = "UTILIZATION"
}
}
resource "google_compute_target_tcp_proxy" "proxy" {
name = local.name
backend_service = google_compute_backend_service.backend.self_link
}
resource "google_compute_global_forwarding_rule" "forwarding" {
name = local.name
ip_address = var.ip_address
ip_protocol = "TCP"
load_balancing_scheme = "EXTERNAL"
port_range = var.port
target = google_compute_target_tcp_proxy.proxy.self_link
labels = var.frontend_labels
}

View file

@ -0,0 +1,35 @@
variable "name" {
type = string
description = "Base name of the load balancer."
}
variable "health_check" {
type = string
description = "The type of the health check. 'HTTPS' or 'TCP'."
}
variable "backend_port_name" {
type = string
description = "Name of backend port. The same name should appear in the instance groups referenced by this service."
}
variable "backend_instance_group" {
type = string
description = "The URL of the instance group resource from which the load balancer will direct traffic."
}
variable "ip_address" {
type = string
description = "The IP address that this forwarding rule serves. An address can be specified either by a literal IP address or a reference to an existing Address resource."
}
variable "port" {
type = number
description = "The port on which to listen for incoming traffic."
}
variable "frontend_labels" {
type = map(string)
default = {}
description = "Labels to apply to the forwarding rule."
}

View file

@ -0,0 +1,3 @@
output "ip" {
value = google_compute_global_address.loadbalancer_ip.address
}

View file

@ -0,0 +1,63 @@
variable "name" {
type = string
default = "constell"
description = "Base name of the cluster."
}
variable "control_plane_count" {
type = number
description = "The number of control plane nodes to deploy."
}
variable "worker_count" {
type = number
description = "The number of worker nodes to deploy."
}
variable "state_disk_size" {
type = number
default = 30
description = "The size of the state disk in GB."
}
variable "project" {
type = string
description = "The GCP project to deploy the cluster in."
}
variable "region" {
type = string
description = "The GCP region to deploy the cluster in."
}
variable "zone" {
type = string
description = "The GCP zone to deploy the cluster in."
}
variable "credentials_file" {
type = string
description = "The path to the GCP credentials file."
}
variable "instance_type" {
type = string
description = "The GCP instance type to deploy."
}
variable "state_disk_type" {
type = string
default = "pd-ssd"
description = "The type of the state disk."
}
variable "image_id" {
type = string
description = "The GCP image to use for the cluster nodes."
}
variable "debug" {
type = bool
default = false
description = "Enable debug mode. This opens up a debugd port that can be used to deploy a custom bootstrapper."
}

View file

@ -22,9 +22,9 @@ import (
"go.uber.org/multierr"
)
func TestCreateInstances(t *testing.T) {
someErr := errors.New("error")
getState := func() *tfjson.State {
func TestCreateCluster(t *testing.T) {
someErr := errors.New("failed")
newTestState := func() *tfjson.State {
workingState := tfjson.State{
Values: &tfjson.StateValues{
Outputs: map[string]*tfjson.StateOutput{
@ -34,52 +34,59 @@ func TestCreateInstances(t *testing.T) {
},
},
}
return &workingState
}
qemuVars := &QEMUVariables{
CommonVariables: CommonVariables{
Name: "name",
CountControlPlanes: 1,
CountWorkers: 2,
StateDiskSizeGB: 11,
},
CPUCount: 1,
MemorySizeMiB: 1024,
IPRangeStart: 100,
ImagePath: "path",
ImageFormat: "format",
MetadataAPIImage: "api",
}
testCases := map[string]struct {
provider cloudprovider.Provider
input CreateClusterInput
vars Variables
tf *stubTerraform
fs afero.Fs
wantErr bool
}{
"works": {
provider: cloudprovider.QEMU,
tf: &stubTerraform{
showState: getState(),
},
fs: afero.NewMemMapFs(),
vars: qemuVars,
tf: &stubTerraform{showState: newTestState()},
fs: afero.NewMemMapFs(),
},
"init fails": {
provider: cloudprovider.QEMU,
tf: &stubTerraform{
initErr: someErr,
showState: getState(),
},
fs: afero.NewMemMapFs(),
wantErr: true,
tf: &stubTerraform{initErr: someErr},
fs: afero.NewMemMapFs(),
wantErr: true,
},
"apply fails": {
provider: cloudprovider.QEMU,
tf: &stubTerraform{
applyErr: someErr,
showState: getState(),
},
fs: afero.NewMemMapFs(),
wantErr: true,
vars: qemuVars,
tf: &stubTerraform{applyErr: someErr},
fs: afero.NewMemMapFs(),
wantErr: true,
},
"show fails": {
provider: cloudprovider.QEMU,
tf: &stubTerraform{
showErr: someErr,
},
fs: afero.NewMemMapFs(),
wantErr: true,
vars: qemuVars,
tf: &stubTerraform{showErr: someErr},
fs: afero.NewMemMapFs(),
wantErr: true,
},
"no ip": {
provider: cloudprovider.QEMU,
vars: qemuVars,
tf: &stubTerraform{
showState: &tfjson.State{
Values: &tfjson.StateValues{
@ -90,13 +97,24 @@ func TestCreateInstances(t *testing.T) {
fs: afero.NewMemMapFs(),
wantErr: true,
},
"ip has wrong type": {
provider: cloudprovider.QEMU,
vars: qemuVars,
tf: &stubTerraform{
showState: &tfjson.State{
Values: &tfjson.StateValues{
Outputs: map[string]*tfjson.StateOutput{"ip": {Value: 42}},
},
},
},
fs: afero.NewMemMapFs(),
wantErr: true,
},
"prepare workspace fails": {
provider: cloudprovider.QEMU,
tf: &stubTerraform{
showState: getState(),
},
fs: afero.NewReadOnlyFs(afero.NewMemMapFs()),
wantErr: true,
tf: &stubTerraform{showState: newTestState()},
fs: afero.NewReadOnlyFs(afero.NewMemMapFs()),
wantErr: true,
},
}
@ -110,12 +128,12 @@ func TestCreateInstances(t *testing.T) {
file: file.NewHandler(tc.fs),
}
err := c.CreateCluster(context.Background(), "test", tc.input)
err := c.CreateCluster(context.Background(), "test", tc.vars)
if tc.wantErr {
assert.Error(err)
return
}
assert.NoError(err)
})
}

View file

@ -0,0 +1,117 @@
/*
Copyright (c) Edgeless Systems GmbH
SPDX-License-Identifier: AGPL-3.0-only
*/
package terraform
import (
"fmt"
"strings"
)
// Variables is a struct that holds all variables that are passed to Terraform.
type Variables interface {
fmt.Stringer
}
// CommonVariables is user configuration for creating a cluster with Terraform.
type CommonVariables struct {
// Name of the cluster.
Name string
// CountControlPlanes is the number of control-plane nodes to create.
CountControlPlanes int
// CountWorkers is the number of worker nodes to create.
CountWorkers int
// StateDiskSizeGB is the size of the state disk to allocate to each node, in GB.
StateDiskSizeGB int
}
// String returns a string representation of the variables, formatted as Terraform variables.
func (v *CommonVariables) String() string {
b := &strings.Builder{}
writeLinef(b, "name = %q", v.Name)
writeLinef(b, "control_plane_count = %d", v.CountControlPlanes)
writeLinef(b, "worker_count = %d", v.CountWorkers)
writeLinef(b, "state_disk_size = %d", v.StateDiskSizeGB)
return b.String()
}
// GCPVariables is user configuration for creating a cluster with Terraform on GCP.
type GCPVariables struct {
// CommonVariables contains common variables.
CommonVariables
// Project is the ID of the GCP project to use.
Project string
// Region is the GCP region to use.
Region string
// Zone is the GCP zone to use.
Zone string
// CredentialsFile is the path to the GCP credentials file.
CredentialsFile string
// InstanceType is the GCP instance type to use.
InstanceType string
// StateDiskType is the GCP disk type to use for the state disk.
StateDiskType string
// ImageID is the ID of the GCP image to use.
ImageID string
// Debug is true if debug mode is enabled.
Debug bool
}
// String returns a string representation of the variables, formatted as Terraform variables.
func (v *GCPVariables) String() string {
b := &strings.Builder{}
b.WriteString(v.CommonVariables.String())
writeLinef(b, "project = %q", v.Project)
writeLinef(b, "region = %q", v.Region)
writeLinef(b, "zone = %q", v.Zone)
writeLinef(b, "credentials_file = %q", v.CredentialsFile)
writeLinef(b, "instance_type = %q", v.InstanceType)
writeLinef(b, "state_disk_type = %q", v.StateDiskType)
writeLinef(b, "image_id = %q", v.ImageID)
writeLinef(b, "debug = %t", v.Debug)
return b.String()
}
// QEMUVariables is user configuration for creating a QEMU cluster with Terraform.
type QEMUVariables struct {
// CommonVariables contains common variables.
CommonVariables
// CPUCount is the number of CPUs to allocate to each node.
CPUCount int
// MemorySizeMiB is the amount of memory to allocate to each node, in MiB.
MemorySizeMiB int
// IPRangeStart is the first IP address in the IP range to allocate to the cluster.
IPRangeStart int
// ImagePath is the path to the image to use for the nodes.
ImagePath string
// ImageFormat is the format of the image from ImagePath.
ImageFormat string
// MetadataAPIImage is the container image to use for the metadata API.
MetadataAPIImage string
}
// String returns a string representation of the variables, formatted as Terraform variables.
func (v *QEMUVariables) String() string {
b := &strings.Builder{}
b.WriteString(v.CommonVariables.String())
writeLinef(b, "constellation_coreos_image = %q", v.ImagePath)
writeLinef(b, "image_format = %q", v.ImageFormat)
writeLinef(b, "vcpus = %d", v.CPUCount)
writeLinef(b, "memory = %d", v.MemorySizeMiB)
writeLinef(b, "ip_range_start = %d", v.IPRangeStart)
writeLinef(b, "metadata_api_image = %q", v.MetadataAPIImage)
return b.String()
}
func writeLinef(builder *strings.Builder, format string, a ...interface{}) {
builder.WriteString(fmt.Sprintf(format, a...))
builder.WriteByte('\n')
}