From b1879665813c0374ee4010268477ffbe0ba26524 Mon Sep 17 00:00:00 2001 From: miampf Date: Fri, 19 Apr 2024 09:07:57 +0000 Subject: [PATCH] cli: allow tagging cloud resources with custom tags (#3033) --- cli/internal/cloudcmd/tfvars.go | 12 +++++++ cli/internal/cmd/configgenerate.go | 33 +++++++++++++++++++ cli/internal/terraform/variables.go | 10 +++++- cli/internal/terraform/variables_test.go | 4 +++ docs/docs/reference/cli.md | 1 + internal/cloud/cloudprovider/cloudprovider.go | 3 ++ internal/config/config.go | 4 +++ internal/config/config_doc.go | 31 +++++++++-------- terraform/infrastructure/aws/main.tf | 14 ++++---- .../aws/modules/jump_host/main.tf | 4 +-- .../aws/modules/jump_host/variables.tf | 5 +++ terraform/infrastructure/aws/variables.tf | 5 +++ terraform/infrastructure/azure/main.tf | 15 ++++++--- .../azure/modules/jump_host/main.tf | 3 ++ .../azure/modules/jump_host/variables.tf | 5 +++ .../load_balancer_backend/variables.tf | 1 + terraform/infrastructure/azure/variables.tf | 5 +++ terraform/infrastructure/gcp/main.tf | 9 ++--- terraform/infrastructure/gcp/variables.tf | 5 +++ terraform/infrastructure/openstack/main.tf | 2 +- .../infrastructure/openstack/variables.tf | 5 +++ .../legacy-module/aws-constellation/main.tf | 1 + .../aws-constellation/variables.tf | 5 +++ .../legacy-module/azure-constellation/main.tf | 1 + .../azure-constellation/variables.tf | 5 +++ .../legacy-module/gcp-constellation/main.tf | 21 ++++++------ .../gcp-constellation/variables.tf | 5 +++ 27 files changed, 172 insertions(+), 42 deletions(-) diff --git a/cli/internal/cloudcmd/tfvars.go b/cli/internal/cloudcmd/tfvars.go index 818590599..50df8dbcc 100644 --- a/cli/internal/cloudcmd/tfvars.go +++ b/cli/internal/cloudcmd/tfvars.go @@ -104,6 +104,7 @@ func awsTerraformVars(conf *config.Config, imageRef string) *terraform.AWSCluste EnableSNP: conf.GetAttestationConfig().GetVariant().Equal(variant.AWSSEVSNP{}), CustomEndpoint: conf.CustomEndpoint, InternalLoadBalancer: conf.InternalLoadBalancer, + AdditionalTags: conf.Tags, } } @@ -158,6 +159,7 @@ func azureTerraformVars(conf *config.Config, imageRef string) (*terraform.AzureC CustomEndpoint: conf.CustomEndpoint, InternalLoadBalancer: conf.InternalLoadBalancer, MarketplaceImage: nil, + AdditionalTags: conf.Tags, } if conf.UseMarketplaceImage() { @@ -226,6 +228,7 @@ func gcpTerraformVars(conf *config.Config, imageRef string) *terraform.GCPCluste CustomEndpoint: conf.CustomEndpoint, InternalLoadBalancer: conf.InternalLoadBalancer, CCTechnology: ccTech, + AdditionalLabels: conf.Tags, } } @@ -261,6 +264,14 @@ func openStackTerraformVars(conf *config.Config, imageRef string) (*terraform.Op StateDiskType: group.StateDiskType, } } + + // since openstack does not support tags in the form of key = value, the tags will be converted + // to an array of "key=value" strings + tags := []string{} + for key, value := range conf.Tags { + tags = append(tags, fmt.Sprintf("%s=%s", key, value)) + } + return &terraform.OpenStackClusterVariables{ Name: conf.Name, Cloud: toPtr(conf.Provider.OpenStack.Cloud), @@ -272,6 +283,7 @@ func openStackTerraformVars(conf *config.Config, imageRef string) (*terraform.Op CustomEndpoint: conf.CustomEndpoint, InternalLoadBalancer: conf.InternalLoadBalancer, STACKITProjectID: conf.Provider.OpenStack.STACKITProjectID, + AdditionalTags: tags, }, nil } diff --git a/cli/internal/cmd/configgenerate.go b/cli/internal/cmd/configgenerate.go index 4fabe40e3..fd9796e2c 100644 --- a/cli/internal/cmd/configgenerate.go +++ b/cli/internal/cmd/configgenerate.go @@ -37,6 +37,7 @@ func newConfigGenerateCmd() *cobra.Command { } cmd.Flags().StringP("kubernetes", "k", semver.MajorMinor(string(config.Default().KubernetesVersion)), "Kubernetes version to use in format MAJOR.MINOR") cmd.Flags().StringP("attestation", "a", "", fmt.Sprintf("attestation variant to use %s. If not specified, the default for the cloud provider is used", printFormattedSlice(variant.GetAvailableAttestationVariants()))) + cmd.Flags().StringSliceP("tags", "t", nil, "additional tags for created resources given a list of key=value") return cmd } @@ -45,6 +46,7 @@ type generateFlags struct { rootFlags k8sVersion versions.ValidK8sVersion attestationVariant variant.Variant + tags cloudprovider.Tags } func (f *generateFlags) parse(flags *pflag.FlagSet) error { @@ -64,6 +66,12 @@ func (f *generateFlags) parse(flags *pflag.FlagSet) error { } f.attestationVariant = variant + tags, err := parseTagsFlags(flags) + if err != nil { + return err + } + f.tags = tags + return nil } @@ -99,6 +107,7 @@ func (cg *configGenerateCmd) configGenerate(cmd *cobra.Command, fileHandler file return fmt.Errorf("creating config: %w", err) } conf.KubernetesVersion = cg.flags.k8sVersion + conf.Tags = cg.flags.tags cg.log.Debug("Writing YAML data to configuration file") if err := fileHandler.WriteYAML(constants.ConfigFilename, conf, file.OptMkdirAll); err != nil { return fmt.Errorf("writing config file: %w", err) @@ -221,3 +230,27 @@ func parseAttestationFlag(flags *pflag.FlagSet) (variant.Variant, error) { return attestationVariant, nil } + +func parseTagsFlags(flags *pflag.FlagSet) (cloudprovider.Tags, error) { + tagsSlice, err := flags.GetStringSlice("tags") + if err != nil { + return nil, fmt.Errorf("getting tags flag: %w", err) + } + + // no tags given + if tagsSlice == nil { + return nil, nil + } + + tags := make(cloudprovider.Tags) + for _, tag := range tagsSlice { + tagSplit := strings.Split(tag, "=") + if len(tagSplit) != 2 { + return nil, fmt.Errorf("wrong format of tags: expected \"key=value\", got %q", tag) + } + + tags[tagSplit[0]] = tagSplit[1] + } + + return tags, nil +} diff --git a/cli/internal/terraform/variables.go b/cli/internal/terraform/variables.go index f258a2d92..081ae5946 100644 --- a/cli/internal/terraform/variables.go +++ b/cli/internal/terraform/variables.go @@ -9,6 +9,7 @@ package terraform import ( "fmt" + "github.com/edgelesssys/constellation/v2/internal/cloud/cloudprovider" "github.com/hashicorp/hcl/v2" "github.com/hashicorp/hcl/v2/gohcl" "github.com/hashicorp/hcl/v2/hclsyntax" @@ -69,6 +70,8 @@ type AWSClusterVariables struct { CustomEndpoint string `hcl:"custom_endpoint" cty:"custom_endpoint"` // InternalLoadBalancer is true if an internal load balancer should be created. InternalLoadBalancer bool `hcl:"internal_load_balancer" cty:"internal_load_balancer"` + // AdditionalTags describes (optional) additional tags that should be applied to created resources. + AdditionalTags cloudprovider.Tags `hcl:"additional_tags" cty:"additional_tags"` } // GetCreateMAA gets the CreateMAA variable. @@ -138,6 +141,8 @@ type GCPClusterVariables struct { InternalLoadBalancer bool `hcl:"internal_load_balancer" cty:"internal_load_balancer"` // CCTechnology is the confidential computing technology to use on the VMs. (`SEV` or `SEV_SNP`) CCTechnology string `hcl:"cc_technology" cty:"cc_technology"` + // AdditionalLables are (optional) additional labels that should be applied to created resources. + AdditionalLabels cloudprovider.Tags `hcl:"additional_labels" cty:"additional_labels"` } // GetCreateMAA gets the CreateMAA variable. @@ -214,6 +219,8 @@ type AzureClusterVariables struct { InternalLoadBalancer bool `hcl:"internal_load_balancer" cty:"internal_load_balancer"` // MarketplaceImage is the (optional) Azure Marketplace image to use. MarketplaceImage *AzureMarketplaceImageVariables `hcl:"marketplace_image" cty:"marketplace_image"` + // AdditionalTags are (optional) additional tags that get applied to created resources. + AdditionalTags cloudprovider.Tags `hcl:"additional_tags" cty:"additional_tags"` } // GetCreateMAA gets the CreateMAA variable. @@ -295,7 +302,8 @@ type OpenStackClusterVariables struct { // CustomEndpoint is the (optional) custom dns hostname for the kubernetes api server. CustomEndpoint string `hcl:"custom_endpoint" cty:"custom_endpoint"` // InternalLoadBalancer is true if an internal load balancer should be created. - InternalLoadBalancer bool `hcl:"internal_load_balancer" cty:"internal_load_balancer"` + InternalLoadBalancer bool `hcl:"internal_load_balancer" cty:"internal_load_balancer"` + AdditionalTags []string `hcl:"additional_tags" cty:"additional_tags"` } // GetCreateMAA gets the CreateMAA variable. diff --git a/cli/internal/terraform/variables_test.go b/cli/internal/terraform/variables_test.go index 1c0ccb76b..306acd6ae 100644 --- a/cli/internal/terraform/variables_test.go +++ b/cli/internal/terraform/variables_test.go @@ -76,6 +76,7 @@ node_groups = { } custom_endpoint = "example.com" internal_load_balancer = false +additional_tags = null ` got := vars.String() assert.Equal(t, strings.Fields(want), strings.Fields(got)) // to ignore whitespace differences @@ -153,6 +154,7 @@ node_groups = { custom_endpoint = "example.com" internal_load_balancer = false cc_technology = "SEV_SNP" +additional_labels = null ` got := vars.String() assert.Equal(t, strings.Fields(want), strings.Fields(got)) // to ignore whitespace differences @@ -231,6 +233,7 @@ marketplace_image = { publisher = "edgelesssys" version = "2.13.0" } +additional_tags = null ` got := vars.String() assert.Equal(t, strings.Fields(want), strings.Fields(got)) // to ignore whitespace differences @@ -294,6 +297,7 @@ image_id = "8e10b92d-8f7a-458c-91c6-59b42f82ef81" debug = true custom_endpoint = "example.com" internal_load_balancer = false +additional_tags = null ` got := vars.String() assert.Equal(t, strings.Fields(want), strings.Fields(got)) // to ignore whitespace differences diff --git a/docs/docs/reference/cli.md b/docs/docs/reference/cli.md index f536ea914..909495b7f 100644 --- a/docs/docs/reference/cli.md +++ b/docs/docs/reference/cli.md @@ -81,6 +81,7 @@ constellation config generate {aws|azure|gcp|openstack|qemu|stackit} [flags] -a, --attestation string attestation variant to use {aws-sev-snp|aws-nitro-tpm|azure-sev-snp|azure-tdx|azure-trustedlaunch|gcp-sev-es|gcp-sev-snp|qemu-vtpm}. If not specified, the default for the cloud provider is used -h, --help help for generate -k, --kubernetes string Kubernetes version to use in format MAJOR.MINOR (default "v1.28") + -t, --tags strings additional tags for created resources given a list of key=value ``` ### Options inherited from parent commands diff --git a/internal/cloud/cloudprovider/cloudprovider.go b/internal/cloud/cloudprovider/cloudprovider.go index 47791f943..204ae305c 100644 --- a/internal/cloud/cloudprovider/cloudprovider.go +++ b/internal/cloud/cloudprovider/cloudprovider.go @@ -16,6 +16,9 @@ import ( // Provider is cloud provider used by the CLI. type Provider uint32 +// Tags is the type that holds additional tags for cloud resources. +type Tags map[string]string + const ( // Unknown is default value for Provider. Unknown Provider = iota diff --git a/internal/config/config.go b/internal/config/config.go index c3ea6b34d..ca4d68bed 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -89,6 +89,9 @@ type Config struct { // The Kubernetes Service CIDR to be used for the cluster. This value will only be used during the first initialization of the Constellation. ServiceCIDR string `yaml:"serviceCIDR" validate:"omitempty,cidrv4"` // description: | + // Additional tags that are applied to created resources. + Tags cloudprovider.Tags `yaml:"tags" validate:"omitempty"` + // description: | // Supported cloud providers and their specific configurations. Provider ProviderConfig `yaml:"provider" validate:"dive"` // description: | @@ -322,6 +325,7 @@ func Default() *Config { KubernetesVersion: versions.Default, DebugCluster: toPtr(false), ServiceCIDR: "10.96.0.0/12", + Tags: cloudprovider.Tags{}, Provider: ProviderConfig{ AWS: &AWSConfig{ Region: "", diff --git a/internal/config/config_doc.go b/internal/config/config_doc.go index 56f358d03..f155db5c2 100644 --- a/internal/config/config_doc.go +++ b/internal/config/config_doc.go @@ -37,7 +37,7 @@ func init() { ConfigDoc.Type = "Config" ConfigDoc.Comments[encoder.LineComment] = "Config defines configuration used by CLI." ConfigDoc.Description = "Config defines configuration used by CLI." - ConfigDoc.Fields = make([]encoder.Doc, 12) + ConfigDoc.Fields = make([]encoder.Doc, 13) ConfigDoc.Fields[0].Name = "version" ConfigDoc.Fields[0].Type = "string" ConfigDoc.Fields[0].Note = "" @@ -83,21 +83,26 @@ func init() { ConfigDoc.Fields[8].Note = "" ConfigDoc.Fields[8].Description = "The Kubernetes Service CIDR to be used for the cluster. This value will only be used during the first initialization of the Constellation." ConfigDoc.Fields[8].Comments[encoder.LineComment] = "The Kubernetes Service CIDR to be used for the cluster. This value will only be used during the first initialization of the Constellation." - ConfigDoc.Fields[9].Name = "provider" - ConfigDoc.Fields[9].Type = "ProviderConfig" + ConfigDoc.Fields[9].Name = "tags" + ConfigDoc.Fields[9].Type = "Tags" ConfigDoc.Fields[9].Note = "" - ConfigDoc.Fields[9].Description = "Supported cloud providers and their specific configurations." - ConfigDoc.Fields[9].Comments[encoder.LineComment] = "Supported cloud providers and their specific configurations." - ConfigDoc.Fields[10].Name = "nodeGroups" - ConfigDoc.Fields[10].Type = "map[string]NodeGroup" + ConfigDoc.Fields[9].Description = "Additional tags that are applied to created resources." + ConfigDoc.Fields[9].Comments[encoder.LineComment] = "Additional tags that are applied to created resources." + ConfigDoc.Fields[10].Name = "provider" + ConfigDoc.Fields[10].Type = "ProviderConfig" ConfigDoc.Fields[10].Note = "" - ConfigDoc.Fields[10].Description = "Node groups to be created in the cluster." - ConfigDoc.Fields[10].Comments[encoder.LineComment] = "Node groups to be created in the cluster." - ConfigDoc.Fields[11].Name = "attestation" - ConfigDoc.Fields[11].Type = "AttestationConfig" + ConfigDoc.Fields[10].Description = "Supported cloud providers and their specific configurations." + ConfigDoc.Fields[10].Comments[encoder.LineComment] = "Supported cloud providers and their specific configurations." + ConfigDoc.Fields[11].Name = "nodeGroups" + ConfigDoc.Fields[11].Type = "map[string]NodeGroup" ConfigDoc.Fields[11].Note = "" - ConfigDoc.Fields[11].Description = "Configuration for attestation validation. This configuration provides sensible defaults for the Constellation version it was created for.\nSee the docs for an overview on attestation: https://docs.edgeless.systems/constellation/architecture/attestation" - ConfigDoc.Fields[11].Comments[encoder.LineComment] = "Configuration for attestation validation. This configuration provides sensible defaults for the Constellation version it was created for.\nSee the docs for an overview on attestation: https://docs.edgeless.systems/constellation/architecture/attestation" + ConfigDoc.Fields[11].Description = "Node groups to be created in the cluster." + ConfigDoc.Fields[11].Comments[encoder.LineComment] = "Node groups to be created in the cluster." + ConfigDoc.Fields[12].Name = "attestation" + ConfigDoc.Fields[12].Type = "AttestationConfig" + ConfigDoc.Fields[12].Note = "" + ConfigDoc.Fields[12].Description = "Configuration for attestation validation. This configuration provides sensible defaults for the Constellation version it was created for.\nSee the docs for an overview on attestation: https://docs.edgeless.systems/constellation/architecture/attestation" + ConfigDoc.Fields[12].Comments[encoder.LineComment] = "Configuration for attestation validation. This configuration provides sensible defaults for the Constellation version it was created for.\nSee the docs for an overview on attestation: https://docs.edgeless.systems/constellation/architecture/attestation" ProviderConfigDoc.Type = "ProviderConfig" ProviderConfigDoc.Comments[encoder.LineComment] = "ProviderConfig are cloud-provider specific configuration values used by the CLI." diff --git a/terraform/infrastructure/aws/main.tf b/terraform/infrastructure/aws/main.tf index 9204e638e..5f6012de3 100644 --- a/terraform/infrastructure/aws/main.tf +++ b/terraform/infrastructure/aws/main.tf @@ -68,7 +68,7 @@ resource "random_password" "init_secret" { resource "aws_vpc" "vpc" { cidr_block = "192.168.0.0/16" - tags = merge(local.tags, { Name = "${local.name}-vpc" }) + tags = merge(local.tags, var.additional_tags, { Name = "${local.name}-vpc" }) } module "public_private_subnet" { @@ -79,7 +79,7 @@ module "public_private_subnet" { cidr_vpc_subnet_internet = "192.168.0.0/20" zone = var.zone zones = local.zones - tags = local.tags + tags = merge(local.tags, var.additional_tags) } resource "aws_eip" "lb" { @@ -89,14 +89,14 @@ resource "aws_eip" "lb" { # control-plane. for_each = var.internal_load_balancer ? [] : toset([var.zone]) domain = "vpc" - tags = merge(local.tags, { "constellation-ip-endpoint" = each.key == var.zone ? "legacy-primary-zone" : "additional-zone" }) + tags = merge(local.tags, var.additional_tags, { "constellation-ip-endpoint" = each.key == var.zone ? "legacy-primary-zone" : "additional-zone" }) } resource "aws_lb" "front_end" { name = "${local.name}-loadbalancer" internal = var.internal_load_balancer load_balancer_type = "network" - tags = local.tags + tags = merge(local.tags, var.additional_tags) security_groups = [aws_security_group.security_group.id] dynamic "subnet_mapping" { @@ -123,7 +123,7 @@ resource "aws_security_group" "security_group" { name = local.name vpc_id = aws_vpc.vpc.id description = "Security group for ${local.name}" - tags = local.tags + tags = merge(local.tags, var.additional_tags) egress { from_port = 0 @@ -171,7 +171,7 @@ module "load_balancer_targets" { healthcheck_path = each.value.name == "kubernetes" ? "/readyz" : "" vpc_id = aws_vpc.vpc.id lb_arn = aws_lb.front_end.arn - tags = local.tags + tags = merge(local.tags, var.additional_tags) } module "instance_group" { @@ -194,6 +194,7 @@ module "instance_group" { enable_snp = var.enable_snp tags = merge( local.tags, + var.additional_tags, { Name = "${local.name}-${each.value.role}" }, { constellation-role = each.value.role }, { constellation-node-group = each.key }, @@ -212,4 +213,5 @@ module "jump_host" { ports = [for port in local.load_balancer_ports : port.port] security_groups = [aws_security_group.security_group.id] iam_instance_profile = var.iam_instance_profile_name_worker_nodes + additional_tags = var.additional_tags } diff --git a/terraform/infrastructure/aws/modules/jump_host/main.tf b/terraform/infrastructure/aws/modules/jump_host/main.tf index dc3df3e2d..24f8edd82 100644 --- a/terraform/infrastructure/aws/modules/jump_host/main.tf +++ b/terraform/infrastructure/aws/modules/jump_host/main.tf @@ -26,9 +26,9 @@ resource "aws_instance" "jump_host" { subnet_id = var.subnet_id vpc_security_group_ids = var.security_groups - tags = { + tags = merge(var.additional_tags, { "Name" = "${var.base_name}-jump-host" - } + }) user_data = <