From 2a61861a1c1efba65214ec0690eac33bbc1b5af3 Mon Sep 17 00:00:00 2001 From: 3u13r Date: Thu, 22 Feb 2024 17:39:34 +0100 Subject: [PATCH] stackit: add k8s api load balancer (#2925) --- cli/internal/cloudcmd/tfvars.go | 1 + cli/internal/terraform/variables.go | 2 + cli/internal/terraform/variables_test.go | 2 + internal/cloud/openstack/api.go | 1 + internal/cloud/openstack/api_test.go | 34 +-- internal/cloud/openstack/imds.go | 29 ++- internal/cloud/openstack/openstack.go | 72 +----- internal/cloud/openstack/openstack_test.go | 209 +----------------- internal/config/config.go | 6 +- internal/config/config_doc.go | 77 ++++--- internal/constellation/helm/overrides.go | 4 +- terraform/BUILD.bazel | 2 + .../openstack/.terraform.lock.hcl | 27 +++ terraform/infrastructure/openstack/main.tf | 104 +++++---- .../openstack/modules/instance_group/main.tf | 15 +- .../modules/instance_group/outputs.tf | 2 +- .../modules/instance_group/variables.tf | 5 + .../modules/stackit_loadbalancer/main.tf | 47 ++++ .../modules/stackit_loadbalancer/variables.tf | 30 +++ .../infrastructure/openstack/variables.tf | 7 + 20 files changed, 298 insertions(+), 378 deletions(-) create mode 100644 terraform/infrastructure/openstack/modules/stackit_loadbalancer/main.tf create mode 100644 terraform/infrastructure/openstack/modules/stackit_loadbalancer/variables.tf diff --git a/cli/internal/cloudcmd/tfvars.go b/cli/internal/cloudcmd/tfvars.go index 6c952210f..ea53cff5d 100644 --- a/cli/internal/cloudcmd/tfvars.go +++ b/cli/internal/cloudcmd/tfvars.go @@ -266,6 +266,7 @@ func openStackTerraformVars(conf *config.Config, imageRef string) (*terraform.Op NodeGroups: nodeGroups, CustomEndpoint: conf.CustomEndpoint, InternalLoadBalancer: conf.InternalLoadBalancer, + STACKITProjectID: conf.Provider.OpenStack.STACKITProjectID, }, nil } diff --git a/cli/internal/terraform/variables.go b/cli/internal/terraform/variables.go index 6dbc117d8..f48ae0d88 100644 --- a/cli/internal/terraform/variables.go +++ b/cli/internal/terraform/variables.go @@ -280,6 +280,8 @@ type OpenStackClusterVariables struct { NodeGroups map[string]OpenStackNodeGroup `hcl:"node_groups" cty:"node_groups"` // Cloud is the (optional) name of the OpenStack cloud to use when reading the "clouds.yaml" configuration file. If empty, environment variables are used. Cloud *string `hcl:"cloud" cty:"cloud"` + // (STACKIT only) STACKITProjectID is the ID of the STACKIT project to use. + STACKITProjectID string `hcl:"stackit_project_id" cty:"stackit_project_id"` // FloatingIPPoolID is the ID of the OpenStack floating IP pool to use for public IPs. FloatingIPPoolID string `hcl:"floating_ip_pool_id" cty:"floating_ip_pool_id"` // ImageID is the ID of the OpenStack image to use. diff --git a/cli/internal/terraform/variables_test.go b/cli/internal/terraform/variables_test.go index 74f8d4ee8..56940e976 100644 --- a/cli/internal/terraform/variables_test.go +++ b/cli/internal/terraform/variables_test.go @@ -260,6 +260,7 @@ func TestOpenStackClusterVariables(t *testing.T) { OpenstackUsername: "my-username", OpenstackPassword: "my-password", Debug: true, + STACKITProjectID: "my-stackit-project-id", NodeGroups: map[string]OpenStackNodeGroup{ constants.ControlPlaneDefault: { Role: "control-plane", @@ -286,6 +287,7 @@ node_groups = { } } cloud = "my-cloud" +stackit_project_id = "my-stackit-project-id" floating_ip_pool_id = "fip-pool-0123456789abcdef" image_id = "8e10b92d-8f7a-458c-91c6-59b42f82ef81" openstack_user_domain_name = "my-user-domain" diff --git a/internal/cloud/openstack/api.go b/internal/cloud/openstack/api.go index aa8d93bc7..b133d2a47 100644 --- a/internal/cloud/openstack/api.go +++ b/internal/cloud/openstack/api.go @@ -24,6 +24,7 @@ type imdsAPI interface { initSecretHash(ctx context.Context) (string, error) role(ctx context.Context) (role.Role, error) vpcIP(ctx context.Context) (string, error) + loadBalancerEndpoint(ctx context.Context) (string, error) } type serversAPI interface { diff --git a/internal/cloud/openstack/api_test.go b/internal/cloud/openstack/api_test.go index e0ec56841..44c26c4ba 100644 --- a/internal/cloud/openstack/api_test.go +++ b/internal/cloud/openstack/api_test.go @@ -17,20 +17,22 @@ import ( ) type stubIMDSClient struct { - providerIDResult string - providerIDErr error - nameResult string - nameErr error - projectIDResult string - projectIDErr error - uidResult string - uidErr error - initSecretHashResult string - initSecretHashErr error - roleResult role.Role - roleErr error - vpcIPResult string - vpcIPErr error + providerIDResult string + providerIDErr error + nameResult string + nameErr error + projectIDResult string + projectIDErr error + uidResult string + uidErr error + initSecretHashResult string + initSecretHashErr error + roleResult role.Role + roleErr error + vpcIPResult string + vpcIPErr error + loadBalancerEndpointResult string + loadBalancerEndpointErr error } func (c *stubIMDSClient) providerID(_ context.Context) (string, error) { @@ -61,6 +63,10 @@ func (c *stubIMDSClient) vpcIP(_ context.Context) (string, error) { return c.vpcIPResult, c.vpcIPErr } +func (c *stubIMDSClient) loadBalancerEndpoint(_ context.Context) (string, error) { + return c.loadBalancerEndpointResult, c.loadBalancerEndpointErr +} + type stubServersClient struct { serversPager stubPager netsPager stubPager diff --git a/internal/cloud/openstack/imds.go b/internal/cloud/openstack/imds.go index b4cabe5d6..c9e3332c8 100644 --- a/internal/cloud/openstack/imds.go +++ b/internal/cloud/openstack/imds.go @@ -128,6 +128,20 @@ func (c *imdsClient) role(ctx context.Context) (role.Role, error) { return role.FromString(c.cache.Tags.Role), nil } +func (c *imdsClient) loadBalancerEndpoint(ctx context.Context) (string, error) { + if c.timeForUpdate(c.cacheTime) || c.cache.Tags.LoadBalancerEndpoint == "" { + if err := c.update(ctx); err != nil { + return "", err + } + } + + if c.cache.Tags.LoadBalancerEndpoint == "" { + return "", errors.New("unable to get load balancer endpoint") + } + + return c.cache.Tags.LoadBalancerEndpoint, nil +} + func (c *imdsClient) authURL(ctx context.Context) (string, error) { if c.timeForUpdate(c.cacheTime) || c.cache.Tags.AuthURL == "" { if err := c.update(ctx); err != nil { @@ -245,13 +259,14 @@ type metadataResponse struct { } type metadataTags struct { - InitSecretHash string `json:"constellation-init-secret-hash,omitempty"` - Role string `json:"constellation-role,omitempty"` - UID string `json:"constellation-uid,omitempty"` - AuthURL string `json:"openstack-auth-url,omitempty"` - UserDomainName string `json:"openstack-user-domain-name,omitempty"` - Username string `json:"openstack-username,omitempty"` - Password string `json:"openstack-password,omitempty"` + InitSecretHash string `json:"constellation-init-secret-hash,omitempty"` + Role string `json:"constellation-role,omitempty"` + UID string `json:"constellation-uid,omitempty"` + AuthURL string `json:"openstack-auth-url,omitempty"` + UserDomainName string `json:"openstack-user-domain-name,omitempty"` + Username string `json:"openstack-username,omitempty"` + Password string `json:"openstack-password,omitempty"` + LoadBalancerEndpoint string `json:"openstack-load-balancer-endpoint,omitempty"` } type httpClient interface { diff --git a/internal/cloud/openstack/openstack.go b/internal/cloud/openstack/openstack.go index 7da227ccc..2e16014f0 100644 --- a/internal/cloud/openstack/openstack.go +++ b/internal/cloud/openstack/openstack.go @@ -8,7 +8,6 @@ package openstack import ( "context" - "errors" "fmt" "net/http" "net/netip" @@ -234,80 +233,13 @@ func (c *Cloud) InitSecretHash(ctx context.Context) ([]byte, error) { // a control plane node. // TODO(malt3): Rewrite to use real load balancer once it is available. func (c *Cloud) GetLoadBalancerEndpoint(ctx context.Context) (host, port string, err error) { - host, err = c.getLoadBalancerHost(ctx) + host, err = c.imds.loadBalancerEndpoint(ctx) if err != nil { - return "", "", fmt.Errorf("getting load balancer host: %w", err) + return "", "", fmt.Errorf("getting load balancer endpoint: %w", err) } return host, strconv.FormatInt(constants.KubernetesPort, 10), nil } -func (c *Cloud) getLoadBalancerHost(ctx context.Context) (string, error) { - uid, err := c.imds.uid(ctx) - if err != nil { - return "", fmt.Errorf("getting uid: %w", err) - } - - uidTag := fmt.Sprintf("constellation-uid-%s", uid) - - subnet, err := c.getSubnetCIDR(uidTag) - if err != nil { - return "", err - } - - srvs, err := c.getServers(uidTag) - if err != nil { - return "", err - } - - for _, s := range srvs { - if s.Name == "" { - continue - } - if s.ID == "" { - continue - } - if s.Tags == nil { - continue - } - - subnetAddrs, err := parseSeverAddresses(s.Addresses) - if err != nil { - return "", fmt.Errorf("parsing server %q addresses: %w", s.Name, err) - } - - // In a best effort approach, we take the first fixed IPv4 address that is outside the subnet - // belonging to our cluster and assume it is the "load balancer" floating ip. - for _, serverSubnet := range subnetAddrs { - for _, addr := range serverSubnet.Addresses { - if addr.Type != floatingIP { - continue - } - - if addr.IPVersion != ipV4 { - continue - } - - if addr.IP == "" { - continue - } - - parsedAddr, err := netip.ParseAddr(addr.IP) - if err != nil { - continue - } - - if subnet.Contains(parsedAddr) { - continue - } - - return addr.IP, nil - } - } - } - - return "", errors.New("no load balancer endpoint found") -} - func (c *Cloud) getSubnetCIDR(uidTag string) (netip.Prefix, error) { listNetworksOpts := networks.ListOpts{Tags: uidTag} networksPage, err := c.api.ListNetworks(listNetworksOpts).AllPages() diff --git a/internal/cloud/openstack/openstack_test.go b/internal/cloud/openstack/openstack_test.go index 3fa2153a5..88e9ff7fd 100644 --- a/internal/cloud/openstack/openstack_test.go +++ b/internal/cloud/openstack/openstack_test.go @@ -465,203 +465,18 @@ func TestInitSecretHash(t *testing.T) { } func TestGetLoadBalancerEndpoint(t *testing.T) { - // newTestAddrs returns a set of raw server addresses as we would get from - // a ListServers call and as expected by the parseSeverAddresses function. - // The hardcoded addresses don't match what we are looking for. A valid - // address can be injected. You can pass a second valid address to test - // that the first valid one is chosen. - newTestAddrs := func(floatingIP1, floatingIP2 string, fixedIP1 string) map[string]any { - return map[string]any{ - "network1": []any{ - map[string]any{ - "addr": "192.0.2.2", - "version": 4, - "OS-EXT-IPS:type": "floating", - "OS-EXT-IPS-MAC:mac_addr": "fa:16:3e:0c:0c:0c", - }, - }, - "network2": []any{ - map[string]any{ - "addr": fixedIP1, - "version": 4, - "OS-EXT-IPS:type": "fixed", - "OS-EXT-IPS-MAC:mac_addr": "fa:16:3e:0c:0c:0c", - }, - map[string]any{ - "addr": "2001:db8:3333:4444:5555:6666:7777:8888", - "version": 6, - "OS-EXT-IPS:type": "floating", - "OS-EXT-IPS-MAC:mac_addr": "fa:16:3e:0c:0c:0c", - }, - map[string]any{ - "addr": floatingIP1, - "version": 4, - "OS-EXT-IPS:type": "floating", - "OS-EXT-IPS-MAC:mac_addr": "fa:16:3e:0c:0c:0c", - }, - map[string]any{ - "addr": floatingIP2, - "version": 4, - "OS-EXT-IPS:type": "floating", - "OS-EXT-IPS-MAC:mac_addr": "fa:16:3e:0c:0c:0c", - }, - }, - } - } - testCases := map[string]struct { - imds *stubIMDSClient - api *stubServersClient - wantHost string - wantErr bool + imds *stubIMDSClient + want string + wantErr bool }{ "error returned from IMDS client": { - imds: &stubIMDSClient{uidErr: errors.New("failed")}, + imds: &stubIMDSClient{loadBalancerEndpointErr: errors.New("failed")}, wantErr: true, }, - "error returned from getSubnetCIDR": { - imds: &stubIMDSClient{}, - api: &stubServersClient{ - netsPager: newNetPager([]networks.Network{{Name: "mynet"}}, nil), - subnetsPager: newSubnetPager(nil, errors.New("failed")), - }, - wantErr: true, - }, - "error returned from getServers": { - imds: &stubIMDSClient{}, - api: &stubServersClient{ - netsPager: newNetPager([]networks.Network{{Name: "mynet"}}, nil), - subnetsPager: newSubnetPager([]subnets.Subnet{{Name: "mynet", CIDR: "192.0.2.0/24"}}, nil), - serversPager: newSeverPager(nil, errors.New("failed")), - }, - wantErr: true, - }, - "sever with empty name skipped": { - imds: &stubIMDSClient{}, - api: &stubServersClient{ - netsPager: newNetPager([]networks.Network{{Name: "mynet"}}, nil), - subnetsPager: newSubnetPager([]subnets.Subnet{{Name: "mynet", CIDR: "192.0.2.0/24"}}, nil), - serversPager: newSeverPager([]servers.Server{ - { - ID: "id1", - Tags: &[]string{"constellation-role-control-plane", "constellation-uid-7777"}, - Addresses: newTestAddrs("198.51.100.0", "", "192.0.2.1"), - }, - }, nil), - }, - wantErr: true, - }, - "server with empty ID skipped": { - imds: &stubIMDSClient{}, - api: &stubServersClient{ - netsPager: newNetPager([]networks.Network{{Name: "mynet"}}, nil), - subnetsPager: newSubnetPager([]subnets.Subnet{{Name: "mynet", CIDR: "192.0.2.0/24"}}, nil), - serversPager: newSeverPager([]servers.Server{ - { - Name: "name1", - Tags: &[]string{"constellation-role-control-plane", "constellation-uid-7777"}, - Addresses: newTestAddrs("198.51.100.0", "", "192.0.2.1"), - }, - }, nil), - }, - wantErr: true, - }, - "sever with nil tags skipped": { - imds: &stubIMDSClient{}, - api: &stubServersClient{ - netsPager: newNetPager([]networks.Network{{Name: "mynet"}}, nil), - subnetsPager: newSubnetPager([]subnets.Subnet{{Name: "mynet", CIDR: "192.0.2.0/24"}}, nil), - serversPager: newSeverPager([]servers.Server{ - { - Name: "name1", - ID: "id1", - Addresses: newTestAddrs("198.51.100.0", "", "192.0.2.1"), - }, - }, nil), - }, - wantErr: true, - }, - "server has invalid address": { - imds: &stubIMDSClient{}, - api: &stubServersClient{ - netsPager: newNetPager([]networks.Network{{Name: "mynet"}}, nil), - subnetsPager: newSubnetPager([]subnets.Subnet{{Name: "mynet", CIDR: "192.0.2.0/24"}}, nil), - serversPager: newSeverPager([]servers.Server{ - { - Name: "name1", - ID: "id1", - Tags: &[]string{"constellation-role-control-plane", "constellation-uid-7777"}, - Addresses: newTestAddrs("", "", "invalidIP"), - }, - }, nil), - }, - wantErr: true, - }, - "server without parseable addresses skipped": { - imds: &stubIMDSClient{}, - api: &stubServersClient{ - netsPager: newNetPager([]networks.Network{{Name: "mynet"}}, nil), - subnetsPager: newSubnetPager([]subnets.Subnet{{Name: "mynet", CIDR: "192.0.2.0/24"}}, nil), - serversPager: newSeverPager([]servers.Server{ - { - Name: "name1", - ID: "id1", - Tags: &[]string{"constellation-role-control-plane", "constellation-uid-7777"}, - Addresses: map[string]any{ - "somekey": "invalid", - }, - }, - }, nil), - }, - wantErr: true, - }, - "invalid endpoint returned from server addresses": { - imds: &stubIMDSClient{}, - api: &stubServersClient{ - netsPager: newNetPager([]networks.Network{{Name: "mynet"}}, nil), - subnetsPager: newSubnetPager([]subnets.Subnet{{Name: "mynet", CIDR: "192.0.2.0/24"}}, nil), - serversPager: newSeverPager([]servers.Server{ - { - Name: "name1", - ID: "id1", - Tags: &[]string{"constellation-role-control-plane", "constellation-uid-7777"}, - Addresses: newTestAddrs("invalidIP", "", "192.0.2.1"), - }, - }, nil), - }, - wantErr: true, - }, - "valid endpoint returned from server addresses not in subnet CIDR": { - imds: &stubIMDSClient{}, - api: &stubServersClient{ - netsPager: newNetPager([]networks.Network{{Name: "mynet"}}, nil), - subnetsPager: newSubnetPager([]subnets.Subnet{{Name: "mynet", CIDR: "192.0.2.0/24"}}, nil), - serversPager: newSeverPager([]servers.Server{ - { - Name: "name1", - ID: "id1", - Tags: &[]string{"constellation-role-control-plane", "constellation-uid-7777"}, - Addresses: newTestAddrs("198.51.100.0", "", "192.0.2.1"), - }, - }, nil), - }, - wantHost: "198.51.100.0", - }, - "first valid endpoint returned from server addresses not in subnet CIDR": { - imds: &stubIMDSClient{}, - api: &stubServersClient{ - netsPager: newNetPager([]networks.Network{{Name: "mynet"}}, nil), - subnetsPager: newSubnetPager([]subnets.Subnet{{Name: "mynet", CIDR: "192.0.2.0/24"}}, nil), - serversPager: newSeverPager([]servers.Server{ - { - Name: "name1", - ID: "id1", - Tags: &[]string{"constellation-role-control-plane", "constellation-uid-7777"}, - Addresses: newTestAddrs("198.51.100.0", "198.51.100.1", "192.0.2.1"), - }, - }, nil), - }, - wantHost: "198.51.100.0", + "UID returned from IMDS client": { + imds: &stubIMDSClient{loadBalancerEndpointResult: "some.endpoint"}, + want: "some.endpoint", }, } @@ -669,19 +484,15 @@ func TestGetLoadBalancerEndpoint(t *testing.T) { t.Run(name, func(t *testing.T) { assert := assert.New(t) - c := &Cloud{ - imds: tc.imds, - api: tc.api, - } + c := &Cloud{imds: tc.imds} - gotHost, gotPort, err := c.GetLoadBalancerEndpoint(context.Background()) + got, _, err := c.GetLoadBalancerEndpoint(context.Background()) if tc.wantErr { assert.Error(err) } else { assert.NoError(err) - assert.Equal(tc.wantHost, gotHost) - assert.Equal("6443", gotPort) + assert.Equal(tc.want, got) } }) } diff --git a/internal/config/config.go b/internal/config/config.go index db0c5dadb..e7417d237 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -207,9 +207,13 @@ type OpenStackConfig struct { // AuthURL is the OpenStack Identity endpoint to use inside the cluster. AuthURL string `yaml:"authURL" validate:"required"` // description: | - // ProjectID is the ID of the project where a user resides. + // ProjectID is the ID of the OpenStack project where a user resides. ProjectID string `yaml:"projectID" validate:"required"` // description: | + // STACKITProjectID is the ID of the STACKIT project where a user resides. + // Only used if cloud is "stackit". + STACKITProjectID string `yaml:"stackitProjectID"` + // description: | // ProjectName is the name of the project where a user resides. ProjectName string `yaml:"projectName" validate:"required"` // description: | diff --git a/internal/config/config_doc.go b/internal/config/config_doc.go index f20c705a0..8665922ed 100644 --- a/internal/config/config_doc.go +++ b/internal/config/config_doc.go @@ -276,7 +276,7 @@ func init() { FieldName: "openstack", }, } - OpenStackConfigDoc.Fields = make([]encoder.Doc, 15) + OpenStackConfigDoc.Fields = make([]encoder.Doc, 16) OpenStackConfigDoc.Fields[0].Name = "cloud" OpenStackConfigDoc.Fields[0].Type = "string" OpenStackConfigDoc.Fields[0].Note = "" @@ -300,58 +300,63 @@ func init() { OpenStackConfigDoc.Fields[4].Name = "projectID" OpenStackConfigDoc.Fields[4].Type = "string" OpenStackConfigDoc.Fields[4].Note = "" - OpenStackConfigDoc.Fields[4].Description = "ProjectID is the ID of the project where a user resides." - OpenStackConfigDoc.Fields[4].Comments[encoder.LineComment] = "ProjectID is the ID of the project where a user resides." - OpenStackConfigDoc.Fields[5].Name = "projectName" + OpenStackConfigDoc.Fields[4].Description = "ProjectID is the ID of the OpenStack project where a user resides." + OpenStackConfigDoc.Fields[4].Comments[encoder.LineComment] = "ProjectID is the ID of the OpenStack project where a user resides." + OpenStackConfigDoc.Fields[5].Name = "stackitProjectID" OpenStackConfigDoc.Fields[5].Type = "string" OpenStackConfigDoc.Fields[5].Note = "" - OpenStackConfigDoc.Fields[5].Description = "ProjectName is the name of the project where a user resides." - OpenStackConfigDoc.Fields[5].Comments[encoder.LineComment] = "ProjectName is the name of the project where a user resides." - OpenStackConfigDoc.Fields[6].Name = "userDomainName" + OpenStackConfigDoc.Fields[5].Description = "STACKITProjectID is the ID of the STACKIT project where a user resides.\nOnly used if cloud is \"stackit\"." + OpenStackConfigDoc.Fields[5].Comments[encoder.LineComment] = "STACKITProjectID is the ID of the STACKIT project where a user resides." + OpenStackConfigDoc.Fields[6].Name = "projectName" OpenStackConfigDoc.Fields[6].Type = "string" OpenStackConfigDoc.Fields[6].Note = "" - OpenStackConfigDoc.Fields[6].Description = "UserDomainName is the name of the domain where a user resides." - OpenStackConfigDoc.Fields[6].Comments[encoder.LineComment] = "UserDomainName is the name of the domain where a user resides." - OpenStackConfigDoc.Fields[7].Name = "projectDomainName" + OpenStackConfigDoc.Fields[6].Description = "ProjectName is the name of the project where a user resides." + OpenStackConfigDoc.Fields[6].Comments[encoder.LineComment] = "ProjectName is the name of the project where a user resides." + OpenStackConfigDoc.Fields[7].Name = "userDomainName" OpenStackConfigDoc.Fields[7].Type = "string" OpenStackConfigDoc.Fields[7].Note = "" - OpenStackConfigDoc.Fields[7].Description = "ProjectDomainName is the name of the domain where a project resides." - OpenStackConfigDoc.Fields[7].Comments[encoder.LineComment] = "ProjectDomainName is the name of the domain where a project resides." - OpenStackConfigDoc.Fields[8].Name = "regionName" + OpenStackConfigDoc.Fields[7].Description = "UserDomainName is the name of the domain where a user resides." + OpenStackConfigDoc.Fields[7].Comments[encoder.LineComment] = "UserDomainName is the name of the domain where a user resides." + OpenStackConfigDoc.Fields[8].Name = "projectDomainName" OpenStackConfigDoc.Fields[8].Type = "string" OpenStackConfigDoc.Fields[8].Note = "" - OpenStackConfigDoc.Fields[8].Description = "description: |\nRegionName is the name of the region to use inside the cluster.\n" - OpenStackConfigDoc.Fields[8].Comments[encoder.LineComment] = "description: |" - OpenStackConfigDoc.Fields[9].Name = "username" + OpenStackConfigDoc.Fields[8].Description = "ProjectDomainName is the name of the domain where a project resides." + OpenStackConfigDoc.Fields[8].Comments[encoder.LineComment] = "ProjectDomainName is the name of the domain where a project resides." + OpenStackConfigDoc.Fields[9].Name = "regionName" OpenStackConfigDoc.Fields[9].Type = "string" OpenStackConfigDoc.Fields[9].Note = "" - OpenStackConfigDoc.Fields[9].Description = "Username to use inside the cluster." - OpenStackConfigDoc.Fields[9].Comments[encoder.LineComment] = "Username to use inside the cluster." - OpenStackConfigDoc.Fields[10].Name = "password" + OpenStackConfigDoc.Fields[9].Description = "description: |\nRegionName is the name of the region to use inside the cluster.\n" + OpenStackConfigDoc.Fields[9].Comments[encoder.LineComment] = "description: |" + OpenStackConfigDoc.Fields[10].Name = "username" OpenStackConfigDoc.Fields[10].Type = "string" OpenStackConfigDoc.Fields[10].Note = "" - OpenStackConfigDoc.Fields[10].Description = "Password to use inside the cluster. You can instead use the environment variable \"CONSTELL_OS_PASSWORD\"." - OpenStackConfigDoc.Fields[10].Comments[encoder.LineComment] = "Password to use inside the cluster. You can instead use the environment variable \"CONSTELL_OS_PASSWORD\"." - OpenStackConfigDoc.Fields[11].Name = "deployYawolLoadBalancer" - OpenStackConfigDoc.Fields[11].Type = "bool" + OpenStackConfigDoc.Fields[10].Description = "Username to use inside the cluster." + OpenStackConfigDoc.Fields[10].Comments[encoder.LineComment] = "Username to use inside the cluster." + OpenStackConfigDoc.Fields[11].Name = "password" + OpenStackConfigDoc.Fields[11].Type = "string" OpenStackConfigDoc.Fields[11].Note = "" - OpenStackConfigDoc.Fields[11].Description = "Deploy Yawol loadbalancer. For details see: https://github.com/stackitcloud/yawol" - OpenStackConfigDoc.Fields[11].Comments[encoder.LineComment] = "Deploy Yawol loadbalancer. For details see: https://github.com/stackitcloud/yawol" - OpenStackConfigDoc.Fields[12].Name = "yawolImageID" - OpenStackConfigDoc.Fields[12].Type = "string" + OpenStackConfigDoc.Fields[11].Description = "Password to use inside the cluster. You can instead use the environment variable \"CONSTELL_OS_PASSWORD\"." + OpenStackConfigDoc.Fields[11].Comments[encoder.LineComment] = "Password to use inside the cluster. You can instead use the environment variable \"CONSTELL_OS_PASSWORD\"." + OpenStackConfigDoc.Fields[12].Name = "deployYawolLoadBalancer" + OpenStackConfigDoc.Fields[12].Type = "bool" OpenStackConfigDoc.Fields[12].Note = "" - OpenStackConfigDoc.Fields[12].Description = "OpenStack OS image used by the yawollet. For details see: https://github.com/stackitcloud/yawol" - OpenStackConfigDoc.Fields[12].Comments[encoder.LineComment] = "OpenStack OS image used by the yawollet. For details see: https://github.com/stackitcloud/yawol" - OpenStackConfigDoc.Fields[13].Name = "yawolFlavorID" + OpenStackConfigDoc.Fields[12].Description = "Deploy Yawol loadbalancer. For details see: https://github.com/stackitcloud/yawol" + OpenStackConfigDoc.Fields[12].Comments[encoder.LineComment] = "Deploy Yawol loadbalancer. For details see: https://github.com/stackitcloud/yawol" + OpenStackConfigDoc.Fields[13].Name = "yawolImageID" OpenStackConfigDoc.Fields[13].Type = "string" OpenStackConfigDoc.Fields[13].Note = "" - OpenStackConfigDoc.Fields[13].Description = "OpenStack flavor id used for yawollets. For details see: https://github.com/stackitcloud/yawol" - OpenStackConfigDoc.Fields[13].Comments[encoder.LineComment] = "OpenStack flavor id used for yawollets. For details see: https://github.com/stackitcloud/yawol" - OpenStackConfigDoc.Fields[14].Name = "deployCSIDriver" - OpenStackConfigDoc.Fields[14].Type = "bool" + OpenStackConfigDoc.Fields[13].Description = "OpenStack OS image used by the yawollet. For details see: https://github.com/stackitcloud/yawol" + OpenStackConfigDoc.Fields[13].Comments[encoder.LineComment] = "OpenStack OS image used by the yawollet. For details see: https://github.com/stackitcloud/yawol" + OpenStackConfigDoc.Fields[14].Name = "yawolFlavorID" + OpenStackConfigDoc.Fields[14].Type = "string" OpenStackConfigDoc.Fields[14].Note = "" - OpenStackConfigDoc.Fields[14].Description = "Deploy Cinder CSI driver with on-node encryption. For details see: https://docs.edgeless.systems/constellation/architecture/encrypted-storage" - OpenStackConfigDoc.Fields[14].Comments[encoder.LineComment] = "Deploy Cinder CSI driver with on-node encryption. For details see: https://docs.edgeless.systems/constellation/architecture/encrypted-storage" + OpenStackConfigDoc.Fields[14].Description = "OpenStack flavor id used for yawollets. For details see: https://github.com/stackitcloud/yawol" + OpenStackConfigDoc.Fields[14].Comments[encoder.LineComment] = "OpenStack flavor id used for yawollets. For details see: https://github.com/stackitcloud/yawol" + OpenStackConfigDoc.Fields[15].Name = "deployCSIDriver" + OpenStackConfigDoc.Fields[15].Type = "bool" + OpenStackConfigDoc.Fields[15].Note = "" + OpenStackConfigDoc.Fields[15].Description = "Deploy Cinder CSI driver with on-node encryption. For details see: https://docs.edgeless.systems/constellation/architecture/encrypted-storage" + OpenStackConfigDoc.Fields[15].Comments[encoder.LineComment] = "Deploy Cinder CSI driver with on-node encryption. For details see: https://docs.edgeless.systems/constellation/architecture/encrypted-storage" QEMUConfigDoc.Type = "QEMUConfig" QEMUConfigDoc.Comments[encoder.LineComment] = "QEMUConfig holds config information for QEMU based Constellation deployments." diff --git a/internal/constellation/helm/overrides.go b/internal/constellation/helm/overrides.go index 6e158dc8a..6dfea0c2c 100644 --- a/internal/constellation/helm/overrides.go +++ b/internal/constellation/helm/overrides.go @@ -44,7 +44,9 @@ func extraCiliumValues(provider cloudprovider.Provider, conformanceMode bool, ou } strictMode := map[string]any{} - if provider != cloudprovider.QEMU { + // TODO(@3u13r): Once we are able to set the subnet of the load balancer VMs + // on STACKIT, we can remove the OpenStack exception here. + if provider != cloudprovider.QEMU && provider != cloudprovider.OpenStack { strictMode = map[string]any{ "nodeCIDRList": []string{output.IPCidrNode}, } diff --git a/terraform/BUILD.bazel b/terraform/BUILD.bazel index 74d83041e..f4f99f242 100644 --- a/terraform/BUILD.bazel +++ b/terraform/BUILD.bazel @@ -75,6 +75,8 @@ go_library( "infrastructure/aws/modules/jump_host/output.tf", "infrastructure/aws/modules/load_balancer_target/output.tf", "infrastructure/aws/modules/public_private_subnet/output.tf", + "infrastructure/openstack/modules/stackit_loadbalancer/main.tf", + "infrastructure/openstack/modules/stackit_loadbalancer/variables.tf", ], importpath = "github.com/edgelesssys/constellation/v2/terraform", visibility = ["//visibility:public"], diff --git a/terraform/infrastructure/openstack/.terraform.lock.hcl b/terraform/infrastructure/openstack/.terraform.lock.hcl index 58131d13c..6f96f0f72 100644 --- a/terraform/infrastructure/openstack/.terraform.lock.hcl +++ b/terraform/infrastructure/openstack/.terraform.lock.hcl @@ -25,6 +25,33 @@ provider "registry.terraform.io/hashicorp/random" { ] } +provider "registry.terraform.io/stackitcloud/stackit" { + version = "0.12.0" + constraints = "0.12.0" + hashes = [ + "h1:08k0ihJixjWGyzNF0wdMiOckr+4qfBi50yj4tTLsbMM=", + "h1:8wtUYCXZke9uJiWp3Y7/tRy84UM0TjOzrzhb6BAX5vo=", + "h1:EwUqtQ7b/ShFcNvBMiemsbrvqBwFfkIRtnEIeIisKSA=", + "h1:lPXt86IQA6bHnX6o6xIaOUHqbAs6WHAehwtS1kK3wcg=", + "h1:t+pHh9fQCS+4Rq9STVs+npH3DOe7qp1L0rJfbMjAdjM=", + "zh:0dde99e7b343fa01f8eefc378171fb8621bedb20f59157d6cc8e3d46c738105f", + "zh:13ff6111adb804e3e7a33a0e8e341e494a84a81115b144c950ea9864ce12efdb", + "zh:2b13aff4a4879b833e27d215102c98809fe78d9a1fb33d09ec352760d21fa7c3", + "zh:6562b6ca55bebd7e425fba60ba5683a3cb00d49d50883e37f418b5be8d52d992", + "zh:6ce745a9a2fac88fd7b219dca1d70882e3c1b573e2d27a49de0a04b76ceabdf0", + "zh:70dd57f2e59596f697aaeab377423a041a57e066d1ad8bbfc0ace9cfaf6e9e0d", + "zh:7bb24a57ef0d802c62d23249078d86a0daeba29b7508d46bb8d104c5b820f35b", + "zh:93b57ec66d0f18ef616416f9d39a5a5b45dde604145b66e5184f00840db7a981", + "zh:9646f12a59a3eab161040eee68093b4c55864c865d544fa83d0e56bfbc59c174", + "zh:c23b3433b81eb99e314239add0df206a5388ef79884e924537bf09d4374815a8", + "zh:d2ef1946a5d559a72dac15a38a78f8d2d09bcd13068d9fe1debe7ae82e9c527d", + "zh:d63299ca4bf158573706a0c313dbee0aa79c7b910d85a0a748ba77620f533a5d", + "zh:e796aec8e1c64c7142d1b2877794ff8cb6fc5699292dfea102f2f229375626a2", + "zh:eb4003be226dc810004cd6a50d98f872d61bb49f2891a2966247a245c9d7cc1c", + "zh:f62e5390fca4d920c3db329276e1780ae57cc20aa666ee549dcf452d4f839ba5", + ] +} + provider "registry.terraform.io/terraform-provider-openstack/openstack" { version = "1.54.1" constraints = "1.54.1" diff --git a/terraform/infrastructure/openstack/main.tf b/terraform/infrastructure/openstack/main.tf index 33287f44c..797423654 100644 --- a/terraform/infrastructure/openstack/main.tf +++ b/terraform/infrastructure/openstack/main.tf @@ -5,6 +5,11 @@ terraform { version = "1.54.1" } + stackit = { + source = "stackitcloud/stackit" + version = "0.12.0" + } + random = { source = "hashicorp/random" version = "3.6.0" @@ -16,6 +21,11 @@ provider "openstack" { cloud = var.cloud } +provider "stackit" { + region = "eu01" +} + + data "openstack_identity_auth_scope_v3" "scope" { name = "scope" } @@ -26,15 +36,17 @@ locals { init_secret_hash = random_password.init_secret.bcrypt_hash ports_node_range_start = "30000" ports_node_range_end = "32767" - ports_kubernetes = "6443" - ports_bootstrapper = "9000" - ports_konnectivity = "8132" - ports_verify = "30081" - ports_recovery = "9999" - ports_debugd = "4000" - cidr_vpc_subnet_nodes = "192.168.178.0/24" - cidr_vpc_subnet_lbs = "192.168.177.0/24" - tags = ["constellation-uid-${local.uid}"] + control_plane_named_ports = flatten([ + { name = "kubernetes", port = "6443", health_check = "HTTPS" }, + { name = "bootstrapper", port = "9000", health_check = "TCP" }, + { name = "verify", port = "30081", health_check = "TCP" }, + { name = "recovery", port = "9999", health_check = "TCP" }, + { name = "join", port = "30090", health_check = "TCP" }, + var.debug ? [{ name = "debugd", port = "4000", health_check = "TCP" }] : [], + ]) + cidr_vpc_subnet_nodes = "192.168.178.0/24" + cidr_vpc_subnet_lbs = "192.168.177.0/24" + tags = ["constellation-uid-${local.uid}"] identity_service = [ for entry in data.openstack_identity_auth_scope_v3.scope.service_catalog : entry if entry.type == "identity" @@ -194,46 +206,40 @@ resource "openstack_networking_secgroup_rule_v2" "nodeport_udp" { security_group_id = openstack_networking_secgroup_v2.vpc_secgroup.id } -resource "openstack_networking_secgroup_rule_v2" "tcp_port_forward" { - for_each = toset(flatten([ - local.ports_kubernetes, - local.ports_bootstrapper, - local.ports_konnectivity, - local.ports_verify, - local.ports_recovery, - var.debug ? [local.ports_debugd] : [], - ])) +resource "openstack_networking_secgroup_rule_v2" "tcp_ingress" { + for_each = { for item in local.control_plane_named_ports : item.name => item } direction = "ingress" ethertype = "IPv4" protocol = "tcp" - port_range_min = each.value - port_range_max = each.value + port_range_min = each.value.port + port_range_max = each.value.port security_group_id = openstack_networking_secgroup_v2.vpc_secgroup.id } module "instance_group" { - source = "./modules/instance_group" - for_each = var.node_groups - base_name = local.name - node_group_name = each.key - role = each.value.role - initial_count = each.value.initial_count - disk_size = each.value.state_disk_size - state_disk_type = each.value.state_disk_type - availability_zone = each.value.zone - image_id = var.image_id - flavor_id = each.value.flavor_id - security_groups = [openstack_networking_secgroup_v2.vpc_secgroup.id] - tags = local.tags - uid = local.uid - network_id = openstack_networking_network_v2.vpc_network.id - subnet_id = openstack_networking_subnet_v2.vpc_subnetwork.id - init_secret_hash = local.init_secret_hash - identity_internal_url = local.identity_internal_url - openstack_username = var.openstack_username - openstack_password = var.openstack_password - openstack_user_domain_name = var.openstack_user_domain_name + source = "./modules/instance_group" + for_each = var.node_groups + base_name = local.name + node_group_name = each.key + role = each.value.role + initial_count = each.value.initial_count + disk_size = each.value.state_disk_size + state_disk_type = each.value.state_disk_type + availability_zone = each.value.zone + image_id = var.image_id + flavor_id = each.value.flavor_id + security_groups = [openstack_networking_secgroup_v2.vpc_secgroup.id] + tags = local.tags + uid = local.uid + network_id = openstack_networking_network_v2.vpc_network.id + subnet_id = openstack_networking_subnet_v2.vpc_subnetwork.id + init_secret_hash = local.init_secret_hash + identity_internal_url = local.identity_internal_url + openstack_username = var.openstack_username + openstack_password = var.openstack_password + openstack_user_domain_name = var.openstack_user_domain_name + openstack_load_balancer_endpoint = openstack_networking_floatingip_v2.public_ip.address } resource "openstack_networking_floatingip_v2" "public_ip" { @@ -242,8 +248,8 @@ resource "openstack_networking_floatingip_v2" "public_ip" { tags = local.tags } - resource "openstack_networking_floatingip_associate_v2" "public_ip_associate" { + count = var.cloud == "stackit" ? 0 : 1 floating_ip = openstack_networking_floatingip_v2.public_ip.address port_id = module.instance_group["control_plane_default"].port_ids.0 depends_on = [ @@ -252,6 +258,20 @@ resource "openstack_networking_floatingip_associate_v2" "public_ip_associate" { ] } +module "stackit_loadbalancer" { + count = var.cloud == "stackit" ? 1 : 0 + source = "./modules/stackit_loadbalancer" + name = local.name + stackit_project_id = var.stackit_project_id + member_ips = module.instance_group["control_plane_default"].ips + network_id = openstack_networking_network_v2.vpc_network.id + external_address = openstack_networking_floatingip_v2.public_ip.address + ports = { + for port in local.control_plane_named_ports : port.name => port.port + } +} + + moved { from = module.instance_group_control_plane to = module.instance_group["control_plane_default"] diff --git a/terraform/infrastructure/openstack/modules/instance_group/main.tf b/terraform/infrastructure/openstack/modules/instance_group/main.tf index 1d7cee430..fe175066a 100644 --- a/terraform/infrastructure/openstack/modules/instance_group/main.tf +++ b/terraform/infrastructure/openstack/modules/instance_group/main.tf @@ -65,13 +65,14 @@ resource "openstack_compute_instance_v2" "instance_group_member" { delete_on_termination = true } metadata = { - constellation-role = var.role - constellation-uid = var.uid - constellation-init-secret-hash = var.init_secret_hash - openstack-auth-url = var.identity_internal_url - openstack-username = var.openstack_username - openstack-password = var.openstack_password - openstack-user-domain-name = var.openstack_user_domain_name + constellation-role = var.role + constellation-uid = var.uid + constellation-init-secret-hash = var.init_secret_hash + openstack-auth-url = var.identity_internal_url + openstack-username = var.openstack_username + openstack-password = var.openstack_password + openstack-user-domain-name = var.openstack_user_domain_name + openstack-load-balancer-endpoint = var.openstack_load_balancer_endpoint } availability_zone_hints = var.availability_zone } diff --git a/terraform/infrastructure/openstack/modules/instance_group/outputs.tf b/terraform/infrastructure/openstack/modules/instance_group/outputs.tf index 974c195c1..3a97a0284 100644 --- a/terraform/infrastructure/openstack/modules/instance_group/outputs.tf +++ b/terraform/infrastructure/openstack/modules/instance_group/outputs.tf @@ -1,5 +1,5 @@ output "ips" { - value = openstack_compute_instance_v2.instance_group_member.*.access_ip_v4 + value = [for instance in openstack_compute_instance_v2.instance_group_member : instance.access_ip_v4] description = "Public IP addresses of the instances." } diff --git a/terraform/infrastructure/openstack/modules/instance_group/variables.tf b/terraform/infrastructure/openstack/modules/instance_group/variables.tf index a9d0fdd66..74f0f9e28 100644 --- a/terraform/infrastructure/openstack/modules/instance_group/variables.tf +++ b/terraform/infrastructure/openstack/modules/instance_group/variables.tf @@ -96,3 +96,8 @@ variable "openstack_password" { type = string description = "OpenStack password." } + +variable "openstack_load_balancer_endpoint" { + type = string + description = "OpenStack load balancer endpoint." +} diff --git a/terraform/infrastructure/openstack/modules/stackit_loadbalancer/main.tf b/terraform/infrastructure/openstack/modules/stackit_loadbalancer/main.tf new file mode 100644 index 000000000..cbe08c83b --- /dev/null +++ b/terraform/infrastructure/openstack/modules/stackit_loadbalancer/main.tf @@ -0,0 +1,47 @@ +terraform { + required_providers { + stackit = { + source = "stackitcloud/stackit" + version = "0.12.0" + } + } +} + +resource "stackit_loadbalancer" "loadbalancer" { + project_id = var.stackit_project_id + name = "${var.name}-lb" + target_pools = [ + for portName, port in var.ports : { + name = "target-pool-${portName}" + target_port = port + targets = [ + for ip in var.member_ips : { + display_name = "target-${portName}" + ip = ip + } + ] + active_health_check = { + healthy_threshold = 10 + interval = "3s" + interval_jitter = "3s" + timeout = "3s" + unhealthy_threshold = 10 + } + } + ] + listeners = [ + for portName, port in var.ports : { + name = "listener-${portName}" + port = port + protocol = "PROTOCOL_TCP" + target_pool = "target-pool-${portName}" + } + ] + networks = [ + { + network_id = var.network_id + role = "ROLE_LISTENERS_AND_TARGETS" + } + ] + external_address = var.external_address +} diff --git a/terraform/infrastructure/openstack/modules/stackit_loadbalancer/variables.tf b/terraform/infrastructure/openstack/modules/stackit_loadbalancer/variables.tf new file mode 100644 index 000000000..0141ebde1 --- /dev/null +++ b/terraform/infrastructure/openstack/modules/stackit_loadbalancer/variables.tf @@ -0,0 +1,30 @@ +variable "name" { + type = string + description = "Base name of the load balancer." +} + +variable "member_ips" { + type = list(string) + description = "IP addresses of the members of the load balancer pool." + default = [] +} + +variable "network_id" { + type = string + description = "ID of the network." +} + +variable "external_address" { + type = string + description = "External address of the load balancer." +} + +variable "ports" { + type = map(number) + description = "Ports to listen on incoming traffic." +} + +variable "stackit_project_id" { + type = string + description = "STACKIT project ID." +} diff --git a/terraform/infrastructure/openstack/variables.tf b/terraform/infrastructure/openstack/variables.tf index 01c497730..3b0983d68 100644 --- a/terraform/infrastructure/openstack/variables.tf +++ b/terraform/infrastructure/openstack/variables.tf @@ -67,3 +67,10 @@ variable "openstack_password" { type = string description = "OpenStack password." } + +# STACKIT-specific variables + +variable "stackit_project_id" { + type = string + description = "STACKIT project ID." +}