stackit: add k8s api load balancer (#2925)

This commit is contained in:
3u13r 2024-02-22 17:39:34 +01:00 committed by GitHub
parent 62acec17f6
commit 2a61861a1c
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
20 changed files with 298 additions and 378 deletions

View File

@ -266,6 +266,7 @@ func openStackTerraformVars(conf *config.Config, imageRef string) (*terraform.Op
NodeGroups: nodeGroups,
CustomEndpoint: conf.CustomEndpoint,
InternalLoadBalancer: conf.InternalLoadBalancer,
STACKITProjectID: conf.Provider.OpenStack.STACKITProjectID,
}, nil
}

View File

@ -280,6 +280,8 @@ type OpenStackClusterVariables struct {
NodeGroups map[string]OpenStackNodeGroup `hcl:"node_groups" cty:"node_groups"`
// Cloud is the (optional) name of the OpenStack cloud to use when reading the "clouds.yaml" configuration file. If empty, environment variables are used.
Cloud *string `hcl:"cloud" cty:"cloud"`
// (STACKIT only) STACKITProjectID is the ID of the STACKIT project to use.
STACKITProjectID string `hcl:"stackit_project_id" cty:"stackit_project_id"`
// FloatingIPPoolID is the ID of the OpenStack floating IP pool to use for public IPs.
FloatingIPPoolID string `hcl:"floating_ip_pool_id" cty:"floating_ip_pool_id"`
// ImageID is the ID of the OpenStack image to use.

View File

@ -260,6 +260,7 @@ func TestOpenStackClusterVariables(t *testing.T) {
OpenstackUsername: "my-username",
OpenstackPassword: "my-password",
Debug: true,
STACKITProjectID: "my-stackit-project-id",
NodeGroups: map[string]OpenStackNodeGroup{
constants.ControlPlaneDefault: {
Role: "control-plane",
@ -286,6 +287,7 @@ node_groups = {
}
}
cloud = "my-cloud"
stackit_project_id = "my-stackit-project-id"
floating_ip_pool_id = "fip-pool-0123456789abcdef"
image_id = "8e10b92d-8f7a-458c-91c6-59b42f82ef81"
openstack_user_domain_name = "my-user-domain"

View File

@ -24,6 +24,7 @@ type imdsAPI interface {
initSecretHash(ctx context.Context) (string, error)
role(ctx context.Context) (role.Role, error)
vpcIP(ctx context.Context) (string, error)
loadBalancerEndpoint(ctx context.Context) (string, error)
}
type serversAPI interface {

View File

@ -17,20 +17,22 @@ import (
)
type stubIMDSClient struct {
providerIDResult string
providerIDErr error
nameResult string
nameErr error
projectIDResult string
projectIDErr error
uidResult string
uidErr error
initSecretHashResult string
initSecretHashErr error
roleResult role.Role
roleErr error
vpcIPResult string
vpcIPErr error
providerIDResult string
providerIDErr error
nameResult string
nameErr error
projectIDResult string
projectIDErr error
uidResult string
uidErr error
initSecretHashResult string
initSecretHashErr error
roleResult role.Role
roleErr error
vpcIPResult string
vpcIPErr error
loadBalancerEndpointResult string
loadBalancerEndpointErr error
}
func (c *stubIMDSClient) providerID(_ context.Context) (string, error) {
@ -61,6 +63,10 @@ func (c *stubIMDSClient) vpcIP(_ context.Context) (string, error) {
return c.vpcIPResult, c.vpcIPErr
}
func (c *stubIMDSClient) loadBalancerEndpoint(_ context.Context) (string, error) {
return c.loadBalancerEndpointResult, c.loadBalancerEndpointErr
}
type stubServersClient struct {
serversPager stubPager
netsPager stubPager

View File

@ -128,6 +128,20 @@ func (c *imdsClient) role(ctx context.Context) (role.Role, error) {
return role.FromString(c.cache.Tags.Role), nil
}
func (c *imdsClient) loadBalancerEndpoint(ctx context.Context) (string, error) {
if c.timeForUpdate(c.cacheTime) || c.cache.Tags.LoadBalancerEndpoint == "" {
if err := c.update(ctx); err != nil {
return "", err
}
}
if c.cache.Tags.LoadBalancerEndpoint == "" {
return "", errors.New("unable to get load balancer endpoint")
}
return c.cache.Tags.LoadBalancerEndpoint, nil
}
func (c *imdsClient) authURL(ctx context.Context) (string, error) {
if c.timeForUpdate(c.cacheTime) || c.cache.Tags.AuthURL == "" {
if err := c.update(ctx); err != nil {
@ -245,13 +259,14 @@ type metadataResponse struct {
}
type metadataTags struct {
InitSecretHash string `json:"constellation-init-secret-hash,omitempty"`
Role string `json:"constellation-role,omitempty"`
UID string `json:"constellation-uid,omitempty"`
AuthURL string `json:"openstack-auth-url,omitempty"`
UserDomainName string `json:"openstack-user-domain-name,omitempty"`
Username string `json:"openstack-username,omitempty"`
Password string `json:"openstack-password,omitempty"`
InitSecretHash string `json:"constellation-init-secret-hash,omitempty"`
Role string `json:"constellation-role,omitempty"`
UID string `json:"constellation-uid,omitempty"`
AuthURL string `json:"openstack-auth-url,omitempty"`
UserDomainName string `json:"openstack-user-domain-name,omitempty"`
Username string `json:"openstack-username,omitempty"`
Password string `json:"openstack-password,omitempty"`
LoadBalancerEndpoint string `json:"openstack-load-balancer-endpoint,omitempty"`
}
type httpClient interface {

View File

@ -8,7 +8,6 @@ package openstack
import (
"context"
"errors"
"fmt"
"net/http"
"net/netip"
@ -234,80 +233,13 @@ func (c *Cloud) InitSecretHash(ctx context.Context) ([]byte, error) {
// a control plane node.
// TODO(malt3): Rewrite to use real load balancer once it is available.
func (c *Cloud) GetLoadBalancerEndpoint(ctx context.Context) (host, port string, err error) {
host, err = c.getLoadBalancerHost(ctx)
host, err = c.imds.loadBalancerEndpoint(ctx)
if err != nil {
return "", "", fmt.Errorf("getting load balancer host: %w", err)
return "", "", fmt.Errorf("getting load balancer endpoint: %w", err)
}
return host, strconv.FormatInt(constants.KubernetesPort, 10), nil
}
func (c *Cloud) getLoadBalancerHost(ctx context.Context) (string, error) {
uid, err := c.imds.uid(ctx)
if err != nil {
return "", fmt.Errorf("getting uid: %w", err)
}
uidTag := fmt.Sprintf("constellation-uid-%s", uid)
subnet, err := c.getSubnetCIDR(uidTag)
if err != nil {
return "", err
}
srvs, err := c.getServers(uidTag)
if err != nil {
return "", err
}
for _, s := range srvs {
if s.Name == "" {
continue
}
if s.ID == "" {
continue
}
if s.Tags == nil {
continue
}
subnetAddrs, err := parseSeverAddresses(s.Addresses)
if err != nil {
return "", fmt.Errorf("parsing server %q addresses: %w", s.Name, err)
}
// In a best effort approach, we take the first fixed IPv4 address that is outside the subnet
// belonging to our cluster and assume it is the "load balancer" floating ip.
for _, serverSubnet := range subnetAddrs {
for _, addr := range serverSubnet.Addresses {
if addr.Type != floatingIP {
continue
}
if addr.IPVersion != ipV4 {
continue
}
if addr.IP == "" {
continue
}
parsedAddr, err := netip.ParseAddr(addr.IP)
if err != nil {
continue
}
if subnet.Contains(parsedAddr) {
continue
}
return addr.IP, nil
}
}
}
return "", errors.New("no load balancer endpoint found")
}
func (c *Cloud) getSubnetCIDR(uidTag string) (netip.Prefix, error) {
listNetworksOpts := networks.ListOpts{Tags: uidTag}
networksPage, err := c.api.ListNetworks(listNetworksOpts).AllPages()

View File

@ -465,203 +465,18 @@ func TestInitSecretHash(t *testing.T) {
}
func TestGetLoadBalancerEndpoint(t *testing.T) {
// newTestAddrs returns a set of raw server addresses as we would get from
// a ListServers call and as expected by the parseSeverAddresses function.
// The hardcoded addresses don't match what we are looking for. A valid
// address can be injected. You can pass a second valid address to test
// that the first valid one is chosen.
newTestAddrs := func(floatingIP1, floatingIP2 string, fixedIP1 string) map[string]any {
return map[string]any{
"network1": []any{
map[string]any{
"addr": "192.0.2.2",
"version": 4,
"OS-EXT-IPS:type": "floating",
"OS-EXT-IPS-MAC:mac_addr": "fa:16:3e:0c:0c:0c",
},
},
"network2": []any{
map[string]any{
"addr": fixedIP1,
"version": 4,
"OS-EXT-IPS:type": "fixed",
"OS-EXT-IPS-MAC:mac_addr": "fa:16:3e:0c:0c:0c",
},
map[string]any{
"addr": "2001:db8:3333:4444:5555:6666:7777:8888",
"version": 6,
"OS-EXT-IPS:type": "floating",
"OS-EXT-IPS-MAC:mac_addr": "fa:16:3e:0c:0c:0c",
},
map[string]any{
"addr": floatingIP1,
"version": 4,
"OS-EXT-IPS:type": "floating",
"OS-EXT-IPS-MAC:mac_addr": "fa:16:3e:0c:0c:0c",
},
map[string]any{
"addr": floatingIP2,
"version": 4,
"OS-EXT-IPS:type": "floating",
"OS-EXT-IPS-MAC:mac_addr": "fa:16:3e:0c:0c:0c",
},
},
}
}
testCases := map[string]struct {
imds *stubIMDSClient
api *stubServersClient
wantHost string
wantErr bool
imds *stubIMDSClient
want string
wantErr bool
}{
"error returned from IMDS client": {
imds: &stubIMDSClient{uidErr: errors.New("failed")},
imds: &stubIMDSClient{loadBalancerEndpointErr: errors.New("failed")},
wantErr: true,
},
"error returned from getSubnetCIDR": {
imds: &stubIMDSClient{},
api: &stubServersClient{
netsPager: newNetPager([]networks.Network{{Name: "mynet"}}, nil),
subnetsPager: newSubnetPager(nil, errors.New("failed")),
},
wantErr: true,
},
"error returned from getServers": {
imds: &stubIMDSClient{},
api: &stubServersClient{
netsPager: newNetPager([]networks.Network{{Name: "mynet"}}, nil),
subnetsPager: newSubnetPager([]subnets.Subnet{{Name: "mynet", CIDR: "192.0.2.0/24"}}, nil),
serversPager: newSeverPager(nil, errors.New("failed")),
},
wantErr: true,
},
"sever with empty name skipped": {
imds: &stubIMDSClient{},
api: &stubServersClient{
netsPager: newNetPager([]networks.Network{{Name: "mynet"}}, nil),
subnetsPager: newSubnetPager([]subnets.Subnet{{Name: "mynet", CIDR: "192.0.2.0/24"}}, nil),
serversPager: newSeverPager([]servers.Server{
{
ID: "id1",
Tags: &[]string{"constellation-role-control-plane", "constellation-uid-7777"},
Addresses: newTestAddrs("198.51.100.0", "", "192.0.2.1"),
},
}, nil),
},
wantErr: true,
},
"server with empty ID skipped": {
imds: &stubIMDSClient{},
api: &stubServersClient{
netsPager: newNetPager([]networks.Network{{Name: "mynet"}}, nil),
subnetsPager: newSubnetPager([]subnets.Subnet{{Name: "mynet", CIDR: "192.0.2.0/24"}}, nil),
serversPager: newSeverPager([]servers.Server{
{
Name: "name1",
Tags: &[]string{"constellation-role-control-plane", "constellation-uid-7777"},
Addresses: newTestAddrs("198.51.100.0", "", "192.0.2.1"),
},
}, nil),
},
wantErr: true,
},
"sever with nil tags skipped": {
imds: &stubIMDSClient{},
api: &stubServersClient{
netsPager: newNetPager([]networks.Network{{Name: "mynet"}}, nil),
subnetsPager: newSubnetPager([]subnets.Subnet{{Name: "mynet", CIDR: "192.0.2.0/24"}}, nil),
serversPager: newSeverPager([]servers.Server{
{
Name: "name1",
ID: "id1",
Addresses: newTestAddrs("198.51.100.0", "", "192.0.2.1"),
},
}, nil),
},
wantErr: true,
},
"server has invalid address": {
imds: &stubIMDSClient{},
api: &stubServersClient{
netsPager: newNetPager([]networks.Network{{Name: "mynet"}}, nil),
subnetsPager: newSubnetPager([]subnets.Subnet{{Name: "mynet", CIDR: "192.0.2.0/24"}}, nil),
serversPager: newSeverPager([]servers.Server{
{
Name: "name1",
ID: "id1",
Tags: &[]string{"constellation-role-control-plane", "constellation-uid-7777"},
Addresses: newTestAddrs("", "", "invalidIP"),
},
}, nil),
},
wantErr: true,
},
"server without parseable addresses skipped": {
imds: &stubIMDSClient{},
api: &stubServersClient{
netsPager: newNetPager([]networks.Network{{Name: "mynet"}}, nil),
subnetsPager: newSubnetPager([]subnets.Subnet{{Name: "mynet", CIDR: "192.0.2.0/24"}}, nil),
serversPager: newSeverPager([]servers.Server{
{
Name: "name1",
ID: "id1",
Tags: &[]string{"constellation-role-control-plane", "constellation-uid-7777"},
Addresses: map[string]any{
"somekey": "invalid",
},
},
}, nil),
},
wantErr: true,
},
"invalid endpoint returned from server addresses": {
imds: &stubIMDSClient{},
api: &stubServersClient{
netsPager: newNetPager([]networks.Network{{Name: "mynet"}}, nil),
subnetsPager: newSubnetPager([]subnets.Subnet{{Name: "mynet", CIDR: "192.0.2.0/24"}}, nil),
serversPager: newSeverPager([]servers.Server{
{
Name: "name1",
ID: "id1",
Tags: &[]string{"constellation-role-control-plane", "constellation-uid-7777"},
Addresses: newTestAddrs("invalidIP", "", "192.0.2.1"),
},
}, nil),
},
wantErr: true,
},
"valid endpoint returned from server addresses not in subnet CIDR": {
imds: &stubIMDSClient{},
api: &stubServersClient{
netsPager: newNetPager([]networks.Network{{Name: "mynet"}}, nil),
subnetsPager: newSubnetPager([]subnets.Subnet{{Name: "mynet", CIDR: "192.0.2.0/24"}}, nil),
serversPager: newSeverPager([]servers.Server{
{
Name: "name1",
ID: "id1",
Tags: &[]string{"constellation-role-control-plane", "constellation-uid-7777"},
Addresses: newTestAddrs("198.51.100.0", "", "192.0.2.1"),
},
}, nil),
},
wantHost: "198.51.100.0",
},
"first valid endpoint returned from server addresses not in subnet CIDR": {
imds: &stubIMDSClient{},
api: &stubServersClient{
netsPager: newNetPager([]networks.Network{{Name: "mynet"}}, nil),
subnetsPager: newSubnetPager([]subnets.Subnet{{Name: "mynet", CIDR: "192.0.2.0/24"}}, nil),
serversPager: newSeverPager([]servers.Server{
{
Name: "name1",
ID: "id1",
Tags: &[]string{"constellation-role-control-plane", "constellation-uid-7777"},
Addresses: newTestAddrs("198.51.100.0", "198.51.100.1", "192.0.2.1"),
},
}, nil),
},
wantHost: "198.51.100.0",
"UID returned from IMDS client": {
imds: &stubIMDSClient{loadBalancerEndpointResult: "some.endpoint"},
want: "some.endpoint",
},
}
@ -669,19 +484,15 @@ func TestGetLoadBalancerEndpoint(t *testing.T) {
t.Run(name, func(t *testing.T) {
assert := assert.New(t)
c := &Cloud{
imds: tc.imds,
api: tc.api,
}
c := &Cloud{imds: tc.imds}
gotHost, gotPort, err := c.GetLoadBalancerEndpoint(context.Background())
got, _, err := c.GetLoadBalancerEndpoint(context.Background())
if tc.wantErr {
assert.Error(err)
} else {
assert.NoError(err)
assert.Equal(tc.wantHost, gotHost)
assert.Equal("6443", gotPort)
assert.Equal(tc.want, got)
}
})
}

View File

@ -207,9 +207,13 @@ type OpenStackConfig struct {
// AuthURL is the OpenStack Identity endpoint to use inside the cluster.
AuthURL string `yaml:"authURL" validate:"required"`
// description: |
// ProjectID is the ID of the project where a user resides.
// ProjectID is the ID of the OpenStack project where a user resides.
ProjectID string `yaml:"projectID" validate:"required"`
// description: |
// STACKITProjectID is the ID of the STACKIT project where a user resides.
// Only used if cloud is "stackit".
STACKITProjectID string `yaml:"stackitProjectID"`
// description: |
// ProjectName is the name of the project where a user resides.
ProjectName string `yaml:"projectName" validate:"required"`
// description: |

View File

@ -276,7 +276,7 @@ func init() {
FieldName: "openstack",
},
}
OpenStackConfigDoc.Fields = make([]encoder.Doc, 15)
OpenStackConfigDoc.Fields = make([]encoder.Doc, 16)
OpenStackConfigDoc.Fields[0].Name = "cloud"
OpenStackConfigDoc.Fields[0].Type = "string"
OpenStackConfigDoc.Fields[0].Note = ""
@ -300,58 +300,63 @@ func init() {
OpenStackConfigDoc.Fields[4].Name = "projectID"
OpenStackConfigDoc.Fields[4].Type = "string"
OpenStackConfigDoc.Fields[4].Note = ""
OpenStackConfigDoc.Fields[4].Description = "ProjectID is the ID of the project where a user resides."
OpenStackConfigDoc.Fields[4].Comments[encoder.LineComment] = "ProjectID is the ID of the project where a user resides."
OpenStackConfigDoc.Fields[5].Name = "projectName"
OpenStackConfigDoc.Fields[4].Description = "ProjectID is the ID of the OpenStack project where a user resides."
OpenStackConfigDoc.Fields[4].Comments[encoder.LineComment] = "ProjectID is the ID of the OpenStack project where a user resides."
OpenStackConfigDoc.Fields[5].Name = "stackitProjectID"
OpenStackConfigDoc.Fields[5].Type = "string"
OpenStackConfigDoc.Fields[5].Note = ""
OpenStackConfigDoc.Fields[5].Description = "ProjectName is the name of the project where a user resides."
OpenStackConfigDoc.Fields[5].Comments[encoder.LineComment] = "ProjectName is the name of the project where a user resides."
OpenStackConfigDoc.Fields[6].Name = "userDomainName"
OpenStackConfigDoc.Fields[5].Description = "STACKITProjectID is the ID of the STACKIT project where a user resides.\nOnly used if cloud is \"stackit\"."
OpenStackConfigDoc.Fields[5].Comments[encoder.LineComment] = "STACKITProjectID is the ID of the STACKIT project where a user resides."
OpenStackConfigDoc.Fields[6].Name = "projectName"
OpenStackConfigDoc.Fields[6].Type = "string"
OpenStackConfigDoc.Fields[6].Note = ""
OpenStackConfigDoc.Fields[6].Description = "UserDomainName is the name of the domain where a user resides."
OpenStackConfigDoc.Fields[6].Comments[encoder.LineComment] = "UserDomainName is the name of the domain where a user resides."
OpenStackConfigDoc.Fields[7].Name = "projectDomainName"
OpenStackConfigDoc.Fields[6].Description = "ProjectName is the name of the project where a user resides."
OpenStackConfigDoc.Fields[6].Comments[encoder.LineComment] = "ProjectName is the name of the project where a user resides."
OpenStackConfigDoc.Fields[7].Name = "userDomainName"
OpenStackConfigDoc.Fields[7].Type = "string"
OpenStackConfigDoc.Fields[7].Note = ""
OpenStackConfigDoc.Fields[7].Description = "ProjectDomainName is the name of the domain where a project resides."
OpenStackConfigDoc.Fields[7].Comments[encoder.LineComment] = "ProjectDomainName is the name of the domain where a project resides."
OpenStackConfigDoc.Fields[8].Name = "regionName"
OpenStackConfigDoc.Fields[7].Description = "UserDomainName is the name of the domain where a user resides."
OpenStackConfigDoc.Fields[7].Comments[encoder.LineComment] = "UserDomainName is the name of the domain where a user resides."
OpenStackConfigDoc.Fields[8].Name = "projectDomainName"
OpenStackConfigDoc.Fields[8].Type = "string"
OpenStackConfigDoc.Fields[8].Note = ""
OpenStackConfigDoc.Fields[8].Description = "description: |\nRegionName is the name of the region to use inside the cluster.\n"
OpenStackConfigDoc.Fields[8].Comments[encoder.LineComment] = "description: |"
OpenStackConfigDoc.Fields[9].Name = "username"
OpenStackConfigDoc.Fields[8].Description = "ProjectDomainName is the name of the domain where a project resides."
OpenStackConfigDoc.Fields[8].Comments[encoder.LineComment] = "ProjectDomainName is the name of the domain where a project resides."
OpenStackConfigDoc.Fields[9].Name = "regionName"
OpenStackConfigDoc.Fields[9].Type = "string"
OpenStackConfigDoc.Fields[9].Note = ""
OpenStackConfigDoc.Fields[9].Description = "Username to use inside the cluster."
OpenStackConfigDoc.Fields[9].Comments[encoder.LineComment] = "Username to use inside the cluster."
OpenStackConfigDoc.Fields[10].Name = "password"
OpenStackConfigDoc.Fields[9].Description = "description: |\nRegionName is the name of the region to use inside the cluster.\n"
OpenStackConfigDoc.Fields[9].Comments[encoder.LineComment] = "description: |"
OpenStackConfigDoc.Fields[10].Name = "username"
OpenStackConfigDoc.Fields[10].Type = "string"
OpenStackConfigDoc.Fields[10].Note = ""
OpenStackConfigDoc.Fields[10].Description = "Password to use inside the cluster. You can instead use the environment variable \"CONSTELL_OS_PASSWORD\"."
OpenStackConfigDoc.Fields[10].Comments[encoder.LineComment] = "Password to use inside the cluster. You can instead use the environment variable \"CONSTELL_OS_PASSWORD\"."
OpenStackConfigDoc.Fields[11].Name = "deployYawolLoadBalancer"
OpenStackConfigDoc.Fields[11].Type = "bool"
OpenStackConfigDoc.Fields[10].Description = "Username to use inside the cluster."
OpenStackConfigDoc.Fields[10].Comments[encoder.LineComment] = "Username to use inside the cluster."
OpenStackConfigDoc.Fields[11].Name = "password"
OpenStackConfigDoc.Fields[11].Type = "string"
OpenStackConfigDoc.Fields[11].Note = ""
OpenStackConfigDoc.Fields[11].Description = "Deploy Yawol loadbalancer. For details see: https://github.com/stackitcloud/yawol"
OpenStackConfigDoc.Fields[11].Comments[encoder.LineComment] = "Deploy Yawol loadbalancer. For details see: https://github.com/stackitcloud/yawol"
OpenStackConfigDoc.Fields[12].Name = "yawolImageID"
OpenStackConfigDoc.Fields[12].Type = "string"
OpenStackConfigDoc.Fields[11].Description = "Password to use inside the cluster. You can instead use the environment variable \"CONSTELL_OS_PASSWORD\"."
OpenStackConfigDoc.Fields[11].Comments[encoder.LineComment] = "Password to use inside the cluster. You can instead use the environment variable \"CONSTELL_OS_PASSWORD\"."
OpenStackConfigDoc.Fields[12].Name = "deployYawolLoadBalancer"
OpenStackConfigDoc.Fields[12].Type = "bool"
OpenStackConfigDoc.Fields[12].Note = ""
OpenStackConfigDoc.Fields[12].Description = "OpenStack OS image used by the yawollet. For details see: https://github.com/stackitcloud/yawol"
OpenStackConfigDoc.Fields[12].Comments[encoder.LineComment] = "OpenStack OS image used by the yawollet. For details see: https://github.com/stackitcloud/yawol"
OpenStackConfigDoc.Fields[13].Name = "yawolFlavorID"
OpenStackConfigDoc.Fields[12].Description = "Deploy Yawol loadbalancer. For details see: https://github.com/stackitcloud/yawol"
OpenStackConfigDoc.Fields[12].Comments[encoder.LineComment] = "Deploy Yawol loadbalancer. For details see: https://github.com/stackitcloud/yawol"
OpenStackConfigDoc.Fields[13].Name = "yawolImageID"
OpenStackConfigDoc.Fields[13].Type = "string"
OpenStackConfigDoc.Fields[13].Note = ""
OpenStackConfigDoc.Fields[13].Description = "OpenStack flavor id used for yawollets. For details see: https://github.com/stackitcloud/yawol"
OpenStackConfigDoc.Fields[13].Comments[encoder.LineComment] = "OpenStack flavor id used for yawollets. For details see: https://github.com/stackitcloud/yawol"
OpenStackConfigDoc.Fields[14].Name = "deployCSIDriver"
OpenStackConfigDoc.Fields[14].Type = "bool"
OpenStackConfigDoc.Fields[13].Description = "OpenStack OS image used by the yawollet. For details see: https://github.com/stackitcloud/yawol"
OpenStackConfigDoc.Fields[13].Comments[encoder.LineComment] = "OpenStack OS image used by the yawollet. For details see: https://github.com/stackitcloud/yawol"
OpenStackConfigDoc.Fields[14].Name = "yawolFlavorID"
OpenStackConfigDoc.Fields[14].Type = "string"
OpenStackConfigDoc.Fields[14].Note = ""
OpenStackConfigDoc.Fields[14].Description = "Deploy Cinder CSI driver with on-node encryption. For details see: https://docs.edgeless.systems/constellation/architecture/encrypted-storage"
OpenStackConfigDoc.Fields[14].Comments[encoder.LineComment] = "Deploy Cinder CSI driver with on-node encryption. For details see: https://docs.edgeless.systems/constellation/architecture/encrypted-storage"
OpenStackConfigDoc.Fields[14].Description = "OpenStack flavor id used for yawollets. For details see: https://github.com/stackitcloud/yawol"
OpenStackConfigDoc.Fields[14].Comments[encoder.LineComment] = "OpenStack flavor id used for yawollets. For details see: https://github.com/stackitcloud/yawol"
OpenStackConfigDoc.Fields[15].Name = "deployCSIDriver"
OpenStackConfigDoc.Fields[15].Type = "bool"
OpenStackConfigDoc.Fields[15].Note = ""
OpenStackConfigDoc.Fields[15].Description = "Deploy Cinder CSI driver with on-node encryption. For details see: https://docs.edgeless.systems/constellation/architecture/encrypted-storage"
OpenStackConfigDoc.Fields[15].Comments[encoder.LineComment] = "Deploy Cinder CSI driver with on-node encryption. For details see: https://docs.edgeless.systems/constellation/architecture/encrypted-storage"
QEMUConfigDoc.Type = "QEMUConfig"
QEMUConfigDoc.Comments[encoder.LineComment] = "QEMUConfig holds config information for QEMU based Constellation deployments."

View File

@ -44,7 +44,9 @@ func extraCiliumValues(provider cloudprovider.Provider, conformanceMode bool, ou
}
strictMode := map[string]any{}
if provider != cloudprovider.QEMU {
// TODO(@3u13r): Once we are able to set the subnet of the load balancer VMs
// on STACKIT, we can remove the OpenStack exception here.
if provider != cloudprovider.QEMU && provider != cloudprovider.OpenStack {
strictMode = map[string]any{
"nodeCIDRList": []string{output.IPCidrNode},
}

View File

@ -75,6 +75,8 @@ go_library(
"infrastructure/aws/modules/jump_host/output.tf",
"infrastructure/aws/modules/load_balancer_target/output.tf",
"infrastructure/aws/modules/public_private_subnet/output.tf",
"infrastructure/openstack/modules/stackit_loadbalancer/main.tf",
"infrastructure/openstack/modules/stackit_loadbalancer/variables.tf",
],
importpath = "github.com/edgelesssys/constellation/v2/terraform",
visibility = ["//visibility:public"],

View File

@ -25,6 +25,33 @@ provider "registry.terraform.io/hashicorp/random" {
]
}
provider "registry.terraform.io/stackitcloud/stackit" {
version = "0.12.0"
constraints = "0.12.0"
hashes = [
"h1:08k0ihJixjWGyzNF0wdMiOckr+4qfBi50yj4tTLsbMM=",
"h1:8wtUYCXZke9uJiWp3Y7/tRy84UM0TjOzrzhb6BAX5vo=",
"h1:EwUqtQ7b/ShFcNvBMiemsbrvqBwFfkIRtnEIeIisKSA=",
"h1:lPXt86IQA6bHnX6o6xIaOUHqbAs6WHAehwtS1kK3wcg=",
"h1:t+pHh9fQCS+4Rq9STVs+npH3DOe7qp1L0rJfbMjAdjM=",
"zh:0dde99e7b343fa01f8eefc378171fb8621bedb20f59157d6cc8e3d46c738105f",
"zh:13ff6111adb804e3e7a33a0e8e341e494a84a81115b144c950ea9864ce12efdb",
"zh:2b13aff4a4879b833e27d215102c98809fe78d9a1fb33d09ec352760d21fa7c3",
"zh:6562b6ca55bebd7e425fba60ba5683a3cb00d49d50883e37f418b5be8d52d992",
"zh:6ce745a9a2fac88fd7b219dca1d70882e3c1b573e2d27a49de0a04b76ceabdf0",
"zh:70dd57f2e59596f697aaeab377423a041a57e066d1ad8bbfc0ace9cfaf6e9e0d",
"zh:7bb24a57ef0d802c62d23249078d86a0daeba29b7508d46bb8d104c5b820f35b",
"zh:93b57ec66d0f18ef616416f9d39a5a5b45dde604145b66e5184f00840db7a981",
"zh:9646f12a59a3eab161040eee68093b4c55864c865d544fa83d0e56bfbc59c174",
"zh:c23b3433b81eb99e314239add0df206a5388ef79884e924537bf09d4374815a8",
"zh:d2ef1946a5d559a72dac15a38a78f8d2d09bcd13068d9fe1debe7ae82e9c527d",
"zh:d63299ca4bf158573706a0c313dbee0aa79c7b910d85a0a748ba77620f533a5d",
"zh:e796aec8e1c64c7142d1b2877794ff8cb6fc5699292dfea102f2f229375626a2",
"zh:eb4003be226dc810004cd6a50d98f872d61bb49f2891a2966247a245c9d7cc1c",
"zh:f62e5390fca4d920c3db329276e1780ae57cc20aa666ee549dcf452d4f839ba5",
]
}
provider "registry.terraform.io/terraform-provider-openstack/openstack" {
version = "1.54.1"
constraints = "1.54.1"

View File

@ -5,6 +5,11 @@ terraform {
version = "1.54.1"
}
stackit = {
source = "stackitcloud/stackit"
version = "0.12.0"
}
random = {
source = "hashicorp/random"
version = "3.6.0"
@ -16,6 +21,11 @@ provider "openstack" {
cloud = var.cloud
}
provider "stackit" {
region = "eu01"
}
data "openstack_identity_auth_scope_v3" "scope" {
name = "scope"
}
@ -26,15 +36,17 @@ locals {
init_secret_hash = random_password.init_secret.bcrypt_hash
ports_node_range_start = "30000"
ports_node_range_end = "32767"
ports_kubernetes = "6443"
ports_bootstrapper = "9000"
ports_konnectivity = "8132"
ports_verify = "30081"
ports_recovery = "9999"
ports_debugd = "4000"
cidr_vpc_subnet_nodes = "192.168.178.0/24"
cidr_vpc_subnet_lbs = "192.168.177.0/24"
tags = ["constellation-uid-${local.uid}"]
control_plane_named_ports = flatten([
{ name = "kubernetes", port = "6443", health_check = "HTTPS" },
{ name = "bootstrapper", port = "9000", health_check = "TCP" },
{ name = "verify", port = "30081", health_check = "TCP" },
{ name = "recovery", port = "9999", health_check = "TCP" },
{ name = "join", port = "30090", health_check = "TCP" },
var.debug ? [{ name = "debugd", port = "4000", health_check = "TCP" }] : [],
])
cidr_vpc_subnet_nodes = "192.168.178.0/24"
cidr_vpc_subnet_lbs = "192.168.177.0/24"
tags = ["constellation-uid-${local.uid}"]
identity_service = [
for entry in data.openstack_identity_auth_scope_v3.scope.service_catalog :
entry if entry.type == "identity"
@ -194,46 +206,40 @@ resource "openstack_networking_secgroup_rule_v2" "nodeport_udp" {
security_group_id = openstack_networking_secgroup_v2.vpc_secgroup.id
}
resource "openstack_networking_secgroup_rule_v2" "tcp_port_forward" {
for_each = toset(flatten([
local.ports_kubernetes,
local.ports_bootstrapper,
local.ports_konnectivity,
local.ports_verify,
local.ports_recovery,
var.debug ? [local.ports_debugd] : [],
]))
resource "openstack_networking_secgroup_rule_v2" "tcp_ingress" {
for_each = { for item in local.control_plane_named_ports : item.name => item }
direction = "ingress"
ethertype = "IPv4"
protocol = "tcp"
port_range_min = each.value
port_range_max = each.value
port_range_min = each.value.port
port_range_max = each.value.port
security_group_id = openstack_networking_secgroup_v2.vpc_secgroup.id
}
module "instance_group" {
source = "./modules/instance_group"
for_each = var.node_groups
base_name = local.name
node_group_name = each.key
role = each.value.role
initial_count = each.value.initial_count
disk_size = each.value.state_disk_size
state_disk_type = each.value.state_disk_type
availability_zone = each.value.zone
image_id = var.image_id
flavor_id = each.value.flavor_id
security_groups = [openstack_networking_secgroup_v2.vpc_secgroup.id]
tags = local.tags
uid = local.uid
network_id = openstack_networking_network_v2.vpc_network.id
subnet_id = openstack_networking_subnet_v2.vpc_subnetwork.id
init_secret_hash = local.init_secret_hash
identity_internal_url = local.identity_internal_url
openstack_username = var.openstack_username
openstack_password = var.openstack_password
openstack_user_domain_name = var.openstack_user_domain_name
source = "./modules/instance_group"
for_each = var.node_groups
base_name = local.name
node_group_name = each.key
role = each.value.role
initial_count = each.value.initial_count
disk_size = each.value.state_disk_size
state_disk_type = each.value.state_disk_type
availability_zone = each.value.zone
image_id = var.image_id
flavor_id = each.value.flavor_id
security_groups = [openstack_networking_secgroup_v2.vpc_secgroup.id]
tags = local.tags
uid = local.uid
network_id = openstack_networking_network_v2.vpc_network.id
subnet_id = openstack_networking_subnet_v2.vpc_subnetwork.id
init_secret_hash = local.init_secret_hash
identity_internal_url = local.identity_internal_url
openstack_username = var.openstack_username
openstack_password = var.openstack_password
openstack_user_domain_name = var.openstack_user_domain_name
openstack_load_balancer_endpoint = openstack_networking_floatingip_v2.public_ip.address
}
resource "openstack_networking_floatingip_v2" "public_ip" {
@ -242,8 +248,8 @@ resource "openstack_networking_floatingip_v2" "public_ip" {
tags = local.tags
}
resource "openstack_networking_floatingip_associate_v2" "public_ip_associate" {
count = var.cloud == "stackit" ? 0 : 1
floating_ip = openstack_networking_floatingip_v2.public_ip.address
port_id = module.instance_group["control_plane_default"].port_ids.0
depends_on = [
@ -252,6 +258,20 @@ resource "openstack_networking_floatingip_associate_v2" "public_ip_associate" {
]
}
module "stackit_loadbalancer" {
count = var.cloud == "stackit" ? 1 : 0
source = "./modules/stackit_loadbalancer"
name = local.name
stackit_project_id = var.stackit_project_id
member_ips = module.instance_group["control_plane_default"].ips
network_id = openstack_networking_network_v2.vpc_network.id
external_address = openstack_networking_floatingip_v2.public_ip.address
ports = {
for port in local.control_plane_named_ports : port.name => port.port
}
}
moved {
from = module.instance_group_control_plane
to = module.instance_group["control_plane_default"]

View File

@ -65,13 +65,14 @@ resource "openstack_compute_instance_v2" "instance_group_member" {
delete_on_termination = true
}
metadata = {
constellation-role = var.role
constellation-uid = var.uid
constellation-init-secret-hash = var.init_secret_hash
openstack-auth-url = var.identity_internal_url
openstack-username = var.openstack_username
openstack-password = var.openstack_password
openstack-user-domain-name = var.openstack_user_domain_name
constellation-role = var.role
constellation-uid = var.uid
constellation-init-secret-hash = var.init_secret_hash
openstack-auth-url = var.identity_internal_url
openstack-username = var.openstack_username
openstack-password = var.openstack_password
openstack-user-domain-name = var.openstack_user_domain_name
openstack-load-balancer-endpoint = var.openstack_load_balancer_endpoint
}
availability_zone_hints = var.availability_zone
}

View File

@ -1,5 +1,5 @@
output "ips" {
value = openstack_compute_instance_v2.instance_group_member.*.access_ip_v4
value = [for instance in openstack_compute_instance_v2.instance_group_member : instance.access_ip_v4]
description = "Public IP addresses of the instances."
}

View File

@ -96,3 +96,8 @@ variable "openstack_password" {
type = string
description = "OpenStack password."
}
variable "openstack_load_balancer_endpoint" {
type = string
description = "OpenStack load balancer endpoint."
}

View File

@ -0,0 +1,47 @@
terraform {
required_providers {
stackit = {
source = "stackitcloud/stackit"
version = "0.12.0"
}
}
}
resource "stackit_loadbalancer" "loadbalancer" {
project_id = var.stackit_project_id
name = "${var.name}-lb"
target_pools = [
for portName, port in var.ports : {
name = "target-pool-${portName}"
target_port = port
targets = [
for ip in var.member_ips : {
display_name = "target-${portName}"
ip = ip
}
]
active_health_check = {
healthy_threshold = 10
interval = "3s"
interval_jitter = "3s"
timeout = "3s"
unhealthy_threshold = 10
}
}
]
listeners = [
for portName, port in var.ports : {
name = "listener-${portName}"
port = port
protocol = "PROTOCOL_TCP"
target_pool = "target-pool-${portName}"
}
]
networks = [
{
network_id = var.network_id
role = "ROLE_LISTENERS_AND_TARGETS"
}
]
external_address = var.external_address
}

View File

@ -0,0 +1,30 @@
variable "name" {
type = string
description = "Base name of the load balancer."
}
variable "member_ips" {
type = list(string)
description = "IP addresses of the members of the load balancer pool."
default = []
}
variable "network_id" {
type = string
description = "ID of the network."
}
variable "external_address" {
type = string
description = "External address of the load balancer."
}
variable "ports" {
type = map(number)
description = "Ports to listen on incoming traffic."
}
variable "stackit_project_id" {
type = string
description = "STACKIT project ID."
}

View File

@ -67,3 +67,10 @@ variable "openstack_password" {
type = string
description = "OpenStack password."
}
# STACKIT-specific variables
variable "stackit_project_id" {
type = string
description = "STACKIT project ID."
}