bootstrapper: add fallback endpoint and custom endpoint to SAN field (#2108)

terraform: collect apiserver cert SANs and support custom endpoint

constants: add new constants for cluster configuration and custom endpoint

cloud: support apiserver cert sans and prepare for endpoint migration on AWS

config: add customEndpoint field

bootstrapper: use per-CSP apiserver cert SANs

cli: route customEndpoint to terraform and add migration for apiserver cert SANs

bootstrapper: change interface of GetLoadBalancerEndpoint to return host and port separately

bootstrapper/cmd/bootstrapper/run.go

@@ -99,5 +99,5 @@ type clusterInitJoiner interface {
 type metadataAPI interface {
 	joinclient.MetadataAPI
 	initserver.MetadataAPI
-	GetLoadBalancerEndpoint(ctx context.Context) (string, error)
+	GetLoadBalancerEndpoint(ctx context.Context) (host, port string, err error)
 }

bootstrapper/cmd/bootstrapper/test.go

@@ -22,7 +22,7 @@ type clusterFake struct{}
 // InitCluster fakes bootstrapping a new cluster with the current node being the master, returning the arguments required to join the cluster.
 func (c *clusterFake) InitCluster(
 	context.Context, string, string, string, []byte,
-	[]byte, bool, components.Components, *logger.Logger,
+	[]byte, bool, components.Components, []string, *logger.Logger,
 ) ([]byte, error) {
 	return []byte{}, nil
 }
@@ -53,8 +53,8 @@ func (f *providerMetadataFake) Self(_ context.Context) (metadata.InstanceMetadat
 	}, nil
 }
 
-func (f *providerMetadataFake) GetLoadBalancerEndpoint(_ context.Context) (string, error) {
+func (f *providerMetadataFake) GetLoadBalancerEndpoint(_ context.Context) (string, string, error) {
-	return "", nil
+	return "", "", nil
 }
 
 func (f *providerMetadataFake) InitSecretHash(_ context.Context) ([]byte, error) {

bootstrapper/initproto/init.pb.go

@@ -38,6 +38,7 @@ type InitRequest struct {
 	KubernetesComponents   []*KubernetesComponent `protobuf:"bytes,15,rep,name=kubernetes_components,json=kubernetesComponents,proto3" json:"kubernetes_components,omitempty"`
 	InitSecret             []byte                 `protobuf:"bytes,16,opt,name=init_secret,json=initSecret,proto3" json:"init_secret,omitempty"`
 	ClusterName            string                 `protobuf:"bytes,17,opt,name=cluster_name,json=clusterName,proto3" json:"cluster_name,omitempty"`
+	ApiserverCertSans      []string               `protobuf:"bytes,18,rep,name=apiserver_cert_sans,json=apiserverCertSans,proto3" json:"apiserver_cert_sans,omitempty"`
 }
 
 func (x *InitRequest) Reset() {
@@ -135,6 +136,13 @@ func (x *InitRequest) GetClusterName() string {
 	return ""
 }
 
+func (x *InitRequest) GetApiserverCertSans() []string {
+	if x != nil {
+		return x.ApiserverCertSans
+	}
+	return nil
+}
+
 type InitResponse struct {
 	state         protoimpl.MessageState
 	sizeCache     protoimpl.SizeCache
@@ -463,7 +471,7 @@ var File_bootstrapper_initproto_init_proto protoreflect.FileDescriptor
 var file_bootstrapper_initproto_init_proto_rawDesc = []byte{
 	0x0a, 0x21, 0x62, 0x6f, 0x6f, 0x74, 0x73, 0x74, 0x72, 0x61, 0x70, 0x70, 0x65, 0x72, 0x2f, 0x69,
 	0x6e, 0x69, 0x74, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x69, 0x6e, 0x69, 0x74, 0x2e, 0x70, 0x72,
-	0x6f, 0x74, 0x6f, 0x12, 0x04, 0x69, 0x6e, 0x69, 0x74, 0x22, 0x9b, 0x03, 0x0a, 0x0b, 0x49, 0x6e,
+	0x6f, 0x74, 0x6f, 0x12, 0x04, 0x69, 0x6e, 0x69, 0x74, 0x22, 0xcb, 0x03, 0x0a, 0x0b, 0x49, 0x6e,
 	0x69, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x17, 0x0a, 0x07, 0x6b, 0x6d, 0x73,
 	0x5f, 0x75, 0x72, 0x69, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x6b, 0x6d, 0x73, 0x55,
 	0x72, 0x69, 0x12, 0x1f, 0x0a, 0x0b, 0x73, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x5f, 0x75, 0x72,
@@ -489,7 +497,10 @@ var file_bootstrapper_initproto_init_proto_rawDesc = []byte{
 	0x65, 0x74, 0x18, 0x10, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0a, 0x69, 0x6e, 0x69, 0x74, 0x53, 0x65,
 	0x63, 0x72, 0x65, 0x74, 0x12, 0x21, 0x0a, 0x0c, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x5f,
 	0x6e, 0x61, 0x6d, 0x65, 0x18, 0x11, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x63, 0x6c, 0x75, 0x73,
-	0x74, 0x65, 0x72, 0x4e, 0x61, 0x6d, 0x65, 0x22, 0xc1, 0x01, 0x0a, 0x0c, 0x49, 0x6e, 0x69, 0x74,
+	0x74, 0x65, 0x72, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x2e, 0x0a, 0x13, 0x61, 0x70, 0x69, 0x73, 0x65,
+	0x72, 0x76, 0x65, 0x72, 0x5f, 0x63, 0x65, 0x72, 0x74, 0x5f, 0x73, 0x61, 0x6e, 0x73, 0x18, 0x12,
+	0x20, 0x03, 0x28, 0x09, 0x52, 0x11, 0x61, 0x70, 0x69, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x43,
+	0x65, 0x72, 0x74, 0x53, 0x61, 0x6e, 0x73, 0x22, 0xc1, 0x01, 0x0a, 0x0c, 0x49, 0x6e, 0x69, 0x74,
 	0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x3e, 0x0a, 0x0c, 0x69, 0x6e, 0x69, 0x74,
 	0x5f, 0x73, 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x19,
 	0x2e, 0x69, 0x6e, 0x69, 0x74, 0x2e, 0x49, 0x6e, 0x69, 0x74, 0x53, 0x75, 0x63, 0x63, 0x65, 0x73,

bootstrapper/initproto/init.proto

@@ -26,6 +26,7 @@ message InitRequest {
   repeated KubernetesComponent kubernetes_components = 15;
   bytes init_secret = 16;
   string cluster_name = 17;
+  repeated string apiserver_cert_sans = 18;
 }
 
 message InitResponse {

bootstrapper/internal/helm/helm.go

@@ -12,9 +12,7 @@ import (
 	"context"
 	"errors"
 	"fmt"
-	"net"
 	"os/exec"
-	"strconv"
 	"strings"
 	"time"
 
@@ -109,9 +107,9 @@ func (h *Client) InstallCilium(ctx context.Context, kubectl k8sapi.Client, relea
 
 	switch in.CloudProvider {
 	case "aws", "azure", "openstack", "qemu":
-		return h.installCiliumGeneric(ctx, release, in.LoadBalancerEndpoint)
+		return h.installCiliumGeneric(ctx, release, in.LoadBalancerHost, in.LoadBalancerPort)
 	case "gcp":
-		return h.installCiliumGCP(ctx, release, in.NodeName, in.FirstNodePodCIDR, in.SubnetworkPodCIDR, in.LoadBalancerEndpoint)
+		return h.installCiliumGCP(ctx, release, in.NodeName, in.FirstNodePodCIDR, in.SubnetworkPodCIDR, in.LoadBalancerHost, in.LoadBalancerPort)
 	default:
 		return fmt.Errorf("unsupported cloud provider %q", in.CloudProvider)
 	}
@@ -120,33 +118,25 @@ func (h *Client) InstallCilium(ctx context.Context, kubectl k8sapi.Client, relea
 // installCiliumGeneric installs cilium with the given load balancer endpoint.
 // This is used for cloud providers that do not require special server-side configuration.
 // Currently this is AWS, Azure, and QEMU.
-func (h *Client) installCiliumGeneric(ctx context.Context, release helm.Release, kubeAPIEndpoint string) error {
+func (h *Client) installCiliumGeneric(ctx context.Context, release helm.Release, kubeAPIHost, kubeAPIPort string) error {
-	host := kubeAPIEndpoint
+	release.Values["k8sServiceHost"] = kubeAPIHost
-	release.Values["k8sServiceHost"] = host
+	release.Values["k8sServicePort"] = kubeAPIPort
-	release.Values["k8sServicePort"] = strconv.Itoa(constants.KubernetesPort)
 
 	return h.install(ctx, release.Chart, release.Values)
 }
 
-func (h *Client) installCiliumGCP(ctx context.Context, release helm.Release, nodeName, nodePodCIDR, subnetworkPodCIDR, kubeAPIEndpoint string) error {
+func (h *Client) installCiliumGCP(ctx context.Context, release helm.Release, nodeName, nodePodCIDR, subnetworkPodCIDR, kubeAPIHost, kubeAPIPort string) error {
 	out, err := exec.CommandContext(ctx, constants.KubectlPath, "--kubeconfig", constants.ControlPlaneAdminConfFilename, "patch", "node", nodeName, "-p", "{\"spec\":{\"podCIDR\": \""+nodePodCIDR+"\"}}").CombinedOutput()
 	if err != nil {
 		err = errors.New(string(out))
 		return err
 	}
 
-	host, port, err := net.SplitHostPort(kubeAPIEndpoint)
-	if err != nil {
-		return err
-	}
-
 	// configure pod network CIDR
 	release.Values["ipv4NativeRoutingCIDR"] = subnetworkPodCIDR
 	release.Values["strictModeCIDR"] = subnetworkPodCIDR
-	release.Values["k8sServiceHost"] = host
+	release.Values["k8sServiceHost"] = kubeAPIHost
-	if port != "" {
+	release.Values["k8sServicePort"] = kubeAPIPort
-		release.Values["k8sServicePort"] = port
-	}
 
 	return h.install(ctx, release.Chart, release.Values)
 }

bootstrapper/internal/initserver/initserver.go

@@ -220,6 +220,7 @@ func (s *Server) Init(req *initproto.InitRequest, stream initproto.API_InitServe
 		req.HelmDeployments,
 		req.ConformanceMode,
 		components.NewComponentsFromInitProto(req.KubernetesComponents),
+		req.ApiserverCertSans,
 		s.log,
 	)
 	if err != nil {
@@ -348,6 +349,7 @@ type ClusterInitializer interface {
 		helmDeployments []byte,
 		conformanceMode bool,
 		kubernetesComponents components.Components,
+		apiServerCertSANs []string,
 		log *logger.Logger,
 	) ([]byte, error)
 }

bootstrapper/internal/initserver/initserver_test.go

@@ -407,7 +407,7 @@ type stubClusterInitializer struct {
 
 func (i *stubClusterInitializer) InitCluster(
 	context.Context, string, string, string, []byte,
-	[]byte, bool, components.Components, *logger.Logger,
+	[]byte, bool, components.Components, []string, *logger.Logger,
 ) ([]byte, error) {
 	return i.initClusterKubeconfig, i.initClusterErr
 }

bootstrapper/internal/kubernetes/cloud_provider.go

@@ -19,12 +19,12 @@ type ProviderMetadata interface {
 	// Self retrieves the current instance.
 	Self(ctx context.Context) (metadata.InstanceMetadata, error)
 	// GetLoadBalancerEndpoint retrieves the load balancer endpoint.
-	GetLoadBalancerEndpoint(ctx context.Context) (string, error)
+	GetLoadBalancerEndpoint(ctx context.Context) (host, port string, err error)
 }
 
 type stubProviderMetadata struct {
 	getLoadBalancerEndpointErr                       error
-	getLoadBalancerEndpointResp string
+	getLoadBalancerHostResp, getLoadBalancerPortResp string
 
 	selfErr  error
 	selfResp metadata.InstanceMetadata
@@ -33,8 +33,8 @@ type stubProviderMetadata struct {
 	uidResp string
 }
 
-func (m *stubProviderMetadata) GetLoadBalancerEndpoint(_ context.Context) (string, error) {
+func (m *stubProviderMetadata) GetLoadBalancerEndpoint(_ context.Context) (string, string, error) {
-	return m.getLoadBalancerEndpointResp, m.getLoadBalancerEndpointErr
+	return m.getLoadBalancerHostResp, m.getLoadBalancerPortResp, m.getLoadBalancerEndpointErr
 }
 
 func (m *stubProviderMetadata) Self(_ context.Context) (metadata.InstanceMetadata, error) {

bootstrapper/internal/kubernetes/k8sapi/k8sutil.go

@@ -91,7 +91,7 @@ func (k *KubernetesUtil) InstallComponents(ctx context.Context, kubernetesCompon
 // InitCluster instruments kubeadm to initialize the K8s cluster.
 // On success an admin kubeconfig file is returned.
 func (k *KubernetesUtil) InitCluster(
-	ctx context.Context, initConfig []byte, nodeName, clusterName string, ips []net.IP, controlPlaneEndpoint string, conformanceMode bool, log *logger.Logger,
+	ctx context.Context, initConfig []byte, nodeName, clusterName string, ips []net.IP, controlPlaneHost, controlPlanePort string, conformanceMode bool, log *logger.Logger,
 ) ([]byte, error) {
 	// TODO(3u13r): audit policy should be user input
 	auditPolicy, err := resources.NewDefaultAuditPolicy().Marshal()
@@ -148,6 +148,7 @@ func (k *KubernetesUtil) InitCluster(
 	}
 
 	log.Infof("Preparing node for Konnectivity")
+	controlPlaneEndpoint := net.JoinHostPort(controlPlaneHost, controlPlanePort)
 	if err := k.prepareControlPlaneForKonnectivity(ctx, controlPlaneEndpoint); err != nil {
 		return nil, fmt.Errorf("setup konnectivity: %w", err)
 	}
@@ -245,7 +246,8 @@ type SetupPodNetworkInput struct {
 	NodeName          string
 	FirstNodePodCIDR  string
 	SubnetworkPodCIDR string
-	LoadBalancerEndpoint string
+	LoadBalancerHost  string
+	LoadBalancerPort  string
 }
 
 // WaitForCilium waits until Cilium reports a healthy status over its /healthz endpoint.
@@ -314,7 +316,7 @@ func (k *KubernetesUtil) FixCilium(ctx context.Context) error {
 }
 
 // JoinCluster joins existing Kubernetes cluster using kubeadm join.
-func (k *KubernetesUtil) JoinCluster(ctx context.Context, joinConfig []byte, peerRole role.Role, controlPlaneEndpoint string, log *logger.Logger) error {
+func (k *KubernetesUtil) JoinCluster(ctx context.Context, joinConfig []byte, peerRole role.Role, controlPlaneHost, controlPlanePort string, log *logger.Logger) error {
 	// TODO(3u13r): audit policy should be user input
 	auditPolicy, err := resources.NewDefaultAuditPolicy().Marshal()
 	if err != nil {
@@ -341,6 +343,7 @@ func (k *KubernetesUtil) JoinCluster(ctx context.Context, joinConfig []byte, pee
 
 	if peerRole == role.ControlPlane {
 		log.Infof("Prep Init Kubernetes cluster")
+		controlPlaneEndpoint := net.JoinHostPort(controlPlaneHost, controlPlanePort)
 		if err := k.prepareControlPlaneForKonnectivity(ctx, controlPlaneEndpoint); err != nil {
 			return fmt.Errorf("setup konnectivity: %w", err)
 		}

bootstrapper/internal/kubernetes/k8sutil.go

@@ -19,8 +19,8 @@ import (
 
 type clusterUtil interface {
 	InstallComponents(ctx context.Context, kubernetesComponents components.Components) error
-	InitCluster(ctx context.Context, initConfig []byte, nodeName, clusterName string, ips []net.IP, controlPlaneEndpoint string, conformanceMode bool, log *logger.Logger) ([]byte, error)
+	InitCluster(ctx context.Context, initConfig []byte, nodeName, clusterName string, ips []net.IP, controlPlaneHost, controlPlanePort string, conformanceMode bool, log *logger.Logger) ([]byte, error)
-	JoinCluster(ctx context.Context, joinConfig []byte, peerRole role.Role, controlPlaneEndpoint string, log *logger.Logger) error
+	JoinCluster(ctx context.Context, joinConfig []byte, peerRole role.Role, controlPlaneHost, controlPlanePort string, log *logger.Logger) error
 	WaitForCilium(ctx context.Context, log *logger.Logger) error
 	FixCilium(ctx context.Context) error
 	StartKubelet() error

bootstrapper/internal/kubernetes/kubernetes.go

@@ -79,7 +79,7 @@ func New(cloudProvider string, clusterUtil clusterUtil, configProvider configura
 // InitCluster initializes a new Kubernetes cluster and applies pod network provider.
 func (k *KubeWrapper) InitCluster(
 	ctx context.Context, cloudServiceAccountURI, versionString, clusterName string, measurementSalt []byte,
-	helmReleasesRaw []byte, conformanceMode bool, kubernetesComponents components.Components, log *logger.Logger,
+	helmReleasesRaw []byte, conformanceMode bool, kubernetesComponents components.Components, apiServerCertSANs []string, log *logger.Logger,
 ) ([]byte, error) {
 	log.With(zap.String("version", versionString)).Infof("Installing Kubernetes components")
 	if err := k.clusterUtil.InstallComponents(ctx, kubernetesComponents); err != nil {
@@ -110,16 +110,24 @@ func (k *KubeWrapper) InitCluster(
 	}
 
 	// this is the endpoint in "kubeadm init --control-plane-endpoint=<IP/DNS>:<port>"
-	controlPlaneEndpoint, err := k.providerMetadata.GetLoadBalancerEndpoint(ctx)
+	// TODO(malt3): switch over to DNS name on AWS and Azure
+	// soon as every apiserver certificate of every control-plane node
+	// has the dns endpoint in its SAN list.
+	controlPlaneHost, controlPlanePort, err := k.providerMetadata.GetLoadBalancerEndpoint(ctx)
 	if err != nil {
 		return nil, fmt.Errorf("retrieving load balancer endpoint: %w", err)
 	}
 
+	certSANs := []string{nodeIP}
+	certSANs = append(certSANs, apiServerCertSANs...)
+
 	log.With(
 		zap.String("nodeName", nodeName),
 		zap.String("providerID", instance.ProviderID),
 		zap.String("nodeIP", nodeIP),
-		zap.String("controlPlaneEndpoint", controlPlaneEndpoint),
+		zap.String("controlPlaneHost", controlPlaneHost),
+		zap.String("controlPlanePort", controlPlanePort),
+		zap.String("certSANs", strings.Join(certSANs, ",")),
 		zap.String("podCIDR", subnetworkPodCIDR),
 	).Infof("Setting information for node")
 
@@ -130,16 +138,16 @@ func (k *KubeWrapper) InitCluster(
 	initConfig := k.configProvider.InitConfiguration(ccmSupported, versionString)
 	initConfig.SetNodeIP(nodeIP)
 	initConfig.SetClusterName(clusterName)
-	initConfig.SetCertSANs([]string{nodeIP})
+	initConfig.SetCertSANs(certSANs)
 	initConfig.SetNodeName(nodeName)
 	initConfig.SetProviderID(instance.ProviderID)
-	initConfig.SetControlPlaneEndpoint(controlPlaneEndpoint)
+	initConfig.SetControlPlaneEndpoint(controlPlaneHost)
 	initConfigYAML, err := initConfig.Marshal()
 	if err != nil {
 		return nil, fmt.Errorf("encoding kubeadm init configuration as YAML: %w", err)
 	}
 	log.Infof("Initializing Kubernetes cluster")
-	kubeConfig, err := k.clusterUtil.InitCluster(ctx, initConfigYAML, nodeName, clusterName, validIPs, controlPlaneEndpoint, conformanceMode, log)
+	kubeConfig, err := k.clusterUtil.InitCluster(ctx, initConfigYAML, nodeName, clusterName, validIPs, controlPlaneHost, controlPlanePort, conformanceMode, log)
 	if err != nil {
 		return nil, fmt.Errorf("kubeadm init: %w", err)
 	}
@@ -175,7 +183,8 @@ func (k *KubeWrapper) InitCluster(
 		NodeName:          nodeName,
 		FirstNodePodCIDR:  nodePodCIDR,
 		SubnetworkPodCIDR: subnetworkPodCIDR,
-		LoadBalancerEndpoint: controlPlaneEndpoint,
+		LoadBalancerHost:  controlPlaneHost,
+		LoadBalancerPort:  controlPlanePort,
 	}
 
 	var helmReleases helm.Releases
@@ -206,20 +215,11 @@ func (k *KubeWrapper) InitCluster(
 		// Continue and don't throw an error here - things might be okay.
 	}
 
-	var controlPlaneIP string
-	if strings.Contains(controlPlaneEndpoint, ":") {
-		controlPlaneIP, _, err = net.SplitHostPort(controlPlaneEndpoint)
-		if err != nil {
-			return nil, fmt.Errorf("parsing control plane endpoint: %w", err)
-		}
-	} else {
-		controlPlaneIP = controlPlaneEndpoint
-	}
 	serviceConfig := constellationServicesConfig{
 		measurementSalt:        measurementSalt,
 		subnetworkPodCIDR:      subnetworkPodCIDR,
 		cloudServiceAccountURI: cloudServiceAccountURI,
-		loadBalancerIP:         controlPlaneIP,
+		loadBalancerIP:         controlPlaneHost,
 	}
 	extraVals, err := k.setupExtraVals(ctx, serviceConfig)
 	if err != nil {
@@ -300,7 +300,7 @@ func (k *KubeWrapper) JoinCluster(ctx context.Context, args *kubeadm.BootstrapTo
 		return fmt.Errorf("generating node name: %w", err)
 	}
 
-	loadbalancerEndpoint, err := k.providerMetadata.GetLoadBalancerEndpoint(ctx)
+	loadBalancerHost, loadBalancerPort, err := k.providerMetadata.GetLoadBalancerEndpoint(ctx)
 	if err != nil {
 		return fmt.Errorf("retrieving own instance metadata: %w", err)
 	}
@@ -309,6 +309,8 @@ func (k *KubeWrapper) JoinCluster(ctx context.Context, args *kubeadm.BootstrapTo
 		zap.String("nodeName", nodeName),
 		zap.String("providerID", providerID),
 		zap.String("nodeIP", nodeInternalIP),
+		zap.String("loadBalancerHost", loadBalancerHost),
+		zap.String("loadBalancerPort", loadBalancerPort),
 	).Infof("Setting information for node")
 
 	// Step 2: configure kubeadm join config
@@ -329,7 +331,7 @@ func (k *KubeWrapper) JoinCluster(ctx context.Context, args *kubeadm.BootstrapTo
 		return fmt.Errorf("encoding kubeadm join configuration as YAML: %w", err)
 	}
 	log.With(zap.String("apiServerEndpoint", args.APIServerEndpoint)).Infof("Joining Kubernetes cluster")
-	if err := k.clusterUtil.JoinCluster(ctx, joinConfigYAML, peerRole, loadbalancerEndpoint, log); err != nil {
+	if err := k.clusterUtil.JoinCluster(ctx, joinConfigYAML, peerRole, loadBalancerHost, loadBalancerPort, log); err != nil {
 		return fmt.Errorf("joining cluster: %v; %w ", string(joinConfigYAML), err)
 	}
 

bootstrapper/internal/kubernetes/kubernetes_test.go

@@ -63,7 +63,8 @@ func TestInitCluster(t *testing.T) {
 					VPCIP:         privateIP,
 					AliasIPRanges: []string{aliasIPRange},
 				},
-				getLoadBalancerEndpointResp: loadbalancerIP,
+				getLoadBalancerHostResp: loadbalancerIP,
+				getLoadBalancerPortResp: strconv.Itoa(constants.KubernetesPort),
 			},
 			wantConfig: k8sapi.KubeadmInitYAML{
 				InitConfiguration: kubeadm.InitConfiguration{
@@ -96,7 +97,8 @@ func TestInitCluster(t *testing.T) {
 					VPCIP:         privateIP,
 					AliasIPRanges: []string{aliasIPRange},
 				},
-				getLoadBalancerEndpointResp: loadbalancerIP,
+				getLoadBalancerHostResp: loadbalancerIP,
+				getLoadBalancerPortResp: strconv.Itoa(constants.KubernetesPort),
 			},
 			kubectl:    stubKubectl{annotateNodeErr: assert.AnError},
 			wantErr:    true,
@@ -191,7 +193,7 @@ func TestInitCluster(t *testing.T) {
 
 			_, err := kube.InitCluster(
 				context.Background(), serviceAccountURI, string(tc.k8sVersion), "kubernetes",
-				nil, []byte("{}"), false, nil, logger.NewTest(t),
+				nil, []byte("{}"), false, nil, nil, logger.NewTest(t),
 			)
 
 			if tc.wantErr {
@@ -449,7 +451,7 @@ func (s *stubClusterUtil) InstallComponents(_ context.Context, _ components.Comp
 	return s.installComponentsErr
 }
 
-func (s *stubClusterUtil) InitCluster(_ context.Context, initConfig []byte, _, _ string, _ []net.IP, _ string, _ bool, _ *logger.Logger) ([]byte, error) {
+func (s *stubClusterUtil) InitCluster(_ context.Context, initConfig []byte, _, _ string, _ []net.IP, _, _ string, _ bool, _ *logger.Logger) ([]byte, error) {
 	s.initConfigs = append(s.initConfigs, initConfig)
 	return s.kubeconfig, s.initClusterErr
 }
@@ -474,7 +476,7 @@ func (s *stubClusterUtil) SetupNodeOperator(_ context.Context, _ k8sapi.Client,
 	return s.setupNodeOperatorErr
 }
 
-func (s *stubClusterUtil) JoinCluster(_ context.Context, joinConfig []byte, _ role.Role, _ string, _ *logger.Logger) error {
+func (s *stubClusterUtil) JoinCluster(_ context.Context, joinConfig []byte, _ role.Role, _, _ string, _ *logger.Logger) error {
 	s.joinConfigs = append(s.joinConfigs, joinConfig)
 	return s.joinClusterErr
 }

cli/internal/cloudcmd/clients.go

@@ -26,7 +26,7 @@ type imageFetcher interface {
 
 type terraformClient interface {
 	PrepareWorkspace(path string, input terraform.Variables) error
-	CreateCluster(ctx context.Context, logLevel terraform.LogLevel) (terraform.CreateOutput, error)
+	CreateCluster(ctx context.Context, logLevel terraform.LogLevel) (terraform.ApplyOutput, error)
 	CreateIAMConfig(ctx context.Context, provider cloudprovider.Provider, logLevel terraform.LogLevel) (terraform.IAMOutput, error)
 	Destroy(ctx context.Context, logLevel terraform.LogLevel) error
 	CleanUpWorkspace() error

cli/internal/cloudcmd/clients_test.go

@@ -45,8 +45,8 @@ type stubTerraformClient struct {
 	showErr                error
 }
 
-func (c *stubTerraformClient) CreateCluster(_ context.Context, _ terraform.LogLevel) (terraform.CreateOutput, error) {
+func (c *stubTerraformClient) CreateCluster(_ context.Context, _ terraform.LogLevel) (terraform.ApplyOutput, error) {
-	return terraform.CreateOutput{
+	return terraform.ApplyOutput{
 		IP:             c.ip,
 		Secret:         c.initSecret,
 		UID:            c.uid,

cli/internal/cloudcmd/create.go

@@ -83,7 +83,7 @@ func (c *Creator) Create(ctx context.Context, opts CreateOptions) (clusterid.Fil
 	}
 	defer cl.RemoveInstaller()
 
-	var tfOutput terraform.CreateOutput
+	var tfOutput terraform.ApplyOutput
 	switch opts.Provider {
 	case cloudprovider.AWS:
 
@@ -119,46 +119,47 @@ func (c *Creator) Create(ctx context.Context, opts CreateOptions) (clusterid.Fil
 	return clusterid.File{
 		CloudProvider:     opts.Provider,
 		IP:                tfOutput.IP,
+		APIServerCertSANs: tfOutput.APIServerCertSANs,
 		InitSecret:        []byte(tfOutput.Secret),
 		UID:               tfOutput.UID,
 		AttestationURL:    tfOutput.AttestationURL,
 	}, nil
 }
 
-func (c *Creator) createAWS(ctx context.Context, cl terraformClient, opts CreateOptions) (tfOutput terraform.CreateOutput, retErr error) {
+func (c *Creator) createAWS(ctx context.Context, cl terraformClient, opts CreateOptions) (tfOutput terraform.ApplyOutput, retErr error) {
 	vars := awsTerraformVars(opts.Config, opts.image, &opts.ControlPlaneCount, &opts.WorkerCount)
 
 	tfOutput, err := runTerraformCreate(ctx, cl, cloudprovider.AWS, vars, c.out, opts.TFLogLevel)
 	if err != nil {
-		return terraform.CreateOutput{}, err
+		return terraform.ApplyOutput{}, err
 	}
 
 	return tfOutput, nil
 }
 
-func (c *Creator) createGCP(ctx context.Context, cl terraformClient, opts CreateOptions) (tfOutput terraform.CreateOutput, retErr error) {
+func (c *Creator) createGCP(ctx context.Context, cl terraformClient, opts CreateOptions) (tfOutput terraform.ApplyOutput, retErr error) {
 	vars := gcpTerraformVars(opts.Config, opts.image, &opts.ControlPlaneCount, &opts.WorkerCount)
 
 	tfOutput, err := runTerraformCreate(ctx, cl, cloudprovider.GCP, vars, c.out, opts.TFLogLevel)
 	if err != nil {
-		return terraform.CreateOutput{}, err
+		return terraform.ApplyOutput{}, err
 	}
 
 	return tfOutput, nil
 }
 
-func (c *Creator) createAzure(ctx context.Context, cl terraformClient, opts CreateOptions) (tfOutput terraform.CreateOutput, retErr error) {
+func (c *Creator) createAzure(ctx context.Context, cl terraformClient, opts CreateOptions) (tfOutput terraform.ApplyOutput, retErr error) {
 	vars := azureTerraformVars(opts.Config, opts.image, &opts.ControlPlaneCount, &opts.WorkerCount)
 
 	tfOutput, err := runTerraformCreate(ctx, cl, cloudprovider.Azure, vars, c.out, opts.TFLogLevel)
 	if err != nil {
-		return terraform.CreateOutput{}, err
+		return terraform.ApplyOutput{}, err
 	}
 
 	if vars.GetCreateMAA() {
 		// Patch the attestation policy to allow the cluster to boot while having secure boot disabled.
 		if err := c.policyPatcher.Patch(ctx, tfOutput.AttestationURL); err != nil {
-			return terraform.CreateOutput{}, err
+			return terraform.ApplyOutput{}, err
 		}
 	}
 
@@ -196,13 +197,13 @@ func normalizeAzureURIs(vars *terraform.AzureClusterVariables) *terraform.AzureC
 	return vars
 }
 
-func (c *Creator) createOpenStack(ctx context.Context, cl terraformClient, opts CreateOptions) (tfOutput terraform.CreateOutput, retErr error) {
+func (c *Creator) createOpenStack(ctx context.Context, cl terraformClient, opts CreateOptions) (tfOutput terraform.ApplyOutput, retErr error) {
 	// TODO(malt3): Remove this once OpenStack is supported.
 	if os.Getenv("CONSTELLATION_OPENSTACK_DEV") != "1" {
-		return terraform.CreateOutput{}, errors.New("OpenStack isn't supported yet")
+		return terraform.ApplyOutput{}, errors.New("OpenStack isn't supported yet")
 	}
 	if _, hasOSAuthURL := os.LookupEnv("OS_AUTH_URL"); !hasOSAuthURL && opts.Config.Provider.OpenStack.Cloud == "" {
-		return terraform.CreateOutput{}, errors.New(
+		return terraform.ApplyOutput{}, errors.New(
 			"neither environment variable OS_AUTH_URL nor cloud name for \"clouds.yaml\" is set. OpenStack authentication requires a set of " +
 				"OS_* environment variables that are typically sourced into the current shell with an openrc file " +
 				"or a cloud name for \"clouds.yaml\". " +
@@ -214,21 +215,21 @@ func (c *Creator) createOpenStack(ctx context.Context, cl terraformClient, opts
 
 	tfOutput, err := runTerraformCreate(ctx, cl, cloudprovider.OpenStack, vars, c.out, opts.TFLogLevel)
 	if err != nil {
-		return terraform.CreateOutput{}, err
+		return terraform.ApplyOutput{}, err
 	}
 
 	return tfOutput, nil
 }
 
-func runTerraformCreate(ctx context.Context, cl terraformClient, provider cloudprovider.Provider, vars terraform.Variables, outWriter io.Writer, loglevel terraform.LogLevel) (output terraform.CreateOutput, retErr error) {
+func runTerraformCreate(ctx context.Context, cl terraformClient, provider cloudprovider.Provider, vars terraform.Variables, outWriter io.Writer, loglevel terraform.LogLevel) (output terraform.ApplyOutput, retErr error) {
 	if err := cl.PrepareWorkspace(path.Join("terraform", strings.ToLower(provider.String())), vars); err != nil {
-		return terraform.CreateOutput{}, err
+		return terraform.ApplyOutput{}, err
 	}
 
 	defer rollbackOnError(outWriter, &retErr, &rollbackerTerraform{client: cl}, loglevel)
 	tfOutput, err := cl.CreateCluster(ctx, loglevel)
 	if err != nil {
-		return terraform.CreateOutput{}, err
+		return terraform.ApplyOutput{}, err
 	}
 
 	return tfOutput, nil
@@ -239,7 +240,7 @@ type qemuCreateOptions struct {
 	CreateOptions
 }
 
-func (c *Creator) createQEMU(ctx context.Context, cl terraformClient, lv libvirtRunner, opts qemuCreateOptions) (tfOutput terraform.CreateOutput, retErr error) {
+func (c *Creator) createQEMU(ctx context.Context, cl terraformClient, lv libvirtRunner, opts qemuCreateOptions) (tfOutput terraform.ApplyOutput, retErr error) {
 	qemuRollbacker := &rollbackerQEMU{client: cl, libvirt: lv, createdWorkspace: false}
 	defer rollbackOnError(c.out, &retErr, qemuRollbacker, opts.TFLogLevel)
 
@@ -247,7 +248,7 @@ func (c *Creator) createQEMU(ctx context.Context, cl terraformClient, lv libvirt
 	downloader := c.newRawDownloader()
 	imagePath, err := downloader.Download(ctx, c.out, false, opts.source, opts.Config.Image)
 	if err != nil {
-		return terraform.CreateOutput{}, fmt.Errorf("download raw image: %w", err)
+		return terraform.ApplyOutput{}, fmt.Errorf("download raw image: %w", err)
 	}
 
 	libvirtURI := opts.Config.Provider.QEMU.LibvirtURI
@@ -257,7 +258,7 @@ func (c *Creator) createQEMU(ctx context.Context, cl terraformClient, lv libvirt
 	// if no libvirt URI is specified, start a libvirt container
 	case libvirtURI == "":
 		if err := lv.Start(ctx, opts.Config.Name, opts.Config.Provider.QEMU.LibvirtContainerImage); err != nil {
-			return terraform.CreateOutput{}, fmt.Errorf("start libvirt container: %w", err)
+			return terraform.ApplyOutput{}, fmt.Errorf("start libvirt container: %w", err)
 		}
 		libvirtURI = libvirt.LibvirtTCPConnectURI
 
@@ -273,11 +274,11 @@ func (c *Creator) createQEMU(ctx context.Context, cl terraformClient, lv libvirt
 	case strings.HasPrefix(libvirtURI, "qemu+unix://"):
 		unixURI, err := url.Parse(strings.TrimPrefix(libvirtURI, "qemu+unix://"))
 		if err != nil {
-			return terraform.CreateOutput{}, err
+			return terraform.ApplyOutput{}, err
 		}
 		libvirtSocketPath = unixURI.Query().Get("socket")
 		if libvirtSocketPath == "" {
-			return terraform.CreateOutput{}, fmt.Errorf("socket path not specified in qemu+unix URI: %s", libvirtURI)
+			return terraform.ApplyOutput{}, fmt.Errorf("socket path not specified in qemu+unix URI: %s", libvirtURI)
 		}
 	}
 
@@ -293,7 +294,7 @@ func (c *Creator) createQEMU(ctx context.Context, cl terraformClient, lv libvirt
 	}
 
 	if err := cl.PrepareWorkspace(path.Join("terraform", strings.ToLower(cloudprovider.QEMU.String())), vars); err != nil {
-		return terraform.CreateOutput{}, fmt.Errorf("prepare workspace: %w", err)
+		return terraform.ApplyOutput{}, fmt.Errorf("prepare workspace: %w", err)
 	}
 
 	// Allow rollback of QEMU Terraform workspace from this point on
@@ -301,7 +302,7 @@ func (c *Creator) createQEMU(ctx context.Context, cl terraformClient, lv libvirt
 
 	tfOutput, err = cl.CreateCluster(ctx, opts.TFLogLevel)
 	if err != nil {
-		return terraform.CreateOutput{}, fmt.Errorf("create cluster: %w", err)
+		return terraform.ApplyOutput{}, fmt.Errorf("create cluster: %w", err)
 	}
 
 	return tfOutput, nil

cli/internal/cloudcmd/terraform.go

@@ -64,6 +64,7 @@ func awsTerraformVars(conf *config.Config, imageRef string, controlPlaneCount, w
 		IAMProfileWorkerNodes:  conf.Provider.AWS.IAMProfileWorkerNodes,
 		Debug:                  conf.IsDebugCluster(),
 		EnableSNP:              conf.GetAttestationConfig().GetVariant().Equal(variant.AWSSEVSNP{}),
+		CustomEndpoint:         conf.CustomEndpoint,
 	}
 }
 
@@ -98,6 +99,7 @@ func azureTerraformVars(conf *config.Config, imageRef string, controlPlaneCount,
 		SecureBoot:           conf.Provider.Azure.SecureBoot,
 		UserAssignedIdentity: conf.Provider.Azure.UserAssignedIdentity,
 		ResourceGroup:        conf.Provider.Azure.ResourceGroup,
+		CustomEndpoint:       conf.CustomEndpoint,
 	}
 
 	vars = normalizeAzureURIs(vars)
@@ -132,6 +134,7 @@ func gcpTerraformVars(conf *config.Config, imageRef string, controlPlaneCount, w
 		Zone:           conf.Provider.GCP.Zone,
 		ImageID:        imageRef,
 		Debug:          conf.IsDebugCluster(),
+		CustomEndpoint: conf.CustomEndpoint,
 	}
 }
 
@@ -165,6 +168,7 @@ func openStackTerraformVars(conf *config.Config, imageRef string, controlPlaneCo
 				StateDiskSizeGB: conf.StateDiskSizeGB,
 			},
 		},
+		CustomEndpoint: conf.CustomEndpoint,
 	}
 }
 

cli/internal/clusterid/id.go

@@ -10,7 +10,9 @@ import (
 	"github.com/edgelesssys/constellation/v2/internal/cloud/cloudprovider"
 )
 
-// File contains identifying information about a cluster.
+// File contains state information about a cluster.
+// This information is accessible after the creation
+// and can be used by further operations such as initialization and upgrades.
 type File struct {
 	// ClusterID is the unique identifier of the cluster.
 	ClusterID string `json:"clusterID,omitempty"`
@@ -22,6 +24,9 @@ type File struct {
 	CloudProvider cloudprovider.Provider `json:"cloudprovider,omitempty"`
 	// IP is the IP address the cluster can be reached at (often the load balancer).
 	IP string `json:"ip,omitempty"`
+	// APIServerCertSANs are subject alternative names (SAN) that are added to
+	// the TLS certificate of each apiserver instance.
+	APIServerCertSANs []string `json:"apiServerCertSANs,omitempty"`
 	// InitSecret is the secret the first Bootstrapper uses to verify the user.
 	InitSecret []byte `json:"initsecret,omitempty"`
 	// AttestationURL is the URL of the attestation service.

cli/internal/cmd/init.go

@@ -197,6 +197,7 @@ func (i *initCmd) initialize(cmd *cobra.Command, newDialer func(validator atls.V
 		ConformanceMode:        flags.conformance,
 		InitSecret:             idFile.InitSecret,
 		ClusterName:            clusterName,
+		ApiserverCertSans:      idFile.APIServerCertSANs,
 	}
 	i.log.Debugf("Sending initialization request")
 	resp, err := i.initCall(cmd.Context(), newDialer(validator), idFile.IP, req)

cli/internal/cmd/upgradeapply.go

@@ -113,6 +113,20 @@ func (u *upgradeApplyCmd) upgradeApply(cmd *cobra.Command, fileHandler file.Hand
 	if err := u.migrateTerraform(cmd, u.imageFetcher, conf, flags); err != nil {
 		return fmt.Errorf("performing Terraform migrations: %w", err)
 	}
+	// reload idFile after terraform migration
+	// it might have been updated by the migration
+	if err := fileHandler.ReadJSON(constants.ClusterIDsFileName, &idFile); err != nil {
+		return fmt.Errorf("reading updated cluster ID file: %w", err)
+	}
+
+	// extend the clusterConfig cert SANs with any of the supported endpoints:
+	// - (legacy) public IP
+	// - fallback endpoint
+	// - custom (user-provided) endpoint
+	sans := append([]string{idFile.IP, conf.CustomEndpoint}, idFile.APIServerCertSANs...)
+	if err := u.upgrader.ExtendClusterConfigCertSANs(cmd.Context(), sans); err != nil {
+		return fmt.Errorf("extending cert SANs: %w", err)
+	}
 
 	if conf.GetProvider() == cloudprovider.Azure || conf.GetProvider() == cloudprovider.GCP || conf.GetProvider() == cloudprovider.AWS {
 		var upgradeErr *compatibility.InvalidUpgradeError
@@ -350,6 +364,7 @@ type cloudUpgrader interface {
 	UpgradeNodeVersion(ctx context.Context, conf *config.Config, force bool) error
 	UpgradeHelmServices(ctx context.Context, config *config.Config, timeout time.Duration, allowDestructive bool, force bool) error
 	UpdateAttestationConfig(ctx context.Context, newConfig config.AttestationCfg) error
+	ExtendClusterConfigCertSANs(ctx context.Context, alternativeNames []string) error
 	GetClusterAttestationConfig(ctx context.Context, variant variant.Variant) (config.AttestationCfg, *corev1.ConfigMap, error)
 	PlanTerraformMigrations(ctx context.Context, opts upgrade.TerraformUpgradeOptions) (bool, error)
 	ApplyTerraformMigrations(ctx context.Context, opts upgrade.TerraformUpgradeOptions) error

cli/internal/cmd/upgradeapply_test.go

@@ -197,6 +197,10 @@ func (u stubUpgrader) ApplyTerraformMigrations(context.Context, upgrade.Terrafor
 	return u.applyTerraformErr
 }
 
+func (u stubUpgrader) ExtendClusterConfigCertSANs(_ context.Context, _ []string) error {
+	return nil
+}
+
 // AddManualStateMigration is not used in this test.
 // TODO(AB#3248): remove this method together with the definition in the interfaces.
 func (u stubUpgrader) AddManualStateMigration(_ terraform.StateMigration) {

cli/internal/kubernetes/BUILD.bazel

@@ -37,6 +37,8 @@ go_library(
         "@io_k8s_client_go//dynamic",
         "@io_k8s_client_go//kubernetes",
         "@io_k8s_client_go//tools/clientcmd",
+        "@io_k8s_kubernetes//cmd/kubeadm/app/apis/kubeadm/v1beta3",
+        "@io_k8s_sigs_yaml//:yaml",
     ],
 )
 

cli/internal/kubernetes/upgrade.go

@@ -13,6 +13,7 @@ import (
 	"fmt"
 	"io"
 	"path/filepath"
+	"sort"
 	"strings"
 	"time"
 
@@ -42,6 +43,8 @@ import (
 	"k8s.io/client-go/dynamic"
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/tools/clientcmd"
+	kubeadmv1beta3 "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta3"
+	"sigs.k8s.io/yaml"
 )
 
 // UpgradeCmdKind is the kind of the upgrade command (check, apply).
@@ -343,6 +346,69 @@ func (u *Upgrader) GetClusterAttestationConfig(ctx context.Context, variant vari
 	return existingAttestationConfig, existingConf, nil
 }
 
+// ExtendClusterConfigCertSANs extends the ClusterConfig stored under "kube-system/kubeadm-config" with the given SANs.
+// Existing SANs are preserved.
+func (u *Upgrader) ExtendClusterConfigCertSANs(ctx context.Context, alternativeNames []string) error {
+	clusterConfiguration, kubeadmConfig, err := u.GetClusterConfiguration(ctx)
+	if err != nil {
+		return fmt.Errorf("getting ClusterConfig: %w", err)
+	}
+
+	existingSANs := make(map[string]struct{})
+	for _, existingSAN := range clusterConfiguration.APIServer.CertSANs {
+		existingSANs[existingSAN] = struct{}{}
+	}
+
+	var missingSANs []string
+	for _, san := range alternativeNames {
+		if _, ok := existingSANs[san]; !ok {
+			missingSANs = append(missingSANs, san)
+		}
+	}
+
+	if len(missingSANs) == 0 {
+		return nil
+	}
+	u.log.Debugf("Extending the cluster's apiserver SAN field with the following SANs: %s\n", strings.Join(missingSANs, ", "))
+
+	clusterConfiguration.APIServer.CertSANs = append(clusterConfiguration.APIServer.CertSANs, missingSANs...)
+	sort.Strings(clusterConfiguration.APIServer.CertSANs)
+
+	newConfigYAML, err := yaml.Marshal(clusterConfiguration)
+	if err != nil {
+		return fmt.Errorf("marshaling ClusterConfiguration: %w", err)
+	}
+
+	kubeadmConfig.Data[constants.ClusterConfigurationKey] = string(newConfigYAML)
+	u.log.Debugf("Triggering kubeadm config update now")
+	if _, err = u.stableInterface.UpdateConfigMap(ctx, kubeadmConfig); err != nil {
+		return fmt.Errorf("setting new kubeadm config: %w", err)
+	}
+
+	fmt.Fprintln(u.outWriter, "Successfully extended the cluster's apiserver SAN field")
+	return nil
+}
+
+// GetClusterConfiguration fetches the kubeadm-config configmap from the cluster, extracts the config
+// and returns both the full configmap and the ClusterConfiguration.
+func (u *Upgrader) GetClusterConfiguration(ctx context.Context) (kubeadmv1beta3.ClusterConfiguration, *corev1.ConfigMap, error) {
+	existingConf, err := u.stableInterface.GetCurrentConfigMap(ctx, constants.KubeadmConfigMap)
+	if err != nil {
+		return kubeadmv1beta3.ClusterConfiguration{}, nil, fmt.Errorf("retrieving current kubeadm-config: %w", err)
+	}
+	clusterConf, ok := existingConf.Data[constants.ClusterConfigurationKey]
+	if !ok {
+		return kubeadmv1beta3.ClusterConfiguration{}, nil, errors.New("ClusterConfiguration missing from kubeadm-config")
+	}
+
+	var existingClusterConfig kubeadmv1beta3.ClusterConfiguration
+	if err := yaml.Unmarshal([]byte(clusterConf), &existingClusterConfig); err != nil {
+		return kubeadmv1beta3.ClusterConfiguration{}, nil, fmt.Errorf("unmarshaling ClusterConfiguration: %w", err)
+	}
+
+	return existingClusterConfig, existingConf, nil
+}
+
 // applyComponentsCM applies the k8s components ConfigMap to the cluster.
 func (u *Upgrader) applyComponentsCM(ctx context.Context, components *corev1.ConfigMap) error {
 	_, err := u.stableInterface.CreateConfigMap(ctx, components)

cli/internal/terraform/terraform.go

@@ -103,53 +103,66 @@ func (c *Client) PrepareUpgradeWorkspace(path, oldWorkingDir, newWorkingDir, bac
 }
 
 // CreateCluster creates a Constellation cluster using Terraform.
-func (c *Client) CreateCluster(ctx context.Context, logLevel LogLevel) (CreateOutput, error) {
+func (c *Client) CreateCluster(ctx context.Context, logLevel LogLevel) (ApplyOutput, error) {
 	if err := c.setLogLevel(logLevel); err != nil {
-		return CreateOutput{}, fmt.Errorf("set terraform log level %s: %w", logLevel.String(), err)
+		return ApplyOutput{}, fmt.Errorf("set terraform log level %s: %w", logLevel.String(), err)
 	}
 
 	if err := c.tf.Init(ctx); err != nil {
-		return CreateOutput{}, fmt.Errorf("terraform init: %w", err)
+		return ApplyOutput{}, fmt.Errorf("terraform init: %w", err)
 	}
 
 	if err := c.applyManualStateMigrations(ctx); err != nil {
-		return CreateOutput{}, fmt.Errorf("apply manual state migrations: %w", err)
+		return ApplyOutput{}, fmt.Errorf("apply manual state migrations: %w", err)
 	}
 
 	if err := c.tf.Apply(ctx); err != nil {
-		return CreateOutput{}, fmt.Errorf("terraform apply: %w", err)
+		return ApplyOutput{}, fmt.Errorf("terraform apply: %w", err)
 	}
 
 	tfState, err := c.tf.Show(ctx)
 	if err != nil {
-		return CreateOutput{}, fmt.Errorf("terraform show: %w", err)
+		return ApplyOutput{}, fmt.Errorf("terraform show: %w", err)
 	}
 
 	ipOutput, ok := tfState.Values.Outputs["ip"]
 	if !ok {
-		return CreateOutput{}, errors.New("no IP output found")
+		return ApplyOutput{}, errors.New("no IP output found")
 	}
 	ip, ok := ipOutput.Value.(string)
 	if !ok {
-		return CreateOutput{}, errors.New("invalid type in IP output: not a string")
+		return ApplyOutput{}, errors.New("invalid type in IP output: not a string")
+	}
+
+	apiServerCertSANsOutput, ok := tfState.Values.Outputs["api_server_cert_sans"]
+	if !ok {
+		return ApplyOutput{}, errors.New("no api_server_cert_sans output found")
+	}
+	apiServerCertSANsUntyped, ok := apiServerCertSANsOutput.Value.([]any)
+	if !ok {
+		return ApplyOutput{}, fmt.Errorf("invalid type in api_server_cert_sans output: %s is not a list of elements", apiServerCertSANsOutput.Type.FriendlyName())
+	}
+	apiServerCertSANs, err := toStringSlice(apiServerCertSANsUntyped)
+	if err != nil {
+		return ApplyOutput{}, fmt.Errorf("convert api_server_cert_sans output: %w", err)
 	}
 
 	secretOutput, ok := tfState.Values.Outputs["initSecret"]
 	if !ok {
-		return CreateOutput{}, errors.New("no initSecret output found")
+		return ApplyOutput{}, errors.New("no initSecret output found")
 	}
 	secret, ok := secretOutput.Value.(string)
 	if !ok {
-		return CreateOutput{}, errors.New("invalid type in initSecret output: not a string")
+		return ApplyOutput{}, errors.New("invalid type in initSecret output: not a string")
 	}
 
 	uidOutput, ok := tfState.Values.Outputs["uid"]
 	if !ok {
-		return CreateOutput{}, errors.New("no uid output found")
+		return ApplyOutput{}, errors.New("no uid output found")
 	}
 	uid, ok := uidOutput.Value.(string)
 	if !ok {
-		return CreateOutput{}, errors.New("invalid type in uid output: not a string")
+		return ApplyOutput{}, errors.New("invalid type in uid output: not a string")
 	}
 
 	var attestationURL string
@@ -159,17 +172,20 @@ func (c *Client) CreateCluster(ctx context.Context, logLevel LogLevel) (CreateOu
 		}
 	}
 
-	return CreateOutput{
+	return ApplyOutput{
 		IP:                ip,
+		APIServerCertSANs: apiServerCertSANs,
 		Secret:            secret,
 		UID:               uid,
 		AttestationURL:    attestationURL,
 	}, nil
 }
 
-// CreateOutput contains the Terraform output values of a cluster creation.
+// ApplyOutput contains the Terraform output values of a cluster creation
-type CreateOutput struct {
+// or apply operation.
+type ApplyOutput struct {
 	IP                string
+	APIServerCertSANs []string
 	Secret            string
 	UID               string
 	// AttestationURL is the URL of the attestation provider.
@@ -447,6 +463,18 @@ type StateMigration struct {
 	Hook        func(ctx context.Context, tfClient TFMigrator) error
 }
 
+func toStringSlice(in []any) ([]string, error) {
+	out := make([]string, len(in))
+	for i, v := range in {
+		s, ok := v.(string)
+		if !ok {
+			return nil, fmt.Errorf("invalid type in list: item at index %v of list is not a string", i)
+		}
+		out[i] = s
+	}
+	return out, nil
+}
+
 type tfInterface interface {
 	Apply(context.Context, ...tfexec.ApplyOption) error
 	Destroy(context.Context, ...tfexec.DestroyOption) error

cli/internal/terraform/terraform/aws/main.tf

@@ -46,8 +46,13 @@ locals {
   zones = distinct(sort([
     for node_group in var.node_groups : node_group.zone
   ]))
+  // wildcard_lb_dns_name is the DNS name of the load balancer with a wildcard for the name.
+  // example: given "name-1234567890.region.elb.amazonaws.com" it will return "*.region.elb.amazonaws.com"
+  wildcard_lb_dns_name = replace(aws_lb.front_end.dns_name, "/^[^.]*\\./", "*.")
 
-  tags = { constellation-uid = local.uid }
+  tags = {
+    constellation-uid = local.uid,
+  }
 }
 
 resource "random_id" "uid" {
@@ -83,7 +88,7 @@ resource "aws_eip" "lb" {
   # control-plane.
   for_each = toset([var.zone])
   domain   = "vpc"
-  tags     = local.tags
+  tags     = merge(local.tags, { "constellation-ip-endpoint" = each.key == var.zone ? "legacy-primary-zone" : "additional-zone" })
 }
 
 resource "aws_lb" "front_end" {

cli/internal/terraform/terraform/aws/outputs.tf

@@ -2,6 +2,10 @@ output "ip" {
   value = aws_eip.lb[var.zone].public_ip
 }
 
+output "api_server_cert_sans" {
+  value = sort(concat([aws_eip.lb[var.zone].public_ip, local.wildcard_lb_dns_name], var.custom_endpoint == "" ? [] : [var.custom_endpoint]))
+}
+
 output "uid" {
   value = local.uid
 }

cli/internal/terraform/terraform/aws/variables.tf

@@ -63,3 +63,9 @@ variable "enable_snp" {
   default     = true
   description = "Enable AMD SEV SNP. Setting this to true sets the cpu-option AmdSevSnp to enable."
 }
+
+variable "custom_endpoint" {
+  type        = string
+  default     = ""
+  description = "Custom endpoint to use for the Kubernetes apiserver. If not set, the default endpoint will be used."
+}

cli/internal/terraform/terraform/azure/main.tf

@@ -23,7 +23,9 @@ locals {
   uid            = random_id.uid.hex
   name           = "${var.name}-${local.uid}"
   initSecretHash = random_password.initSecret.bcrypt_hash
-  tags                  = { constellation-uid = local.uid }
+  tags = {
+    constellation-uid = local.uid,
+  }
   ports_node_range      = "30000-32767"
   ports_kubernetes      = "6443"
   ports_bootstrapper    = "9000"
@@ -33,6 +35,9 @@ locals {
   ports_debugd          = "4000"
   cidr_vpc_subnet_nodes = "192.168.178.0/24"
   cidr_vpc_subnet_pods  = "10.10.0.0/16"
+  // wildcard_lb_dns_name is the DNS name of the load balancer with a wildcard for the name.
+  // example: given "name-1234567890.location.cloudapp.azure.com" it will return "*.location.cloudapp.azure.com"
+  wildcard_lb_dns_name = replace(azurerm_public_ip.loadbalancer_ip.fqdn, "/^[^.]*\\./", "*.")
 }
 
 resource "random_id" "uid" {
@@ -72,6 +77,7 @@ resource "azurerm_application_insights" "insights" {
 
 resource "azurerm_public_ip" "loadbalancer_ip" {
   name                = "${local.name}-lb"
+  domain_name_label   = local.name
   resource_group_name = var.resource_group
   location            = var.location
   allocation_method   = "Static"

cli/internal/terraform/terraform/azure/outputs.tf

@@ -2,6 +2,10 @@ output "ip" {
   value = azurerm_public_ip.loadbalancer_ip.ip_address
 }
 
+output "api_server_cert_sans" {
+  value = sort(concat([azurerm_public_ip.loadbalancer_ip.ip_address, local.wildcard_lb_dns_name], var.custom_endpoint == "" ? [] : [var.custom_endpoint]))
+}
+
 output "uid" {
   value = local.uid
 }

cli/internal/terraform/terraform/azure/variables.tf

@@ -61,3 +61,9 @@ variable "user_assigned_identity" {
   type        = string
   description = "The name of the user assigned identity to attache to the nodes of the cluster."
 }
+
+variable "custom_endpoint" {
+  type        = string
+  default     = ""
+  description = "Custom endpoint to use for the Kubernetes apiserver. If not set, the default endpoint will be used."
+}

cli/internal/terraform/terraform/gcp/main.tf

@@ -33,7 +33,9 @@ locals {
   uid            = random_id.uid.hex
   name           = "${var.name}-${local.uid}"
   initSecretHash = random_password.initSecret.bcrypt_hash
-  labels                = { constellation-uid = local.uid }
+  labels = {
+    constellation-uid = local.uid,
+  }
   ports_node_range      = "30000-32767"
   ports_kubernetes      = "6443"
   ports_bootstrapper    = "9000"
@@ -170,6 +172,7 @@ module "instance_group" {
   named_ports         = each.value.role == "control-plane" ? local.control_plane_named_ports : []
   labels              = local.labels
   init_secret_hash    = local.initSecretHash
+  custom_endpoint     = var.custom_endpoint
 }
 
 resource "google_compute_global_address" "loadbalancer_ip" {

cli/internal/terraform/terraform/gcp/modules/instance_group/variables.tf

@@ -94,3 +94,8 @@ variable "zone" {
   type        = string
   description = "Zone to deploy the instance group in."
 }
+
+variable "custom_endpoint" {
+  type        = string
+  description = "Custom endpoint to use for the Kubernetes apiserver. If not set, the default endpoint will be used."
+}

cli/internal/terraform/terraform/gcp/outputs.tf

@@ -2,6 +2,14 @@ output "ip" {
   value = google_compute_global_address.loadbalancer_ip.address
 }
 
+output "api_server_cert_sans" {
+  value = sort(concat([google_compute_global_address.loadbalancer_ip.address], var.custom_endpoint == "" ? [] : [var.custom_endpoint]))
+}
+
+output "fallback_endpoint" {
+  value = google_compute_global_address.loadbalancer_ip.address
+}
+
 output "uid" {
   value = local.uid
 }

cli/internal/terraform/terraform/gcp/variables.tf

@@ -45,3 +45,9 @@ variable "debug" {
   default     = false
   description = "Enable debug mode. This opens up a debugd port that can be used to deploy a custom bootstrapper."
 }
+
+variable "custom_endpoint" {
+  type        = string
+  default     = ""
+  description = "Custom endpoint to use for the Kubernetes apiserver. If not set, the default endpoint will be used."
+}

cli/internal/terraform/terraform/openstack/outputs.tf

@@ -2,6 +2,10 @@ output "ip" {
   value = openstack_networking_floatingip_v2.public_ip.address
 }
 
+output "api_server_cert_sans" {
+  value = sort(concat([openstack_networking_floatingip_v2.public_ip.address], var.custom_endpoint == "" ? [] : [var.custom_endpoint]))
+}
+
 output "uid" {
   value = local.uid
 }

cli/internal/terraform/terraform/openstack/variables.tf

@@ -67,3 +67,9 @@ variable "debug" {
   default     = false
   description = "Enable debug mode. This opens up a debugd port that can be used to deploy a custom bootstrapper."
 }
+
+variable "custom_endpoint" {
+  type        = string
+  default     = ""
+  description = "Custom endpoint to use for the Kubernetes apiserver. If not set, the default endpoint will be used."
+}

cli/internal/terraform/terraform/qemu/outputs.tf

@@ -2,6 +2,10 @@ output "ip" {
   value = module.node_group["control_plane_default"].instance_ips[0]
 }
 
+output "api_server_cert_sans" {
+  value = sort(concat([module.node_group["control_plane_default"].instance_ips[0]], var.custom_endpoint == "" ? [] : [var.custom_endpoint]))
+}
+
 output "uid" {
   value = "qemu" // placeholder
 }

cli/internal/terraform/terraform/qemu/variables.tf

@@ -96,3 +96,9 @@ variable "name" {
   default     = "constellation"
   description = "name prefix of the cluster VMs"
 }
+
+variable "custom_endpoint" {
+  type        = string
+  default     = ""
+  description = "Custom endpoint to use for the Kubernetes apiserver. If not set, the default endpoint will be used."
+}

cli/internal/terraform/terraform_test.go

@@ -221,6 +221,9 @@ func TestCreateCluster(t *testing.T) {
 					"uid": {
 						Value: "12345abc",
 					},
+					"api_server_cert_sans": {
+						Value: []any{"192.0.2.100"},
+					},
 				},
 			},
 		}
@@ -242,6 +245,9 @@ func TestCreateCluster(t *testing.T) {
 					"attestationURL": {
 						Value: "https://12345.neu.attest.azure.net",
 					},
+					"api_server_cert_sans": {
+						Value: []any{"192.0.2.100"},
+					},
 				},
 			},
 		}

cli/internal/terraform/variables.go

@@ -30,29 +30,6 @@ type ClusterVariables interface {
 	GetCreateMAA() bool
 }
 
-// CommonVariables is user configuration for creating a cluster with Terraform.
-type CommonVariables struct {
-	// Name of the cluster.
-	Name string
-	// CountControlPlanes is the number of control-plane nodes to create.
-	CountControlPlanes int
-	// CountWorkers is the number of worker nodes to create.
-	CountWorkers int
-	// StateDiskSizeGB is the size of the state disk to allocate to each node, in GB.
-	StateDiskSizeGB int
-}
-
-// String returns a string representation of the variables, formatted as Terraform variables.
-func (v *CommonVariables) String() string {
-	b := &strings.Builder{}
-	writeLinef(b, "name = %q", v.Name)
-	writeLinef(b, "control_plane_count = %d", v.CountControlPlanes)
-	writeLinef(b, "worker_count = %d", v.CountWorkers)
-	writeLinef(b, "state_disk_size = %d", v.StateDiskSizeGB)
-
-	return b.String()
-}
-
 // AWSClusterVariables is user configuration for creating a cluster with Terraform on AWS.
 type AWSClusterVariables struct {
 	// Name of the cluster.
@@ -73,6 +50,8 @@ type AWSClusterVariables struct {
 	EnableSNP bool `hcl:"enable_snp" cty:"enable_snp"`
 	// NodeGroups is a map of node groups to create.
 	NodeGroups map[string]AWSNodeGroup `hcl:"node_groups" cty:"node_groups"`
+	// CustomEndpoint is the (optional) custom dns hostname for the kubernetes api server.
+	CustomEndpoint string `hcl:"custom_endpoint" cty:"custom_endpoint"`
 }
 
 // GetCreateMAA gets the CreateMAA variable.
@@ -138,6 +117,8 @@ type GCPClusterVariables struct {
 	Debug bool `hcl:"debug" cty:"debug"`
 	// NodeGroups is a map of node groups to create.
 	NodeGroups map[string]GCPNodeGroup `hcl:"node_groups" cty:"node_groups"`
+	// CustomEndpoint is the (optional) custom dns hostname for the kubernetes api server.
+	CustomEndpoint string `hcl:"custom_endpoint" cty:"custom_endpoint"`
 }
 
 // GetCreateMAA gets the CreateMAA variable.
@@ -212,6 +193,8 @@ type AzureClusterVariables struct {
 	SecureBoot *bool `hcl:"secure_boot" cty:"secure_boot"`
 	// NodeGroups is a map of node groups to create.
 	NodeGroups map[string]AzureNodeGroup `hcl:"node_groups" cty:"node_groups"`
+	// CustomEndpoint is the (optional) custom dns hostname for the kubernetes api server.
+	CustomEndpoint string `hcl:"custom_endpoint" cty:"custom_endpoint"`
 }
 
 // GetCreateMAA gets the CreateMAA variable.
@@ -287,6 +270,8 @@ type OpenStackClusterVariables struct {
 	OpenstackPassword string `hcl:"openstack_password" cty:"openstack_password"`
 	// Debug is true if debug mode is enabled.
 	Debug bool `hcl:"debug" cty:"debug"`
+	// CustomEndpoint is the (optional) custom dns hostname for the kubernetes api server.
+	CustomEndpoint string `hcl:"custom_endpoint" cty:"custom_endpoint"`
 }
 
 // GetCreateMAA gets the CreateMAA variable.
@@ -354,6 +339,8 @@ type QEMUVariables struct {
 	InitrdPath *string `hcl:"constellation_initrd" cty:"constellation_initrd"`
 	// KernelCmdline is the kernel command line.
 	KernelCmdline *string `hcl:"constellation_cmdline" cty:"constellation_cmdline"`
+	// CustomEndpoint is the (optional) custom dns hostname for the kubernetes api server.
+	CustomEndpoint string `hcl:"custom_endpoint" cty:"custom_endpoint"`
 }
 
 // GetCreateMAA gets the CreateMAA variable.

cli/internal/terraform/variables_test.go

@@ -43,6 +43,7 @@ func TestAWSClusterVariables(t *testing.T) {
 		IAMProfileWorkerNodes:  "arn:aws:iam::123456789012:instance-profile/cluster-name-worker",
 		Debug:                  true,
 		EnableSNP:              true,
+		CustomEndpoint:         "example.com",
 	}
 
 	// test that the variables are correctly rendered
@@ -72,6 +73,7 @@ node_groups = {
     zone          = "eu-central-1c"
   }
 }
+custom_endpoint = "example.com"
 `
 	got := vars.String()
 	assert.Equal(t, want, got)
@@ -117,6 +119,7 @@ func TestGCPClusterVariables(t *testing.T) {
 				DiskType:        "pd-ssd",
 			},
 		},
+		CustomEndpoint: "example.com",
 	}
 
 	// test that the variables are correctly rendered
@@ -144,6 +147,7 @@ node_groups = {
     zone          = "eu-central-1b"
   }
 }
+custom_endpoint = "example.com"
 `
 	got := vars.String()
 	assert.Equal(t, want, got)
@@ -186,6 +190,7 @@ func TestAzureClusterVariables(t *testing.T) {
 		CreateMAA:            to.Ptr(true),
 		Debug:                to.Ptr(true),
 		Location:             "eu-central-1",
+		CustomEndpoint:       "example.com",
 	}
 
 	// test that the variables are correctly rendered
@@ -207,6 +212,7 @@ node_groups = {
     zones         = null
   }
 }
+custom_endpoint = "example.com"
 `
 	got := vars.String()
 	assert.Equal(t, want, got)
@@ -249,6 +255,7 @@ func TestOpenStackClusterVariables(t *testing.T) {
 				StateDiskSizeGB: 30,
 			},
 		},
+		CustomEndpoint: "example.com",
 	}
 
 	// test that the variables are correctly rendered
@@ -271,6 +278,7 @@ openstack_user_domain_name = "my-user-domain"
 openstack_username         = "my-username"
 openstack_password         = "my-password"
 debug                      = true
+custom_endpoint            = "example.com"
 `
 	got := vars.String()
 	assert.Equal(t, want, got)
@@ -299,6 +307,7 @@ func TestQEMUClusterVariables(t *testing.T) {
 		NVRAM:              "production",
 		InitrdPath:         toPtr("/var/lib/libvirt/images/cluster-name-initrd"),
 		KernelCmdline:      toPtr("console=ttyS0,115200n8"),
+		CustomEndpoint:     "example.com",
 	}
 
 	// test that the variables are correctly rendered
@@ -323,6 +332,7 @@ metadata_libvirt_uri    = "qemu:///system"
 nvram                   = "/usr/share/OVMF/constellation_vars.production.fd"
 constellation_initrd    = "/var/lib/libvirt/images/cluster-name-initrd"
 constellation_cmdline   = "console=ttyS0,115200n8"
+custom_endpoint         = "example.com"
 `
 	got := vars.String()
 	assert.Equal(t, want, got)

cli/internal/upgrade/terraform.go

@@ -159,6 +159,7 @@ func (u *TerraformUpgrader) ApplyTerraformMigrations(ctx context.Context, opts T
 		CloudProvider:     opts.CSP,
 		InitSecret:        []byte(tfOutput.Secret),
 		IP:                tfOutput.IP,
+		APIServerCertSANs: tfOutput.APIServerCertSANs,
 		UID:               tfOutput.UID,
 		AttestationURL:    tfOutput.AttestationURL,
 	}
@@ -201,7 +202,7 @@ type tfClient interface {
 	PrepareUpgradeWorkspace(path, oldWorkingDir, newWorkingDir, upgradeID string, vars terraform.Variables) error
 	ShowPlan(ctx context.Context, logLevel terraform.LogLevel, planFilePath string, output io.Writer) error
 	Plan(ctx context.Context, logLevel terraform.LogLevel, planFile string) (bool, error)
-	CreateCluster(ctx context.Context, logLevel terraform.LogLevel) (terraform.CreateOutput, error)
+	CreateCluster(ctx context.Context, logLevel terraform.LogLevel) (terraform.ApplyOutput, error)
 }
 
 // policyPatcher interacts with the CSP (currently only applies for Azure) to update the attestation policy.

cli/internal/upgrade/terraform_test.go

@@ -371,8 +371,8 @@ func (u *stubTerraformClient) Plan(context.Context, terraform.LogLevel, string)
 	return u.hasDiff, u.planErr
 }
 
-func (u *stubTerraformClient) CreateCluster(context.Context, terraform.LogLevel) (terraform.CreateOutput, error) {
+func (u *stubTerraformClient) CreateCluster(context.Context, terraform.LogLevel) (terraform.ApplyOutput, error) {
-	return terraform.CreateOutput{}, u.CreateClusterErr
+	return terraform.ApplyOutput{}, u.CreateClusterErr
 }
 
 type stubPolicyPatcher struct {

debugd/internal/debugd/metadata/cloudprovider/cloudprovider.go

@@ -10,7 +10,6 @@ package cloudprovider
 import (
 	"context"
 	"fmt"
-	"net"
 
 	"github.com/edgelesssys/constellation/v2/internal/cloud/metadata"
 	"github.com/edgelesssys/constellation/v2/internal/role"
@@ -22,7 +21,7 @@ type providerMetadata interface {
 	// Self retrieves the current instance.
 	Self(ctx context.Context) (metadata.InstanceMetadata, error)
 	// GetLoadBalancerEndpoint returns the endpoint of the load balancer.
-	GetLoadBalancerEndpoint(ctx context.Context) (string, error)
+	GetLoadBalancerEndpoint(ctx context.Context) (host, port string, err error)
 	// UID returns the UID of the current instance.
 	UID(ctx context.Context) (string, error)
 }
@@ -91,16 +90,10 @@ func (f *Fetcher) DiscoverDebugdIPs(ctx context.Context) ([]string, error) {
 
 // DiscoverLoadbalancerIP gets load balancer IP from metadata API.
 func (f *Fetcher) DiscoverLoadbalancerIP(ctx context.Context) (string, error) {
-	lbEndpoint, err := f.metaAPI.GetLoadBalancerEndpoint(ctx)
+	lbHost, _, err := f.metaAPI.GetLoadBalancerEndpoint(ctx)
 	if err != nil {
 		return "", fmt.Errorf("retrieving load balancer endpoint: %w", err)
 	}
 
-	// The port of the endpoint is not the port we need. We need to strip it off.
+	return lbHost, nil
-	lbIP, _, err := net.SplitHostPort(lbEndpoint)
-	if err != nil {
-		return "", fmt.Errorf("parsing load balancer endpoint: %w", err)
-	}
-
-	return lbIP, nil
 }

debugd/internal/debugd/metadata/cloudprovider/cloudprovider_test.go

@@ -123,7 +123,6 @@ func TestDiscoverDebugIPs(t *testing.T) {
 
 func TestDiscoverLoadbalancerIP(t *testing.T) {
 	ip := "192.0.2.1"
-	endpoint := ip + ":1234"
 	someErr := errors.New("failed")
 
 	testCases := map[string]struct {
@@ -132,17 +131,13 @@ func TestDiscoverLoadbalancerIP(t *testing.T) {
 		wantErr bool
 	}{
 		"discovery works": {
-			metaAPI: &stubMetadata{getLBEndpointRes: endpoint},
+			metaAPI: &stubMetadata{getLBHostRes: ip},
 			wantIP:  ip,
 		},
 		"get endpoint fails": {
 			metaAPI: &stubMetadata{getLBEndpointErr: someErr},
 			wantErr: true,
 		},
-		"invalid endpoint": {
-			metaAPI: &stubMetadata{getLBEndpointRes: "invalid"},
-			wantErr: true,
-		},
 	}
 
 	for name, tc := range testCases {
@@ -170,7 +165,7 @@ type stubMetadata struct {
 	listErr                    error
 	selfRes                    metadata.InstanceMetadata
 	selfErr                    error
-	getLBEndpointRes string
+	getLBHostRes, getLBPortRes string
 	getLBEndpointErr           error
 	uid                        string
 	uidErr                     error
@@ -184,8 +179,8 @@ func (m *stubMetadata) Self(_ context.Context) (metadata.InstanceMetadata, error
 	return m.selfRes, m.selfErr
 }
 
-func (m *stubMetadata) GetLoadBalancerEndpoint(_ context.Context) (string, error) {
+func (m *stubMetadata) GetLoadBalancerEndpoint(_ context.Context) (string, string, error) {
-	return m.getLBEndpointRes, m.getLBEndpointErr
+	return m.getLBHostRes, m.getLBPortRes, m.getLBEndpointErr
 }
 
 func (m *stubMetadata) UID(_ context.Context) (string, error) {

debugd/internal/debugd/metadata/fallback/fallback.go

@@ -32,8 +32,8 @@ func (fallbackMetadata) Self(context.Context) (metadata.InstanceMetadata, error)
 }
 
 // GetLoadBalancerEndpoint returns the endpoint of the load balancer.
-func (fallbackMetadata) GetLoadBalancerEndpoint(context.Context) (string, error) {
+func (fallbackMetadata) GetLoadBalancerEndpoint(context.Context) (string, string, error) {
-	return "", nil
+	return "", "", nil
 }
 
 // UID returns the UID of the current instance.

hack/go.mod

@@ -109,6 +109,7 @@ require (
 	github.com/benbjohnson/clock v1.3.0 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/blang/semver v3.5.1+incompatible // indirect
+	github.com/blang/semver/v4 v4.0.0 // indirect
 	github.com/cespare/xxhash/v2 v2.2.0 // indirect
 	github.com/chai2010/gettext-go v1.0.2 // indirect
 	github.com/cloudflare/circl v1.3.3 // indirect
@@ -305,10 +306,12 @@ require (
 	k8s.io/apiserver v0.27.3 // indirect
 	k8s.io/cli-runtime v0.27.2 // indirect
 	k8s.io/client-go v0.27.3 // indirect
+	k8s.io/cluster-bootstrap v0.27.3 // indirect
 	k8s.io/component-base v0.27.3 // indirect
 	k8s.io/klog/v2 v2.100.1 // indirect
 	k8s.io/kube-openapi v0.0.0-20230501164219-8b0f38b5fd1f // indirect
 	k8s.io/kubectl v0.27.2 // indirect
+	k8s.io/kubernetes v1.27.3 // indirect
 	k8s.io/utils v0.0.0-20230505201702-9f6742963106 // indirect
 	oras.land/oras-go v1.2.3 // indirect
 	sigs.k8s.io/controller-runtime v0.15.0 // indirect

hack/go.sum

@@ -194,6 +194,8 @@ github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kB
 github.com/bketelsen/crypt v0.0.4/go.mod h1:aI6NrJ0pMGgvZKL1iVgXLnfIFJtfV+bKCoqOes/6LfM=
 github.com/blang/semver v3.5.1+incompatible h1:cQNTCjp13qL8KC3Nbxr/y2Bqb63oX6wdnnjpJbkM4JQ=
 github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk=
+github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM=
+github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ=
 github.com/bshuster-repo/logrus-logstash-hook v1.0.0 h1:e+C0SB5R1pu//O4MQ3f9cFuPGoOVeF2fE4Og9otCc70=
 github.com/bugsnag/bugsnag-go v0.0.0-20141110184014-b1d153021fcd h1:rFt+Y/IK1aEZkEHchZRSq9OQbsSzIT/OrI8YFFmRIng=
 github.com/bugsnag/osext v0.0.0-20130617224835-0dd3f918b21b h1:otBG+dV+YK+Soembjv71DPz3uX/V/6MMlSyD9JBQ6kQ=
@@ -652,8 +654,8 @@ github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7V
 github.com/k0kubun/go-ansi v0.0.0-20180517002512-3bf9e2903213/go.mod h1:vNUNkEQ1e29fT/6vq2aBdFsgNPmy8qMdSay1npru+Sw=
 github.com/karrick/godirwalk v1.8.0/go.mod h1:H5KPZjojv4lE+QYImBI8xVtrBRgYrIVsaRPx4tDPEn4=
 github.com/karrick/godirwalk v1.10.3/go.mod h1:RoGL9dQei4vP9ilrpETWE8CLOZ1kiN0LhBygSwrAsHA=
-github.com/karrick/godirwalk v1.16.1 h1:DynhcF+bztK8gooS0+NDJFrdNZjJ3gzVzC545UNA9iw=
 github.com/karrick/godirwalk v1.16.1/go.mod h1:j4mkqPuvaLI8mp1DroR3P6ad7cyYd4c1qeJ3RV7ULlk=
+github.com/karrick/godirwalk v1.17.0 h1:b4kY7nqDdioR/6qnbHQyDvmA17u5G1cZ6J+CZXwSWoI=
 github.com/kevinburke/ssh_config v1.2.0 h1:x584FjTGwHzMwvHx18PXxbBVzfnxogHaAReU4gf13a4=
 github.com/kevinburke/ssh_config v1.2.0/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM=
 github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=
@@ -1549,6 +1551,8 @@ k8s.io/cli-runtime v0.27.2 h1:9HI8gfReNujKXt16tGOAnb8b4NZ5E+e0mQQHKhFGwYw=
 k8s.io/cli-runtime v0.27.2/go.mod h1:9UecpyPDTkhiYY4d9htzRqN+rKomJgyb4wi0OfrmCjw=
 k8s.io/client-go v0.27.3 h1:7dnEGHZEJld3lYwxvLl7WoehK6lAq7GvgjxpA3nv1E8=
 k8s.io/client-go v0.27.3/go.mod h1:2MBEKuTo6V1lbKy3z1euEGnhPfGZLKTS9tiJ2xodM48=
+k8s.io/cluster-bootstrap v0.27.3 h1:yk1XIWt/mbMgNHFdxd0HyVPq/rnJK7BS3oXj24gHClU=
+k8s.io/cluster-bootstrap v0.27.3/go.mod h1:4/bxgDkpV7XPapJS1585P/nvy3FdBIoFssK4Z5oztrc=
 k8s.io/component-base v0.27.3 h1:g078YmdcdTfrCE4fFobt7qmVXwS8J/3cI1XxRi/2+6k=
 k8s.io/component-base v0.27.3/go.mod h1:JNiKYcGImpQ44iwSYs6dysxzR9SxIIgQalk4HaCNVUY=
 k8s.io/klog/v2 v2.100.1 h1:7WCHKK6K8fNhTqfBhISHQ97KrnJNFZMcQvKp7gP/tmg=
@@ -1557,6 +1561,8 @@ k8s.io/kube-openapi v0.0.0-20230501164219-8b0f38b5fd1f h1:2kWPakN3i/k81b0gvD5C5F
 k8s.io/kube-openapi v0.0.0-20230501164219-8b0f38b5fd1f/go.mod h1:byini6yhqGC14c3ebc/QwanvYwhuMWF6yz2F8uwW8eg=
 k8s.io/kubectl v0.27.2 h1:sSBM2j94MHBFRWfHIWtEXWCicViQzZsb177rNsKBhZg=
 k8s.io/kubectl v0.27.2/go.mod h1:GCOODtxPcrjh+EC611MqREkU8RjYBh10ldQCQ6zpFKw=
+k8s.io/kubernetes v1.27.3 h1:gwufSj7y6X18Q2Gl8v4Ev+AJHdzWkG7A8VNFffS9vu0=
+k8s.io/kubernetes v1.27.3/go.mod h1:U8ZXeKBAPxeb4J4/HOaxjw1A9K6WfSH+fY2SS7CR6IM=
 k8s.io/utils v0.0.0-20230505201702-9f6742963106 h1:EObNQ3TW2D+WptiYXlApGNLVy0zm/JIBVY9i+M4wpAU=
 k8s.io/utils v0.0.0-20230505201702-9f6742963106/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0=
 libvirt.org/go/libvirt v1.9004.0 h1:u+CHhs2OhVmu0MWzBDrlbLzQ5QB3ZfWtfT+lD3EaUIs=

internal/cloud/aws/BUILD.bazel

@@ -12,6 +12,7 @@ go_library(
     deps = [
         "//internal/cloud",
         "//internal/cloud/metadata",
+        "//internal/constants",
         "//internal/role",
         "@com_github_aws_aws_sdk_go_v2//aws",
         "@com_github_aws_aws_sdk_go_v2_config//:config",
@@ -21,6 +22,7 @@ go_library(
         "@com_github_aws_aws_sdk_go_v2_service_ec2//:ec2",
         "@com_github_aws_aws_sdk_go_v2_service_ec2//types",
         "@com_github_aws_aws_sdk_go_v2_service_elasticloadbalancingv2//:elasticloadbalancingv2",
+        "@com_github_aws_aws_sdk_go_v2_service_elasticloadbalancingv2//types",
         "@com_github_aws_aws_sdk_go_v2_service_resourcegroupstaggingapi//:resourcegroupstaggingapi",
         "@com_github_aws_aws_sdk_go_v2_service_resourcegroupstaggingapi//types",
         "@io_k8s_utils//clock",

internal/cloud/aws/aws.go

@@ -19,6 +19,7 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"strconv"
 
 	"github.com/aws/aws-sdk-go-v2/aws"
 	"github.com/aws/aws-sdk-go-v2/config"
@@ -26,10 +27,12 @@ import (
 	"github.com/aws/aws-sdk-go-v2/service/ec2"
 	ec2Types "github.com/aws/aws-sdk-go-v2/service/ec2/types"
 	"github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2"
+	elasticloadbalancingv2types "github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2/types"
 	"github.com/aws/aws-sdk-go-v2/service/resourcegroupstaggingapi"
 	tagType "github.com/aws/aws-sdk-go-v2/service/resourcegroupstaggingapi/types"
 	"github.com/edgelesssys/constellation/v2/internal/cloud"
 	"github.com/edgelesssys/constellation/v2/internal/cloud/metadata"
+	"github.com/edgelesssys/constellation/v2/internal/constants"
 	"github.com/edgelesssys/constellation/v2/internal/role"
 )
 
@@ -48,6 +51,7 @@ type loadbalancerAPI interface {
 
 type ec2API interface {
 	DescribeInstances(context.Context, *ec2.DescribeInstancesInput, ...func(*ec2.Options)) (*ec2.DescribeInstancesOutput, error)
+	DescribeAddresses(context.Context, *ec2.DescribeAddressesInput, ...func(*ec2.Options)) (*ec2.DescribeAddressesOutput, error)
 }
 
 type imdsAPI interface {
@@ -126,46 +130,100 @@ func (c *Cloud) InitSecretHash(ctx context.Context) ([]byte, error) {
 }
 
 // GetLoadBalancerEndpoint returns the endpoint of the load balancer.
-func (c *Cloud) GetLoadBalancerEndpoint(ctx context.Context) (string, error) {
+// TODO(malt3): remove old infrastructure code once we have migrated to using DNS as the load balancer endpoint.
+func (c *Cloud) GetLoadBalancerEndpoint(ctx context.Context) (host, port string, err error) {
+	// try new architecture first
 	uid, err := c.readInstanceTag(ctx, cloud.TagUID)
 	if err != nil {
-		return "", fmt.Errorf("retrieving uid tag: %w", err)
+		return "", "", fmt.Errorf("retrieving uid tag: %w", err)
+	}
+	describeIPsOutput, err := c.ec2.DescribeAddresses(ctx, &ec2.DescribeAddressesInput{
+		Filters: []ec2Types.Filter{
+			{
+				Name:   aws.String(cloud.TagUID),
+				Values: []string{uid},
+			},
+			{
+				Name:   aws.String("constellation-ip-endpoint"),
+				Values: []string{"legacy-primary-zone"},
+			},
+		},
+	})
+	if err == nil && len(describeIPsOutput.Addresses) == 1 && describeIPsOutput.Addresses[0].PublicIp != nil {
+		return *describeIPsOutput.Addresses[0].PublicIp, strconv.FormatInt(constants.KubernetesPort, 10), nil
+	}
+	// fallback to old architecture
+	// this will be removed in the future
+	hostname, err := c.getLoadBalancerIPOldInfrastructure(ctx)
+	if err != nil {
+		return "", "", fmt.Errorf("retrieving load balancer ip: %w", err)
+	}
+	return hostname, strconv.FormatInt(constants.KubernetesPort, 10), nil
+}
+
+// getLoadBalancerIPOldInfrastructure returns the IP of the load balancer.
+// This is only used for the old infrastructure.
+// This will be removed in the future.
+func (c *Cloud) getLoadBalancerIPOldInfrastructure(ctx context.Context) (string, error) {
+	loadbalancer, err := c.getLoadBalancer(ctx)
+	if err != nil {
+		return "", fmt.Errorf("finding Constellation load balancer: %w", err)
+	}
+
+	// TODO(malt3): Add support for multiple availability zones in the lb frontend.
+	// This can only be done after we have migrated to using DNS as the load balancer endpoint.
+	// At that point, we don't need to care about the number of availability zones anymore.
+	if len(loadbalancer.AvailabilityZones) != 1 {
+		return "", fmt.Errorf("%d availability zones found; expected 1", len(loadbalancer.AvailabilityZones))
+	}
+
+	if len(loadbalancer.AvailabilityZones[0].LoadBalancerAddresses) != 1 {
+		return "", fmt.Errorf("%d load balancer addresses found; expected 1", len(loadbalancer.AvailabilityZones[0].LoadBalancerAddresses))
+	}
+	if loadbalancer.AvailabilityZones[0].LoadBalancerAddresses[0].IpAddress == nil {
+		return "", errors.New("load balancer address is nil")
+	}
+
+	return *loadbalancer.AvailabilityZones[0].LoadBalancerAddresses[0].IpAddress, nil
+}
+
+/*
+// TODO(malt3): uncomment and use as soon as we switch the primary endpoint to DNS.
+func (c *Cloud) getLoadBalancerDNSName(ctx context.Context) (string, error) {
+	loadbalancer, err := c.getLoadBalancer(ctx)
+	if err != nil {
+		return "", fmt.Errorf("finding Constellation load balancer: %w", err)
+	}
+	if loadbalancer.DNSName == nil {
+		return "", errors.New("load balancer dns name missing")
+	}
+	return *loadbalancer.DNSName, nil
+}
+*/
+
+func (c *Cloud) getLoadBalancer(ctx context.Context) (*elasticloadbalancingv2types.LoadBalancer, error) {
+	uid, err := c.readInstanceTag(ctx, cloud.TagUID)
+	if err != nil {
+		return nil, fmt.Errorf("retrieving uid tag: %w", err)
 	}
 	arns, err := c.getARNsByTag(ctx, uid, "elasticloadbalancing:loadbalancer")
 	if err != nil {
-		return "", fmt.Errorf("retrieving load balancer ARNs: %w", err)
+		return nil, fmt.Errorf("retrieving load balancer ARNs: %w", err)
 	}
 	if len(arns) != 1 {
-		return "", fmt.Errorf("%d load balancers found", len(arns))
+		return nil, fmt.Errorf("%d load balancers found", len(arns))
 	}
 
 	output, err := c.loadbalancer.DescribeLoadBalancers(ctx, &elasticloadbalancingv2.DescribeLoadBalancersInput{
 		LoadBalancerArns: arns,
 	})
 	if err != nil {
-		return "", fmt.Errorf("retrieving load balancer: %w", err)
+		return nil, fmt.Errorf("retrieving load balancer: %w", err)
 	}
 	if len(output.LoadBalancers) != 1 {
-		return "", fmt.Errorf("%d load balancers found; expected 1", len(output.LoadBalancers))
+		return nil, fmt.Errorf("%d load balancers found; expected 1", len(output.LoadBalancers))
 	}
-
+	return &output.LoadBalancers[0], nil
-	// TODO(malt3): Add support for multiple availability zones in the lb frontend.
-	// This can only be done after we have migrated to using DNS as the load balancer endpoint.
-	// At that point, we don't need to care about the number of availability zones anymore.
-	if len(output.LoadBalancers[0].AvailabilityZones) != 1 {
-		return "", fmt.Errorf("%d availability zones found; expected 1", len(output.LoadBalancers[0].AvailabilityZones))
-	}
-
-	if len(output.LoadBalancers[0].AvailabilityZones[0].LoadBalancerAddresses) != 1 {
-		return "", fmt.Errorf("%d load balancer addresses found; expected 1", len(output.LoadBalancers[0].AvailabilityZones[0].LoadBalancerAddresses))
-	}
-	if output.LoadBalancers[0].AvailabilityZones[0].LoadBalancerAddresses[0].IpAddress == nil {
-		return "", errors.New("load balancer address is nil")
-	}
-
-	// TODO(malt3): ideally, we would use DNS here instead of IP addresses.
-	// Requires changes to the infrastructure.
-	return *output.LoadBalancers[0].AvailabilityZones[0].LoadBalancerAddresses[0].IpAddress, nil
 }
 
 // getARNsByTag returns a list of ARNs that have the given tag.

internal/cloud/aws/aws_test.go

@@ -485,7 +485,7 @@ func TestGetLoadBalancerEndpoint(t *testing.T) {
 		ec2API       *stubEC2
 		loadbalancer *stubLoadbalancer
 		resourceapi  *stubResourceGroupTagging
-		wantAddr     string
+		wantHost     string
 		wantErr      bool
 	}{
 		"success retrieving loadbalancer endpoint": {
@@ -514,6 +514,47 @@ func TestGetLoadBalancerEndpoint(t *testing.T) {
 						},
 					},
 				},
+				describeAddressesResp: &ec2.DescribeAddressesOutput{
+					Addresses: []ec2Types.Address{
+						{
+							Tags: []ec2Types.Tag{
+								{Key: aws.String(cloud.TagUID), Value: aws.String("uid")},
+								{Key: aws.String("constellation-ip-endpoint"), Value: aws.String("legacy-primary-zone")},
+							},
+							PublicIp: aws.String(lbAddr),
+						},
+					},
+				},
+			},
+			wantHost: lbAddr,
+		},
+		"success retrieving loadbalancer endpoint legacy": {
+			imds: &stubIMDS{
+				instanceDocumentResp: &imds.GetInstanceIdentityDocumentOutput{
+					InstanceIdentityDocument: imds.InstanceIdentityDocument{
+						InstanceID: "test-instance-id",
+					},
+				},
+			},
+			ec2API: &stubEC2{
+				selfInstance: &ec2.DescribeInstancesOutput{
+					Reservations: []ec2Types.Reservation{
+						{
+							Instances: []ec2Types.Instance{
+								{
+									InstanceId: aws.String("test-instance-id"),
+									Tags: []ec2Types.Tag{
+										{
+											Key:   aws.String(cloud.TagUID),
+											Value: aws.String("uid"),
+										},
+									},
+								},
+							},
+						},
+					},
+				},
+				describeAddressesErr: errors.New("using legacy infrastructure"),
 			},
 			loadbalancer: &stubLoadbalancer{
 				describeLoadBalancersOut: &elasticloadbalancingv2.DescribeLoadBalancersOutput{
@@ -541,9 +582,9 @@ func TestGetLoadBalancerEndpoint(t *testing.T) {
 					},
 				},
 			},
-			wantAddr: lbAddr,
+			wantHost: lbAddr,
 		},
-		"too many ARNs": {
+		"too many ARNs legacy": {
 			imds: &stubIMDS{
 				instanceDocumentResp: &imds.GetInstanceIdentityDocumentOutput{
 					InstanceIdentityDocument: imds.InstanceIdentityDocument{
@@ -569,6 +610,7 @@ func TestGetLoadBalancerEndpoint(t *testing.T) {
 						},
 					},
 				},
+				describeAddressesErr: errors.New("using legacy infrastructure"),
 			},
 			loadbalancer: &stubLoadbalancer{
 				describeLoadBalancersOut: &elasticloadbalancingv2.DescribeLoadBalancersOutput{
@@ -601,7 +643,7 @@ func TestGetLoadBalancerEndpoint(t *testing.T) {
 			},
 			wantErr: true,
 		},
-		"too many ARNs (paged)": {
+		"too many ARNs (paged) legacy": {
 			imds: &stubIMDS{
 				instanceDocumentResp: &imds.GetInstanceIdentityDocumentOutput{
 					InstanceIdentityDocument: imds.InstanceIdentityDocument{
@@ -627,6 +669,7 @@ func TestGetLoadBalancerEndpoint(t *testing.T) {
 						},
 					},
 				},
+				describeAddressesErr: errors.New("using legacy infrastructure"),
 			},
 			loadbalancer: &stubLoadbalancer{
 				describeLoadBalancersOut: &elasticloadbalancingv2.DescribeLoadBalancersOutput{
@@ -664,7 +707,7 @@ func TestGetLoadBalancerEndpoint(t *testing.T) {
 			},
 			wantErr: true,
 		},
-		"loadbalancer has no availability zones": {
+		"loadbalancer has no availability zones legacy": {
 			imds: &stubIMDS{
 				instanceDocumentResp: &imds.GetInstanceIdentityDocumentOutput{
 					InstanceIdentityDocument: imds.InstanceIdentityDocument{
@@ -690,6 +733,7 @@ func TestGetLoadBalancerEndpoint(t *testing.T) {
 						},
 					},
 				},
+				describeAddressesErr: errors.New("using legacy infrastructure"),
 			},
 			loadbalancer: &stubLoadbalancer{
 				describeLoadBalancersOut: &elasticloadbalancingv2.DescribeLoadBalancersOutput{
@@ -711,7 +755,7 @@ func TestGetLoadBalancerEndpoint(t *testing.T) {
 			},
 			wantErr: true,
 		},
-		"failure to get resources by tag": {
+		"failure to get resources by tag legacy": {
 			imds: &stubIMDS{
 				instanceDocumentResp: &imds.GetInstanceIdentityDocumentOutput{
 					InstanceIdentityDocument: imds.InstanceIdentityDocument{
@@ -737,6 +781,7 @@ func TestGetLoadBalancerEndpoint(t *testing.T) {
 						},
 					},
 				},
+				describeAddressesErr: errors.New("using legacy infrastructure"),
 			},
 			loadbalancer: &stubLoadbalancer{
 				describeLoadBalancersOut: &elasticloadbalancingv2.DescribeLoadBalancersOutput{
@@ -772,14 +817,15 @@ func TestGetLoadBalancerEndpoint(t *testing.T) {
 				resourceapiClient: tc.resourceapi,
 			}
 
-			endpoint, err := m.GetLoadBalancerEndpoint(context.Background())
+			gotHost, gotPort, err := m.GetLoadBalancerEndpoint(context.Background())
 			if tc.wantErr {
 				assert.Error(err)
 				return
 			}
 
 			assert.NoError(err)
-			assert.Equal(tc.wantAddr, endpoint)
+			assert.Equal(tc.wantHost, gotHost)
+			assert.Equal("6443", gotPort)
 		})
 	}
 }
@@ -973,6 +1019,8 @@ type stubEC2 struct {
 	selfInstance           *ec2.DescribeInstancesOutput
 	describeInstancesResp1 *ec2.DescribeInstancesOutput
 	describeInstancesResp2 *ec2.DescribeInstancesOutput
+	describeAddressesErr   error
+	describeAddressesResp  *ec2.DescribeAddressesOutput
 }
 
 func (s *stubEC2) DescribeInstances(_ context.Context, in *ec2.DescribeInstancesInput, _ ...func(*ec2.Options)) (*ec2.DescribeInstancesOutput, error) {
@@ -985,6 +1033,10 @@ func (s *stubEC2) DescribeInstances(_ context.Context, in *ec2.DescribeInstances
 	return s.describeInstancesResp2, s.describeInstancesErr
 }
 
+func (s *stubEC2) DescribeAddresses(context.Context, *ec2.DescribeAddressesInput, ...func(*ec2.Options)) (*ec2.DescribeAddressesOutput, error) {
+	return s.describeAddressesResp, s.describeAddressesErr
+}
+
 type stubLoadbalancer struct {
 	describeLoadBalancersErr error
 	describeLoadBalancersOut *elasticloadbalancingv2.DescribeLoadBalancersOutput

internal/cloud/azure/BUILD.bazel

@@ -15,6 +15,7 @@ go_library(
         "//internal/cloud",
         "//internal/cloud/azureshared",
         "//internal/cloud/metadata",
+        "//internal/constants",
         "//internal/role",
         "@com_github_azure_azure_sdk_for_go_sdk_azcore//runtime",
         "@com_github_azure_azure_sdk_for_go_sdk_azidentity//:azidentity",

internal/cloud/azure/azure.go

@@ -21,6 +21,7 @@ import (
 	"errors"
 	"fmt"
 	"path"
+	"strconv"
 
 	"github.com/Azure/azure-sdk-for-go/sdk/azidentity"
 	"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v4"
@@ -28,6 +29,7 @@ import (
 	"github.com/edgelesssys/constellation/v2/internal/cloud"
 	"github.com/edgelesssys/constellation/v2/internal/cloud/azureshared"
 	"github.com/edgelesssys/constellation/v2/internal/cloud/metadata"
+	"github.com/edgelesssys/constellation/v2/internal/constants"
 	"github.com/edgelesssys/constellation/v2/internal/role"
 )
 
@@ -156,41 +158,12 @@ func (c *Cloud) GetCCMConfig(ctx context.Context, providerID string, cloudServic
 //
 // The returned string is an IP address without a port, but the method name needs to satisfy the
 // metadata interface.
-func (c *Cloud) GetLoadBalancerEndpoint(ctx context.Context) (string, error) {
+func (c *Cloud) GetLoadBalancerEndpoint(ctx context.Context) (host, port string, err error) {
-	resourceGroup, err := c.imds.resourceGroup(ctx)
+	hostname, err := c.getLoadBalancerPublicIP(ctx)
 	if err != nil {
-		return "", fmt.Errorf("retrieving resource group: %w", err)
+		return "", "", fmt.Errorf("retrieving load balancer public IP: %w", err)
 	}
-	uid, err := c.imds.uid(ctx)
+	return hostname, strconv.FormatInt(constants.KubernetesPort, 10), nil
-	if err != nil {
-		return "", fmt.Errorf("retrieving instance UID: %w", err)
-	}
-
-	lb, err := c.getLoadBalancer(ctx, resourceGroup, uid)
-	if err != nil {
-		return "", fmt.Errorf("retrieving load balancer: %w", err)
-	}
-	if lb == nil || lb.Properties == nil {
-		return "", errors.New("could not dereference load balancer IP configuration")
-	}
-
-	var pubIP string
-	for _, fipConf := range lb.Properties.FrontendIPConfigurations {
-		if fipConf == nil || fipConf.Properties == nil || fipConf.Properties.PublicIPAddress == nil || fipConf.Properties.PublicIPAddress.ID == nil {
-			continue
-		}
-		pubIP = path.Base(*fipConf.Properties.PublicIPAddress.ID)
-		break
-	}
-
-	resp, err := c.pubIPAPI.Get(ctx, resourceGroup, pubIP, nil)
-	if err != nil {
-		return "", fmt.Errorf("retrieving load balancer public IP address: %w", err)
-	}
-	if resp.Properties == nil || resp.Properties.IPAddress == nil {
-		return "", fmt.Errorf("could not resolve public IP address reference for load balancer")
-	}
-	return *resp.Properties.IPAddress, nil
 }
 
 // List retrieves all instances belonging to the current constellation.
@@ -409,6 +382,86 @@ func (c *Cloud) getVMInterfaces(ctx context.Context, vm armcompute.VirtualMachin
 	return networkInterfaces, nil
 }
 
+// getLoadBalancerPublicIP retrieves the first load balancer IP from cloud provider metadata.
+func (c *Cloud) getLoadBalancerPublicIP(ctx context.Context) (string, error) {
+	resourceGroup, err := c.imds.resourceGroup(ctx)
+	if err != nil {
+		return "", fmt.Errorf("retrieving resource group: %w", err)
+	}
+	uid, err := c.imds.uid(ctx)
+	if err != nil {
+		return "", fmt.Errorf("retrieving instance UID: %w", err)
+	}
+
+	lb, err := c.getLoadBalancer(ctx, resourceGroup, uid)
+	if err != nil {
+		return "", fmt.Errorf("retrieving load balancer: %w", err)
+	}
+	if lb == nil || lb.Properties == nil {
+		return "", errors.New("could not dereference load balancer IP configuration")
+	}
+
+	var pubIP string
+	for _, fipConf := range lb.Properties.FrontendIPConfigurations {
+		if fipConf == nil || fipConf.Properties == nil || fipConf.Properties.PublicIPAddress == nil || fipConf.Properties.PublicIPAddress.ID == nil {
+			continue
+		}
+		pubIP = path.Base(*fipConf.Properties.PublicIPAddress.ID)
+		break
+	}
+
+	resp, err := c.pubIPAPI.Get(ctx, resourceGroup, pubIP, nil)
+	if err != nil {
+		return "", fmt.Errorf("retrieving load balancer public IP address: %w", err)
+	}
+	if resp.Properties == nil || resp.Properties.IPAddress == nil {
+		return "", fmt.Errorf("could not resolve public IP address reference for load balancer")
+	}
+	return *resp.Properties.IPAddress, nil
+}
+
+/*
+// TODO(malt3): uncomment and use as soon as we switch the primary endpoint to DNS.
+// getLoadBalancerDNSName retrieves the dns name of the load balancer.
+// On Azure, the DNS name is the DNS name of the public IP address of the load balancer.
+func (c *Cloud) getLoadBalancerDNSName(ctx context.Context) (string, error) {
+	resourceGroup, err := c.imds.resourceGroup(ctx)
+	if err != nil {
+		return "", fmt.Errorf("retrieving resource group: %w", err)
+	}
+	uid, err := c.imds.uid(ctx)
+	if err != nil {
+		return "", fmt.Errorf("retrieving instance UID: %w", err)
+	}
+
+	lb, err := c.getLoadBalancer(ctx, resourceGroup, uid)
+	if err != nil {
+		return "", fmt.Errorf("retrieving load balancer: %w", err)
+	}
+	if lb == nil || lb.Properties == nil {
+		return "", errors.New("could not dereference load balancer IP configuration")
+	}
+
+	var pubIP string
+	for _, fipConf := range lb.Properties.FrontendIPConfigurations {
+		if fipConf == nil || fipConf.Properties == nil || fipConf.Properties.PublicIPAddress == nil || fipConf.Properties.PublicIPAddress.ID == nil {
+			continue
+		}
+		pubIP = path.Base(*fipConf.Properties.PublicIPAddress.ID)
+		break
+	}
+
+	resp, err := c.pubIPAPI.Get(ctx, resourceGroup, pubIP, nil)
+	if err != nil {
+		return "", fmt.Errorf("retrieving load balancer public IP address: %w", err)
+	}
+	if resp.Properties == nil || resp.Properties.DNSSettings == nil || resp.Properties.DNSSettings.Fqdn == nil {
+		return "", fmt.Errorf("could not resolve public IP address fqdn for load balancer")
+	}
+	return *resp.Properties.DNSSettings.Fqdn, nil
+}
+*/
+
 type cloudConfig struct {
 	Cloud                       string `json:"cloud,omitempty"`
 	TenantID                    string `json:"tenantId,omitempty"`

internal/cloud/azure/azure_test.go

@@ -1039,13 +1039,14 @@ func TestGetLoadBalancerEndpoint(t *testing.T) {
 				loadBalancerAPI: tc.loadBalancerAPI,
 				pubIPAPI:        tc.publicIPAddressesAPI,
 			}
-			loadbalancerName, err := metadata.GetLoadBalancerEndpoint(context.Background())
+			gotHost, gotPort, err := metadata.GetLoadBalancerEndpoint(context.Background())
 			if tc.wantErr {
 				assert.Error(err)
 				return
 			}
 			require.NoError(err)
-			assert.Equal(tc.wantIP, loadbalancerName)
+			assert.Equal(tc.wantIP, gotHost)
+			assert.Equal("6443", gotPort)
 		})
 	}
 }

internal/cloud/cloud.go

@@ -33,4 +33,7 @@ const (
 	TagUID = "constellation-uid"
 	// TagInitSecretHash is the tag/label key used to identify the hash of the init secret.
 	TagInitSecretHash = "constellation-init-secret-hash"
+	// TagCustomEndpoint is the tag/label key used to identify the custom endpoint
+	// or dns name that should be added to tls cert SANs.
+	TagCustomEndpoint = "constellation-custom-endpoint"
 )

internal/cloud/gcp/gcp.go

@@ -19,7 +19,6 @@ import (
 	"context"
 	"errors"
 	"fmt"
-	"net"
 	"path"
 	"regexp"
 	"strings"
@@ -124,43 +123,44 @@ func (c *Cloud) Close() {
 }
 
 // GetLoadBalancerEndpoint returns the endpoint of the load balancer.
-func (c *Cloud) GetLoadBalancerEndpoint(ctx context.Context) (string, error) {
+func (c *Cloud) GetLoadBalancerEndpoint(ctx context.Context) (host, port string, err error) {
 	project, zone, instanceName, err := c.retrieveInstanceInfo()
 	if err != nil {
-		return "", err
+		return "", "", err
 	}
 	uid, err := c.uid(ctx, project, zone, instanceName)
 	if err != nil {
-		return "", err
+		return "", "", err
 	}
 
 	// First try to find a global forwarding rule.
-	endpoint, err := c.getGlobalForwardingRule(ctx, project, uid)
+	host, port, err = c.getGlobalForwardingRule(ctx, project, uid)
 	if err != nil && !errors.Is(err, errNoForwardingRule) {
-		return "", fmt.Errorf("getting global forwarding rule: %w", err)
+		return "", "", fmt.Errorf("getting global forwarding rule: %w", err)
 	} else if err == nil {
-		return endpoint, nil
+		return host, port, nil
 	}
 
 	// If no global forwarding rule was found, try to find a regional forwarding rule.
 	region := zoneFromRegionRegex.FindString(zone)
 	if region == "" {
-		return "", fmt.Errorf("invalid zone %s", zone)
+		return "", "", fmt.Errorf("invalid zone %s", zone)
 	}
 
-	endpoint, err = c.getRegionalForwardingRule(ctx, project, uid, region)
+	host, port, err = c.getRegionalForwardingRule(ctx, project, uid, region)
 	if err != nil && !errors.Is(err, errNoForwardingRule) {
-		return "", fmt.Errorf("getting regional forwarding rule: %w", err)
+		return "", "", fmt.Errorf("getting regional forwarding rule: %w", err)
-	} else if err == nil {
+	} else if err != nil {
-		return endpoint, nil
+		return "", "", fmt.Errorf("kubernetes load balancer with UID %s not found: %w", uid, err)
 	}
 
-	return "", fmt.Errorf("kubernetes load balancer with UID %s not found: %w", uid, err)
+	return host, port, nil
 }
 
 // getGlobalForwardingRule returns the endpoint of the load balancer if it is a global load balancer.
+// It returns the host, port and optionally an error.
 // This functions returns ErrNoForwardingRule if no forwarding rule was found.
-func (c *Cloud) getGlobalForwardingRule(ctx context.Context, project, uid string) (string, error) {
+func (c *Cloud) getGlobalForwardingRule(ctx context.Context, project, uid string) (string, string, error) {
 	var resp *computepb.ForwardingRule
 	var err error
 	iter := c.globalForwardingRulesAPI.List(ctx, &computepb.ListGlobalForwardingRulesRequest{
@@ -175,19 +175,19 @@ func (c *Cloud) getGlobalForwardingRule(ctx context.Context, project, uid string
 			continue
 		}
 		portRange := strings.Split(*resp.PortRange, "-")
-		return net.JoinHostPort(*resp.IPAddress, portRange[0]), nil
+		return *resp.IPAddress, portRange[0], nil
 	}
 	if err != iterator.Done {
-		return "", fmt.Errorf("error listing global forwarding rules with UID %s: %w", uid, err)
+		return "", "", fmt.Errorf("error listing global forwarding rules with UID %s: %w", uid, err)
 	}
-	return "", errNoForwardingRule
+	return "", "", errNoForwardingRule
 }
 
 // getRegionalForwardingRule returns the endpoint of the load balancer if it is a regional load balancer.
+// It returns the host, port and optionally an error.
 // This functions returns ErrNoForwardingRule if no forwarding rule was found.
-func (c *Cloud) getRegionalForwardingRule(ctx context.Context, project, uid, region string) (string, error) {
+func (c *Cloud) getRegionalForwardingRule(ctx context.Context, project, uid, region string) (host string, port string, err error) {
 	var resp *computepb.ForwardingRule
-	var err error
 	iter := c.regionalForwardingRulesAPI.List(ctx, &computepb.ListForwardingRulesRequest{
 		Project: project,
 		Region:  region,
@@ -204,12 +204,12 @@ func (c *Cloud) getRegionalForwardingRule(ctx context.Context, project, uid, reg
 			continue
 		}
 		portRange := strings.Split(*resp.PortRange, "-")
-		return net.JoinHostPort(*resp.IPAddress, portRange[0]), nil
+		return *resp.IPAddress, portRange[0], nil
 	}
 	if err != iterator.Done {
-		return "", fmt.Errorf("error listing global forwarding rules with UID %s: %w", uid, err)
+		return "", "", fmt.Errorf("error listing global forwarding rules with UID %s: %w", uid, err)
 	}
-	return "", errNoForwardingRule
+	return "", "", errNoForwardingRule
 }
 
 // List retrieves all instances belonging to the current constellation.

internal/cloud/gcp/gcp_test.go

@@ -211,7 +211,7 @@ func TestGetLoadbalancerEndpoint(t *testing.T) {
 		instanceAPI                stubInstanceAPI
 		globalForwardingRulesAPI   stubGlobalForwardingRulesAPI
 		regionalForwardingRulesAPI stubRegionalForwardingRulesAPI
-		wantEndpoint               string
+		wantHost                   string
 		wantErr                    bool
 	}{
 		"success global forwarding rule": {
@@ -236,7 +236,7 @@ func TestGetLoadbalancerEndpoint(t *testing.T) {
 			regionalForwardingRulesAPI: stubRegionalForwardingRulesAPI{
 				iterator: &stubForwardingRulesIterator{},
 			},
-			wantEndpoint: "192.0.2.255:6443",
+			wantHost: "192.0.2.255",
 		},
 		"success regional forwarding rule": {
 			imds: stubIMDS{
@@ -261,7 +261,7 @@ func TestGetLoadbalancerEndpoint(t *testing.T) {
 					},
 				},
 			},
-			wantEndpoint: "192.0.2.255:6443",
+			wantHost: "192.0.2.255",
 		},
 		"regional forwarding rule has no region": {
 			imds: stubIMDS{
@@ -473,13 +473,14 @@ func TestGetLoadbalancerEndpoint(t *testing.T) {
 				regionalForwardingRulesAPI: &tc.regionalForwardingRulesAPI,
 			}
 
-			endpoint, err := cloud.GetLoadBalancerEndpoint(context.Background())
+			gotHost, gotPort, err := cloud.GetLoadBalancerEndpoint(context.Background())
 			if tc.wantErr {
 				assert.Error(err)
 				return
 			}
 			assert.NoError(err)
-			assert.Equal(tc.wantEndpoint, endpoint)
+			assert.Equal(tc.wantHost, gotHost)
+			assert.Equal("6443", gotPort)
 		})
 	}
 }

internal/cloud/openstack/BUILD.bazel

@@ -16,6 +16,7 @@ go_library(
     deps = [
         "//internal/cloud",
         "//internal/cloud/metadata",
+        "//internal/constants",
         "//internal/role",
         "@com_github_gophercloud_gophercloud//:gophercloud",
         "@com_github_gophercloud_gophercloud//openstack/compute/v2/servers",

internal/cloud/openstack/openstack.go

@@ -12,9 +12,11 @@ import (
 	"fmt"
 	"net/http"
 	"net/netip"
+	"strconv"
 	"strings"
 
 	"github.com/edgelesssys/constellation/v2/internal/cloud/metadata"
+	"github.com/edgelesssys/constellation/v2/internal/constants"
 	"github.com/edgelesssys/constellation/v2/internal/role"
 	"github.com/gophercloud/gophercloud/openstack/compute/v2/servers"
 	"github.com/gophercloud/gophercloud/openstack/networking/v2/subnets"
@@ -230,7 +232,15 @@ func (c *Cloud) InitSecretHash(ctx context.Context) ([]byte, error) {
 // For OpenStack, the load balancer is a floating ip attached to
 // a control plane node.
 // TODO(malt3): Rewrite to use real load balancer once it is available.
-func (c *Cloud) GetLoadBalancerEndpoint(ctx context.Context) (string, error) {
+func (c *Cloud) GetLoadBalancerEndpoint(ctx context.Context) (host, port string, err error) {
+	host, err = c.getLoadBalancerHost(ctx)
+	if err != nil {
+		return "", "", fmt.Errorf("getting load balancer host: %w", err)
+	}
+	return host, strconv.FormatInt(constants.KubernetesPort, 10), nil
+}
+
+func (c *Cloud) getLoadBalancerHost(ctx context.Context) (string, error) {
 	uid, err := c.imds.uid(ctx)
 	if err != nil {
 		return "", fmt.Errorf("getting uid: %w", err)

internal/cloud/openstack/openstack_test.go

@@ -489,7 +489,7 @@ func TestGetLoadBalancerEndpoint(t *testing.T) {
 	testCases := map[string]struct {
 		imds     *stubIMDSClient
 		api      *stubServersClient
-		want    string
+		wantHost string
 		wantErr  bool
 	}{
 		"error returned from IMDS client": {
@@ -613,7 +613,7 @@ func TestGetLoadBalancerEndpoint(t *testing.T) {
 					},
 				}, nil),
 			},
-			want: "198.51.100.0",
+			wantHost: "198.51.100.0",
 		},
 		"first valid endpoint returned from server addresses not in subnet CIDR": {
 			imds: &stubIMDSClient{},
@@ -628,7 +628,7 @@ func TestGetLoadBalancerEndpoint(t *testing.T) {
 					},
 				}, nil),
 			},
-			want: "198.51.100.0",
+			wantHost: "198.51.100.0",
 		},
 	}
 
@@ -641,13 +641,14 @@ func TestGetLoadBalancerEndpoint(t *testing.T) {
 				api:  tc.api,
 			}
 
-			got, err := c.GetLoadBalancerEndpoint(context.Background())
+			gotHost, gotPort, err := c.GetLoadBalancerEndpoint(context.Background())
 
 			if tc.wantErr {
 				assert.Error(err)
 			} else {
 				assert.NoError(err)
-				assert.Equal(tc.want, got)
+				assert.Equal(tc.wantHost, gotHost)
+				assert.Equal("6443", gotPort)
 			}
 		})
 	}

internal/cloud/qemu/BUILD.bazel

@@ -8,5 +8,8 @@ go_library(
     ],
     importpath = "github.com/edgelesssys/constellation/v2/internal/cloud/qemu",
     visibility = ["//:__subpackages__"],
-    deps = ["//internal/cloud/metadata"],
+    deps = [
+        "//internal/cloud/metadata",
+        "//internal/constants",
+    ],
 )

internal/cloud/qemu/qemu.go

@@ -16,8 +16,10 @@ import (
 	"io"
 	"net/http"
 	"net/url"
+	"strconv"
 
 	"github.com/edgelesssys/constellation/v2/internal/cloud/metadata"
+	"github.com/edgelesssys/constellation/v2/internal/constants"
 )
 
 const qemuMetadataEndpoint = "10.42.0.1:8080"
@@ -56,7 +58,15 @@ func (c *Cloud) Self(ctx context.Context) (metadata.InstanceMetadata, error) {
 
 // GetLoadBalancerEndpoint returns the endpoint of the load balancer.
 // For QEMU, the load balancer is the first control plane node returned by the metadata API.
-func (c *Cloud) GetLoadBalancerEndpoint(ctx context.Context) (string, error) {
+func (c *Cloud) GetLoadBalancerEndpoint(ctx context.Context) (host, port string, err error) {
+	host, err = c.getLoadBalancerHost(ctx)
+	if err != nil {
+		return "", "", fmt.Errorf("getting load balancer host: %w", err)
+	}
+	return host, strconv.FormatInt(constants.KubernetesPort, 10), nil
+}
+
+func (c *Cloud) getLoadBalancerHost(ctx context.Context) (string, error) {
 	endpointRaw, err := c.retrieveMetadata(ctx, "/endpoint")
 	if err != nil {
 		return "", err

internal/config/config.go

@@ -85,6 +85,12 @@ type Config struct {
 	// description: |
 	//   Configuration for attestation validation. This configuration provides sensible defaults for the Constellation version it was created for.\nSee the docs for an overview on attestation: https://docs.edgeless.systems/constellation/architecture/attestation
 	Attestation AttestationConfig `yaml:"attestation" validate:"dive"`
+	// description: |
+	//   Optional custom endpoint (DNS name) for the Constellation API server.
+	//   This can be used to point a custom dns name at the Constellation API server
+	//   and is added to the Subject Alternative Name (SAN) field of the TLS certificate used by the API server.
+	//   A fallback to DNS name is always available.
+	CustomEndpoint string `yaml:"customEndpoint" validate:"omitempty,hostname_rfc1123"`
 }
 
 // ProviderConfig are cloud-provider specific configuration values used by the CLI.

internal/config/config_doc.go

@@ -34,7 +34,7 @@ func init() {
 	ConfigDoc.Type = "Config"
 	ConfigDoc.Comments[encoder.LineComment] = "Config defines configuration used by CLI."
 	ConfigDoc.Description = "Config defines configuration used by CLI."
-	ConfigDoc.Fields = make([]encoder.Doc, 9)
+	ConfigDoc.Fields = make([]encoder.Doc, 10)
 	ConfigDoc.Fields[0].Name = "version"
 	ConfigDoc.Fields[0].Type = "string"
 	ConfigDoc.Fields[0].Note = ""
@@ -80,6 +80,11 @@ func init() {
 	ConfigDoc.Fields[8].Note = ""
 	ConfigDoc.Fields[8].Description = "Configuration for attestation validation. This configuration provides sensible defaults for the Constellation version it was created for.\nSee the docs for an overview on attestation: https://docs.edgeless.systems/constellation/architecture/attestation"
 	ConfigDoc.Fields[8].Comments[encoder.LineComment] = "Configuration for attestation validation. This configuration provides sensible defaults for the Constellation version it was created for.\nSee the docs for an overview on attestation: https://docs.edgeless.systems/constellation/architecture/attestation"
+	ConfigDoc.Fields[9].Name = "customEndpoint"
+	ConfigDoc.Fields[9].Type = "string"
+	ConfigDoc.Fields[9].Note = ""
+	ConfigDoc.Fields[9].Description = "Optional custom endpoint (DNS name) for the Constellation API server.\nThis can be used to point a custom dns name at the Constellation API server\nand is added to the Subject Alternative Name (SAN) field of the TLS certificate used by the API server.\nA fallback to DNS name is always available."
+	ConfigDoc.Fields[9].Comments[encoder.LineComment] = "Optional custom endpoint (DNS name) for the Constellation API server."
 
 	ProviderConfigDoc.Type = "ProviderConfig"
 	ProviderConfigDoc.Comments[encoder.LineComment] = "ProviderConfig are cloud-provider specific configuration values used by the CLI."

internal/constants/constants.go

@@ -175,6 +175,11 @@ const (
 	JoinConfigMap = "join-config"
 	// InternalConfigMap k8s config map with internal Constellation config.
 	InternalConfigMap = "internal-config"
+	// KubeadmConfigMap k8s config map with kubeadm config
+	// (holds ClusterConfiguration).
+	KubeadmConfigMap = "kubeadm-config"
+	// ClusterConfigurationKey key in kubeadm config map with ClusterConfiguration.
+	ClusterConfigurationKey = "ClusterConfiguration"
 
 	//
 	// Helm.