Git-Mirrors/constellation
1
0
Fork 0
mirror of https://github.com/edgelesssys/constellation.git synced 2024-10-01 01:36:09 -04:00
Code Issues Packages Projects Releases Wiki Activity

bootstrapper: add fallback endpoint and custom endpoint to SAN field (#2108)

Browse Source 
terraform: collect apiserver cert SANs and support custom endpoint

constants: add new constants for cluster configuration and custom endpoint

cloud: support apiserver cert sans and prepare for endpoint migration on AWS

config: add customEndpoint field

bootstrapper: use per-CSP apiserver cert SANs

cli: route customEndpoint to terraform and add migration for apiserver cert SANs

bootstrapper: change interface of GetLoadBalancerEndpoint to return host and port separately
This commit is contained in:
Malte Poll 2023-07-21 16:43:51 +02:00 committed by GitHub
parent 3324a4eba2
commit 8da6a23aa5
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
64 changed files with 724 additions and 301 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
bootstrapper/cmd/bootstrapper/run.go
View File

@ -99,5 +99,5 @@ type clusterInitJoiner interface {
type metadataAPI interface {
joinclient.MetadataAPI
initserver.MetadataAPI
GetLoadBalancerEndpoint(ctx context.Context) (string, error)
GetLoadBalancerEndpoint(ctx context.Context) (host, port string, err error)
}

6
bootstrapper/cmd/bootstrapper/test.go
View File

@ -22,7 +22,7 @@ type clusterFake struct{}
// InitCluster fakes bootstrapping a new cluster with the current node being the master, returning the arguments required to join the cluster.
func (c *clusterFake) InitCluster(
context.Context, string, string, string, []byte,
[]byte, bool, components.Components, *logger.Logger,
[]byte, bool, components.Components, []string, *logger.Logger,
) ([]byte, error) {
return []byte{}, nil
}
@ -53,8 +53,8 @@ func (f *providerMetadataFake) Self(_ context.Context) (metadata.InstanceMetadat
}, nil
}
func (f *providerMetadataFake) GetLoadBalancerEndpoint(_ context.Context) (string, error) {
return "", nil
func (f *providerMetadataFake) GetLoadBalancerEndpoint(_ context.Context) (string, string, error) {
return "", "", nil
}
func (f *providerMetadataFake) InitSecretHash(_ context.Context) ([]byte, error) {

15
bootstrapper/initproto/init.pb.go
View File

@ -38,6 +38,7 @@ type InitRequest struct {
KubernetesComponents []*KubernetesComponent `protobuf:"bytes,15,rep,name=kubernetes_components,json=kubernetesComponents,proto3" json:"kubernetes_components,omitempty"`
InitSecret []byte `protobuf:"bytes,16,opt,name=init_secret,json=initSecret,proto3" json:"init_secret,omitempty"`
ClusterName string `protobuf:"bytes,17,opt,name=cluster_name,json=clusterName,proto3" json:"cluster_name,omitempty"`
ApiserverCertSans []string `protobuf:"bytes,18,rep,name=apiserver_cert_sans,json=apiserverCertSans,proto3" json:"apiserver_cert_sans,omitempty"`
}
func (x *InitRequest) Reset() {
@ -135,6 +136,13 @@ func (x *InitRequest) GetClusterName() string {
return ""
}
func (x *InitRequest) GetApiserverCertSans() []string {
if x != nil {
return x.ApiserverCertSans
}
return nil
}
type InitResponse struct {
state protoimpl.MessageState
sizeCache protoimpl.SizeCache
@ -463,7 +471,7 @@ var File_bootstrapper_initproto_init_proto protoreflect.FileDescriptor
var file_bootstrapper_initproto_init_proto_rawDesc = []byte{
0x0a, 0x21, 0x62, 0x6f, 0x6f, 0x74, 0x73, 0x74, 0x72, 0x61, 0x70, 0x70, 0x65, 0x72, 0x2f, 0x69,
0x6e, 0x69, 0x74, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x69, 0x6e, 0x69, 0x74, 0x2e, 0x70, 0x72,
0x6f, 0x74, 0x6f, 0x12, 0x04, 0x69, 0x6e, 0x69, 0x74, 0x22, 0x9b, 0x03, 0x0a, 0x0b, 0x49, 0x6e,
0x6f, 0x74, 0x6f, 0x12, 0x04, 0x69, 0x6e, 0x69, 0x74, 0x22, 0xcb, 0x03, 0x0a, 0x0b, 0x49, 0x6e,
0x69, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x17, 0x0a, 0x07, 0x6b, 0x6d, 0x73,
0x5f, 0x75, 0x72, 0x69, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x6b, 0x6d, 0x73, 0x55,
0x72, 0x69, 0x12, 0x1f, 0x0a, 0x0b, 0x73, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x5f, 0x75, 0x72,
@ -489,7 +497,10 @@ var file_bootstrapper_initproto_init_proto_rawDesc = []byte{
0x65, 0x74, 0x18, 0x10, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0a, 0x69, 0x6e, 0x69, 0x74, 0x53, 0x65,
0x63, 0x72, 0x65, 0x74, 0x12, 0x21, 0x0a, 0x0c, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x5f,
0x6e, 0x61, 0x6d, 0x65, 0x18, 0x11, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x63, 0x6c, 0x75, 0x73,
0x74, 0x65, 0x72, 0x4e, 0x61, 0x6d, 0x65, 0x22, 0xc1, 0x01, 0x0a, 0x0c, 0x49, 0x6e, 0x69, 0x74,
0x74, 0x65, 0x72, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x2e, 0x0a, 0x13, 0x61, 0x70, 0x69, 0x73, 0x65,
0x72, 0x76, 0x65, 0x72, 0x5f, 0x63, 0x65, 0x72, 0x74, 0x5f, 0x73, 0x61, 0x6e, 0x73, 0x18, 0x12,
0x20, 0x03, 0x28, 0x09, 0x52, 0x11, 0x61, 0x70, 0x69, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x43,
0x65, 0x72, 0x74, 0x53, 0x61, 0x6e, 0x73, 0x22, 0xc1, 0x01, 0x0a, 0x0c, 0x49, 0x6e, 0x69, 0x74,
0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x3e, 0x0a, 0x0c, 0x69, 0x6e, 0x69, 0x74,
0x5f, 0x73, 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x19,
0x2e, 0x69, 0x6e, 0x69, 0x74, 0x2e, 0x49, 0x6e, 0x69, 0x74, 0x53, 0x75, 0x63, 0x63, 0x65, 0x73,

1
bootstrapper/initproto/init.proto
View File

@ -26,6 +26,7 @@ message InitRequest {
repeated KubernetesComponent kubernetes_components = 15;
bytes init_secret = 16;
string cluster_name = 17;
repeated string apiserver_cert_sans = 18;
}
message InitResponse {

26
bootstrapper/internal/helm/helm.go
View File

@ -12,9 +12,7 @@ import (
"context"
"errors"
"fmt"
"net"
"os/exec"
"strconv"
"strings"
"time"
@ -109,9 +107,9 @@ func (h *Client) InstallCilium(ctx context.Context, kubectl k8sapi.Client, relea
switch in.CloudProvider {
case "aws", "azure", "openstack", "qemu":
return h.installCiliumGeneric(ctx, release, in.LoadBalancerEndpoint)
return h.installCiliumGeneric(ctx, release, in.LoadBalancerHost, in.LoadBalancerPort)
case "gcp":
return h.installCiliumGCP(ctx, release, in.NodeName, in.FirstNodePodCIDR, in.SubnetworkPodCIDR, in.LoadBalancerEndpoint)
return h.installCiliumGCP(ctx, release, in.NodeName, in.FirstNodePodCIDR, in.SubnetworkPodCIDR, in.LoadBalancerHost, in.LoadBalancerPort)
default:
return fmt.Errorf("unsupported cloud provider %q", in.CloudProvider)
}
@ -120,33 +118,25 @@ func (h *Client) InstallCilium(ctx context.Context, kubectl k8sapi.Client, relea
// installCiliumGeneric installs cilium with the given load balancer endpoint.
// This is used for cloud providers that do not require special server-side configuration.
// Currently this is AWS, Azure, and QEMU.
func (h *Client) installCiliumGeneric(ctx context.Context, release helm.Release, kubeAPIEndpoint string) error {
host := kubeAPIEndpoint
release.Values["k8sServiceHost"] = host
release.Values["k8sServicePort"] = strconv.Itoa(constants.KubernetesPort)
func (h *Client) installCiliumGeneric(ctx context.Context, release helm.Release, kubeAPIHost, kubeAPIPort string) error {
release.Values["k8sServiceHost"] = kubeAPIHost
release.Values["k8sServicePort"] = kubeAPIPort
return h.install(ctx, release.Chart, release.Values)
}
func (h *Client) installCiliumGCP(ctx context.Context, release helm.Release, nodeName, nodePodCIDR, subnetworkPodCIDR, kubeAPIEndpoint string) error {
func (h *Client) installCiliumGCP(ctx context.Context, release helm.Release, nodeName, nodePodCIDR, subnetworkPodCIDR, kubeAPIHost, kubeAPIPort string) error {
out, err := exec.CommandContext(ctx, constants.KubectlPath, "--kubeconfig", constants.ControlPlaneAdminConfFilename, "patch", "node", nodeName, "-p", "{\"spec\":{\"podCIDR\": \""+nodePodCIDR+"\"}}").CombinedOutput()
if err != nil {
err = errors.New(string(out))
return err
}
host, port, err := net.SplitHostPort(kubeAPIEndpoint)
if err != nil {
return err
}
// configure pod network CIDR
release.Values["ipv4NativeRoutingCIDR"] = subnetworkPodCIDR
release.Values["strictModeCIDR"] = subnetworkPodCIDR
release.Values["k8sServiceHost"] = host
if port != "" {
release.Values["k8sServicePort"] = port
}
release.Values["k8sServiceHost"] = kubeAPIHost
release.Values["k8sServicePort"] = kubeAPIPort
return h.install(ctx, release.Chart, release.Values)
}

2
bootstrapper/internal/initserver/initserver.go
View File

@ -220,6 +220,7 @@ func (s *Server) Init(req *initproto.InitRequest, stream initproto.API_InitServe
req.HelmDeployments,
req.ConformanceMode,
components.NewComponentsFromInitProto(req.KubernetesComponents),
req.ApiserverCertSans,
s.log,
)
if err != nil {
@ -348,6 +349,7 @@ type ClusterInitializer interface {
helmDeployments []byte,
conformanceMode bool,
kubernetesComponents components.Components,
apiServerCertSANs []string,
log *logger.Logger,
) ([]byte, error)
}

2
bootstrapper/internal/initserver/initserver_test.go
View File

@ -407,7 +407,7 @@ type stubClusterInitializer struct {
func (i *stubClusterInitializer) InitCluster(
context.Context, string, string, string, []byte,
[]byte, bool, components.Components, *logger.Logger,
[]byte, bool, components.Components, []string, *logger.Logger,
) ([]byte, error) {
return i.initClusterKubeconfig, i.initClusterErr
}

10
bootstrapper/internal/kubernetes/cloud_provider.go
View File

@ -19,12 +19,12 @@ type ProviderMetadata interface {
// Self retrieves the current instance.
Self(ctx context.Context) (metadata.InstanceMetadata, error)
// GetLoadBalancerEndpoint retrieves the load balancer endpoint.
GetLoadBalancerEndpoint(ctx context.Context) (string, error)
GetLoadBalancerEndpoint(ctx context.Context) (host, port string, err error)
}
type stubProviderMetadata struct {
getLoadBalancerEndpointErr error
getLoadBalancerEndpointResp string
getLoadBalancerEndpointErr error
getLoadBalancerHostResp, getLoadBalancerPortResp string
selfErr error
selfResp metadata.InstanceMetadata
@ -33,8 +33,8 @@ type stubProviderMetadata struct {
uidResp string
}
func (m *stubProviderMetadata) GetLoadBalancerEndpoint(_ context.Context) (string, error) {
return m.getLoadBalancerEndpointResp, m.getLoadBalancerEndpointErr
func (m *stubProviderMetadata) GetLoadBalancerEndpoint(_ context.Context) (string, string, error) {
return m.getLoadBalancerHostResp, m.getLoadBalancerPortResp, m.getLoadBalancerEndpointErr
}
func (m *stubProviderMetadata) Self(_ context.Context) (metadata.InstanceMetadata, error) {

17
bootstrapper/internal/kubernetes/k8sapi/k8sutil.go
View File

@ -91,7 +91,7 @@ func (k *KubernetesUtil) InstallComponents(ctx context.Context, kubernetesCompon
// InitCluster instruments kubeadm to initialize the K8s cluster.
// On success an admin kubeconfig file is returned.
func (k *KubernetesUtil) InitCluster(
ctx context.Context, initConfig []byte, nodeName, clusterName string, ips []net.IP, controlPlaneEndpoint string, conformanceMode bool, log *logger.Logger,
ctx context.Context, initConfig []byte, nodeName, clusterName string, ips []net.IP, controlPlaneHost, controlPlanePort string, conformanceMode bool, log *logger.Logger,
) ([]byte, error) {
// TODO(3u13r): audit policy should be user input
auditPolicy, err := resources.NewDefaultAuditPolicy().Marshal()
@ -148,6 +148,7 @@ func (k *KubernetesUtil) InitCluster(
}
log.Infof("Preparing node for Konnectivity")
controlPlaneEndpoint := net.JoinHostPort(controlPlaneHost, controlPlanePort)
if err := k.prepareControlPlaneForKonnectivity(ctx, controlPlaneEndpoint); err != nil {
return nil, fmt.Errorf("setup konnectivity: %w", err)
}
@ -241,11 +242,12 @@ func (k *KubernetesUtil) prepareControlPlaneForKonnectivity(ctx context.Context,
// SetupPodNetworkInput holds all configuration options to setup the pod network.
type SetupPodNetworkInput struct {
CloudProvider string
NodeName string
FirstNodePodCIDR string
SubnetworkPodCIDR string
LoadBalancerEndpoint string
CloudProvider string
NodeName string
FirstNodePodCIDR string
SubnetworkPodCIDR string
LoadBalancerHost string
LoadBalancerPort string
}
// WaitForCilium waits until Cilium reports a healthy status over its /healthz endpoint.
@ -314,7 +316,7 @@ func (k *KubernetesUtil) FixCilium(ctx context.Context) error {
}
// JoinCluster joins existing Kubernetes cluster using kubeadm join.
func (k *KubernetesUtil) JoinCluster(ctx context.Context, joinConfig []byte, peerRole role.Role, controlPlaneEndpoint string, log *logger.Logger) error {
func (k *KubernetesUtil) JoinCluster(ctx context.Context, joinConfig []byte, peerRole role.Role, controlPlaneHost, controlPlanePort string, log *logger.Logger) error {
// TODO(3u13r): audit policy should be user input
auditPolicy, err := resources.NewDefaultAuditPolicy().Marshal()
if err != nil {
@ -341,6 +343,7 @@ func (k *KubernetesUtil) JoinCluster(ctx context.Context, joinConfig []byte, pee
if peerRole == role.ControlPlane {
log.Infof("Prep Init Kubernetes cluster")
controlPlaneEndpoint := net.JoinHostPort(controlPlaneHost, controlPlanePort)
if err := k.prepareControlPlaneForKonnectivity(ctx, controlPlaneEndpoint); err != nil {
return fmt.Errorf("setup konnectivity: %w", err)
}

4
bootstrapper/internal/kubernetes/k8sutil.go
View File

@ -19,8 +19,8 @@ import (
type clusterUtil interface {
InstallComponents(ctx context.Context, kubernetesComponents components.Components) error
InitCluster(ctx context.Context, initConfig []byte, nodeName, clusterName string, ips []net.IP, controlPlaneEndpoint string, conformanceMode bool, log *logger.Logger) ([]byte, error)
JoinCluster(ctx context.Context, joinConfig []byte, peerRole role.Role, controlPlaneEndpoint string, log *logger.Logger) error
InitCluster(ctx context.Context, initConfig []byte, nodeName, clusterName string, ips []net.IP, controlPlaneHost, controlPlanePort string, conformanceMode bool, log *logger.Logger) ([]byte, error)
JoinCluster(ctx context.Context, joinConfig []byte, peerRole role.Role, controlPlaneHost, controlPlanePort string, log *logger.Logger) error
WaitForCilium(ctx context.Context, log *logger.Logger) error
FixCilium(ctx context.Context) error
StartKubelet() error

48
bootstrapper/internal/kubernetes/kubernetes.go
View File

@ -79,7 +79,7 @@ func New(cloudProvider string, clusterUtil clusterUtil, configProvider configura
// InitCluster initializes a new Kubernetes cluster and applies pod network provider.
func (k *KubeWrapper) InitCluster(
ctx context.Context, cloudServiceAccountURI, versionString, clusterName string, measurementSalt []byte,
helmReleasesRaw []byte, conformanceMode bool, kubernetesComponents components.Components, log *logger.Logger,
helmReleasesRaw []byte, conformanceMode bool, kubernetesComponents components.Components, apiServerCertSANs []string, log *logger.Logger,
) ([]byte, error) {
log.With(zap.String("version", versionString)).Infof("Installing Kubernetes components")
if err := k.clusterUtil.InstallComponents(ctx, kubernetesComponents); err != nil {
@ -110,16 +110,24 @@ func (k *KubeWrapper) InitCluster(
}
// this is the endpoint in "kubeadm init --control-plane-endpoint=<IP/DNS>:<port>"
controlPlaneEndpoint, err := k.providerMetadata.GetLoadBalancerEndpoint(ctx)
// TODO(malt3): switch over to DNS name on AWS and Azure
// soon as every apiserver certificate of every control-plane node
// has the dns endpoint in its SAN list.
controlPlaneHost, controlPlanePort, err := k.providerMetadata.GetLoadBalancerEndpoint(ctx)
if err != nil {
return nil, fmt.Errorf("retrieving load balancer endpoint: %w", err)
}
certSANs := []string{nodeIP}
certSANs = append(certSANs, apiServerCertSANs...)
log.With(
zap.String("nodeName", nodeName),
zap.String("providerID", instance.ProviderID),
zap.String("nodeIP", nodeIP),
zap.String("controlPlaneEndpoint", controlPlaneEndpoint),
zap.String("controlPlaneHost", controlPlaneHost),
zap.String("controlPlanePort", controlPlanePort),
zap.String("certSANs", strings.Join(certSANs, ",")),
zap.String("podCIDR", subnetworkPodCIDR),
).Infof("Setting information for node")
@ -130,16 +138,16 @@ func (k *KubeWrapper) InitCluster(
initConfig := k.configProvider.InitConfiguration(ccmSupported, versionString)
initConfig.SetNodeIP(nodeIP)
initConfig.SetClusterName(clusterName)
initConfig.SetCertSANs([]string{nodeIP})
initConfig.SetCertSANs(certSANs)
initConfig.SetNodeName(nodeName)
initConfig.SetProviderID(instance.ProviderID)
initConfig.SetControlPlaneEndpoint(controlPlaneEndpoint)
initConfig.SetControlPlaneEndpoint(controlPlaneHost)
initConfigYAML, err := initConfig.Marshal()
if err != nil {
return nil, fmt.Errorf("encoding kubeadm init configuration as YAML: %w", err)
}
log.Infof("Initializing Kubernetes cluster")
kubeConfig, err := k.clusterUtil.InitCluster(ctx, initConfigYAML, nodeName, clusterName, validIPs, controlPlaneEndpoint, conformanceMode, log)
kubeConfig, err := k.clusterUtil.InitCluster(ctx, initConfigYAML, nodeName, clusterName, validIPs, controlPlaneHost, controlPlanePort, conformanceMode, log)
if err != nil {
return nil, fmt.Errorf("kubeadm init: %w", err)
}
@ -171,11 +179,12 @@ func (k *KubeWrapper) InitCluster(
// Step 3: configure & start kubernetes controllers
log.Infof("Starting Kubernetes controllers and deployments")
setupPodNetworkInput := k8sapi.SetupPodNetworkInput{
CloudProvider: k.cloudProvider,
NodeName: nodeName,
FirstNodePodCIDR: nodePodCIDR,
SubnetworkPodCIDR: subnetworkPodCIDR,
LoadBalancerEndpoint: controlPlaneEndpoint,
CloudProvider: k.cloudProvider,
NodeName: nodeName,
FirstNodePodCIDR: nodePodCIDR,
SubnetworkPodCIDR: subnetworkPodCIDR,
LoadBalancerHost: controlPlaneHost,
LoadBalancerPort: controlPlanePort,
}
var helmReleases helm.Releases
@ -206,20 +215,11 @@ func (k *KubeWrapper) InitCluster(
// Continue and don't throw an error here - things might be okay.
}
var controlPlaneIP string
if strings.Contains(controlPlaneEndpoint, ":") {
controlPlaneIP, _, err = net.SplitHostPort(controlPlaneEndpoint)
if err != nil {
return nil, fmt.Errorf("parsing control plane endpoint: %w", err)
}
} else {
controlPlaneIP = controlPlaneEndpoint
}
serviceConfig := constellationServicesConfig{
measurementSalt: measurementSalt,
subnetworkPodCIDR: subnetworkPodCIDR,
cloudServiceAccountURI: cloudServiceAccountURI,
loadBalancerIP: controlPlaneIP,
loadBalancerIP: controlPlaneHost,
}
extraVals, err := k.setupExtraVals(ctx, serviceConfig)
if err != nil {
@ -300,7 +300,7 @@ func (k *KubeWrapper) JoinCluster(ctx context.Context, args *kubeadm.BootstrapTo
return fmt.Errorf("generating node name: %w", err)
}
loadbalancerEndpoint, err := k.providerMetadata.GetLoadBalancerEndpoint(ctx)
loadBalancerHost, loadBalancerPort, err := k.providerMetadata.GetLoadBalancerEndpoint(ctx)
if err != nil {
return fmt.Errorf("retrieving own instance metadata: %w", err)
}
@ -309,6 +309,8 @@ func (k *KubeWrapper) JoinCluster(ctx context.Context, args *kubeadm.BootstrapTo
zap.String("nodeName", nodeName),
zap.String("providerID", providerID),
zap.String("nodeIP", nodeInternalIP),
zap.String("loadBalancerHost", loadBalancerHost),
zap.String("loadBalancerPort", loadBalancerPort),
).Infof("Setting information for node")
// Step 2: configure kubeadm join config
@ -329,7 +331,7 @@ func (k *KubeWrapper) JoinCluster(ctx context.Context, args *kubeadm.BootstrapTo
return fmt.Errorf("encoding kubeadm join configuration as YAML: %w", err)
}
log.With(zap.String("apiServerEndpoint", args.APIServerEndpoint)).Infof("Joining Kubernetes cluster")
if err := k.clusterUtil.JoinCluster(ctx, joinConfigYAML, peerRole, loadbalancerEndpoint, log); err != nil {
if err := k.clusterUtil.JoinCluster(ctx, joinConfigYAML, peerRole, loadBalancerHost, loadBalancerPort, log); err != nil {
return fmt.Errorf("joining cluster: %v; %w ", string(joinConfigYAML), err)
}

12
bootstrapper/internal/kubernetes/kubernetes_test.go
View File

@ -63,7 +63,8 @@ func TestInitCluster(t *testing.T) {
VPCIP: privateIP,
AliasIPRanges: []string{aliasIPRange},
},
getLoadBalancerEndpointResp: loadbalancerIP,
getLoadBalancerHostResp: loadbalancerIP,
getLoadBalancerPortResp: strconv.Itoa(constants.KubernetesPort),
},
wantConfig: k8sapi.KubeadmInitYAML{
InitConfiguration: kubeadm.InitConfiguration{
@ -96,7 +97,8 @@ func TestInitCluster(t *testing.T) {
VPCIP: privateIP,
AliasIPRanges: []string{aliasIPRange},
},
getLoadBalancerEndpointResp: loadbalancerIP,
getLoadBalancerHostResp: loadbalancerIP,
getLoadBalancerPortResp: strconv.Itoa(constants.KubernetesPort),
},
kubectl: stubKubectl{annotateNodeErr: assert.AnError},
wantErr: true,
@ -191,7 +193,7 @@ func TestInitCluster(t *testing.T) {
_, err := kube.InitCluster(
context.Background(), serviceAccountURI, string(tc.k8sVersion), "kubernetes",
nil, []byte("{}"), false, nil, logger.NewTest(t),
nil, []byte("{}"), false, nil, nil, logger.NewTest(t),
)
if tc.wantErr {
@ -449,7 +451,7 @@ func (s *stubClusterUtil) InstallComponents(_ context.Context, _ components.Comp
return s.installComponentsErr
}
func (s *stubClusterUtil) InitCluster(_ context.Context, initConfig []byte, _, _ string, _ []net.IP, _ string, _ bool, _ *logger.Logger) ([]byte, error) {
func (s *stubClusterUtil) InitCluster(_ context.Context, initConfig []byte, _, _ string, _ []net.IP, _, _ string, _ bool, _ *logger.Logger) ([]byte, error) {
s.initConfigs = append(s.initConfigs, initConfig)
return s.kubeconfig, s.initClusterErr
}
@ -474,7 +476,7 @@ func (s *stubClusterUtil) SetupNodeOperator(_ context.Context, _ k8sapi.Client,
return s.setupNodeOperatorErr
}
func (s *stubClusterUtil) JoinCluster(_ context.Context, joinConfig []byte, _ role.Role, _ string, _ *logger.Logger) error {
func (s *stubClusterUtil) JoinCluster(_ context.Context, joinConfig []byte, _ role.Role, _, _ string, _ *logger.Logger) error {
s.joinConfigs = append(s.joinConfigs, joinConfig)
return s.joinClusterErr
}

2
cli/internal/cloudcmd/clients.go
View File

@ -26,7 +26,7 @@ type imageFetcher interface {
type terraformClient interface {
PrepareWorkspace(path string, input terraform.Variables) error
CreateCluster(ctx context.Context, logLevel terraform.LogLevel) (terraform.CreateOutput, error)
CreateCluster(ctx context.Context, logLevel terraform.LogLevel) (terraform.ApplyOutput, error)
CreateIAMConfig(ctx context.Context, provider cloudprovider.Provider, logLevel terraform.LogLevel) (terraform.IAMOutput, error)
Destroy(ctx context.Context, logLevel terraform.LogLevel) error
CleanUpWorkspace() error

4
cli/internal/cloudcmd/clients_test.go
View File

@ -45,8 +45,8 @@ type stubTerraformClient struct {
showErr error
}
func (c *stubTerraformClient) CreateCluster(_ context.Context, _ terraform.LogLevel) (terraform.CreateOutput, error) {
return terraform.CreateOutput{
func (c *stubTerraformClient) CreateCluster(_ context.Context, _ terraform.LogLevel) (terraform.ApplyOutput, error) {
return terraform.ApplyOutput{
IP: c.ip,
Secret: c.initSecret,
UID: c.uid,

55
cli/internal/cloudcmd/create.go
View File

@ -83,7 +83,7 @@ func (c *Creator) Create(ctx context.Context, opts CreateOptions) (clusterid.Fil
}
defer cl.RemoveInstaller()
var tfOutput terraform.CreateOutput
var tfOutput terraform.ApplyOutput
switch opts.Provider {
case cloudprovider.AWS:
@ -117,48 +117,49 @@ func (c *Creator) Create(ctx context.Context, opts CreateOptions) (clusterid.Fil
}
return clusterid.File{
CloudProvider: opts.Provider,
IP: tfOutput.IP,
InitSecret: []byte(tfOutput.Secret),
UID: tfOutput.UID,
AttestationURL: tfOutput.AttestationURL,
CloudProvider: opts.Provider,
IP: tfOutput.IP,
APIServerCertSANs: tfOutput.APIServerCertSANs,
InitSecret: []byte(tfOutput.Secret),
UID: tfOutput.UID,
AttestationURL: tfOutput.AttestationURL,
}, nil
}
func (c *Creator) createAWS(ctx context.Context, cl terraformClient, opts CreateOptions) (tfOutput terraform.CreateOutput, retErr error) {
func (c *Creator) createAWS(ctx context.Context, cl terraformClient, opts CreateOptions) (tfOutput terraform.ApplyOutput, retErr error) {
vars := awsTerraformVars(opts.Config, opts.image, &opts.ControlPlaneCount, &opts.WorkerCount)
tfOutput, err := runTerraformCreate(ctx, cl, cloudprovider.AWS, vars, c.out, opts.TFLogLevel)
if err != nil {
return terraform.CreateOutput{}, err
return terraform.ApplyOutput{}, err
}
return tfOutput, nil
}
func (c *Creator) createGCP(ctx context.Context, cl terraformClient, opts CreateOptions) (tfOutput terraform.CreateOutput, retErr error) {
func (c *Creator) createGCP(ctx context.Context, cl terraformClient, opts CreateOptions) (tfOutput terraform.ApplyOutput, retErr error) {
vars := gcpTerraformVars(opts.Config, opts.image, &opts.ControlPlaneCount, &opts.WorkerCount)
tfOutput, err := runTerraformCreate(ctx, cl, cloudprovider.GCP, vars, c.out, opts.TFLogLevel)
if err != nil {
return terraform.CreateOutput{}, err
return terraform.ApplyOutput{}, err
}
return tfOutput, nil
}
func (c *Creator) createAzure(ctx context.Context, cl terraformClient, opts CreateOptions) (tfOutput terraform.CreateOutput, retErr error) {
func (c *Creator) createAzure(ctx context.Context, cl terraformClient, opts CreateOptions) (tfOutput terraform.ApplyOutput, retErr error) {
vars := azureTerraformVars(opts.Config, opts.image, &opts.ControlPlaneCount, &opts.WorkerCount)
tfOutput, err := runTerraformCreate(ctx, cl, cloudprovider.Azure, vars, c.out, opts.TFLogLevel)
if err != nil {
return terraform.CreateOutput{}, err
return terraform.ApplyOutput{}, err
}
if vars.GetCreateMAA() {
// Patch the attestation policy to allow the cluster to boot while having secure boot disabled.
if err := c.policyPatcher.Patch(ctx, tfOutput.AttestationURL); err != nil {
return terraform.CreateOutput{}, err
return terraform.ApplyOutput{}, err
}
}
@ -196,13 +197,13 @@ func normalizeAzureURIs(vars *terraform.AzureClusterVariables) *terraform.AzureC
return vars
}
func (c *Creator) createOpenStack(ctx context.Context, cl terraformClient, opts CreateOptions) (tfOutput terraform.CreateOutput, retErr error) {
func (c *Creator) createOpenStack(ctx context.Context, cl terraformClient, opts CreateOptions) (tfOutput terraform.ApplyOutput, retErr error) {
// TODO(malt3): Remove this once OpenStack is supported.
if os.Getenv("CONSTELLATION_OPENSTACK_DEV") != "1" {
return terraform.CreateOutput{}, errors.New("OpenStack isn't supported yet")
return terraform.ApplyOutput{}, errors.New("OpenStack isn't supported yet")
}
if _, hasOSAuthURL := os.LookupEnv("OS_AUTH_URL"); !hasOSAuthURL && opts.Config.Provider.OpenStack.Cloud == "" {
return terraform.CreateOutput{}, errors.New(
return terraform.ApplyOutput{}, errors.New(
"neither environment variable OS_AUTH_URL nor cloud name for \"clouds.yaml\" is set. OpenStack authentication requires a set of " +
"OS_* environment variables that are typically sourced into the current shell with an openrc file " +
"or a cloud name for \"clouds.yaml\". " +
@ -214,21 +215,21 @@ func (c *Creator) createOpenStack(ctx context.Context, cl terraformClient, opts
tfOutput, err := runTerraformCreate(ctx, cl, cloudprovider.OpenStack, vars, c.out, opts.TFLogLevel)
if err != nil {
return terraform.CreateOutput{}, err
return terraform.ApplyOutput{}, err
}
return tfOutput, nil
}
func runTerraformCreate(ctx context.Context, cl terraformClient, provider cloudprovider.Provider, vars terraform.Variables, outWriter io.Writer, loglevel terraform.LogLevel) (output terraform.CreateOutput, retErr error) {
func runTerraformCreate(ctx context.Context, cl terraformClient, provider cloudprovider.Provider, vars terraform.Variables, outWriter io.Writer, loglevel terraform.LogLevel) (output terraform.ApplyOutput, retErr error) {
if err := cl.PrepareWorkspace(path.Join("terraform", strings.ToLower(provider.String())), vars); err != nil {
return terraform.CreateOutput{}, err
return terraform.ApplyOutput{}, err
}
defer rollbackOnError(outWriter, &retErr, &rollbackerTerraform{client: cl}, loglevel)
tfOutput, err := cl.CreateCluster(ctx, loglevel)
if err != nil {
return terraform.CreateOutput{}, err
return terraform.ApplyOutput{}, err
}
return tfOutput, nil
@ -239,7 +240,7 @@ type qemuCreateOptions struct {
CreateOptions
}
func (c *Creator) createQEMU(ctx context.Context, cl terraformClient, lv libvirtRunner, opts qemuCreateOptions) (tfOutput terraform.CreateOutput, retErr error) {
func (c *Creator) createQEMU(ctx context.Context, cl terraformClient, lv libvirtRunner, opts qemuCreateOptions) (tfOutput terraform.ApplyOutput, retErr error) {
qemuRollbacker := &rollbackerQEMU{client: cl, libvirt: lv, createdWorkspace: false}
defer rollbackOnError(c.out, &retErr, qemuRollbacker, opts.TFLogLevel)
@ -247,7 +248,7 @@ func (c *Creator) createQEMU(ctx context.Context, cl terraformClient, lv libvirt
downloader := c.newRawDownloader()
imagePath, err := downloader.Download(ctx, c.out, false, opts.source, opts.Config.Image)
if err != nil {
return terraform.CreateOutput{}, fmt.Errorf("download raw image: %w", err)
return terraform.ApplyOutput{}, fmt.Errorf("download raw image: %w", err)
}
libvirtURI := opts.Config.Provider.QEMU.LibvirtURI
@ -257,7 +258,7 @@ func (c *Creator) createQEMU(ctx context.Context, cl terraformClient, lv libvirt
// if no libvirt URI is specified, start a libvirt container
case libvirtURI == "":
if err := lv.Start(ctx, opts.Config.Name, opts.Config.Provider.QEMU.LibvirtContainerImage); err != nil {
return terraform.CreateOutput{}, fmt.Errorf("start libvirt container: %w", err)
return terraform.ApplyOutput{}, fmt.Errorf("start libvirt container: %w", err)
}
libvirtURI = libvirt.LibvirtTCPConnectURI
@ -273,11 +274,11 @@ func (c *Creator) createQEMU(ctx context.Context, cl terraformClient, lv libvirt
case strings.HasPrefix(libvirtURI, "qemu+unix://"):
unixURI, err := url.Parse(strings.TrimPrefix(libvirtURI, "qemu+unix://"))
if err != nil {
return terraform.CreateOutput{}, err
return terraform.ApplyOutput{}, err
}
libvirtSocketPath = unixURI.Query().Get("socket")
if libvirtSocketPath == "" {
return terraform.CreateOutput{}, fmt.Errorf("socket path not specified in qemu+unix URI: %s", libvirtURI)
return terraform.ApplyOutput{}, fmt.Errorf("socket path not specified in qemu+unix URI: %s", libvirtURI)
}
}
@ -293,7 +294,7 @@ func (c *Creator) createQEMU(ctx context.Context, cl terraformClient, lv libvirt
}
if err := cl.PrepareWorkspace(path.Join("terraform", strings.ToLower(cloudprovider.QEMU.String())), vars); err != nil {
return terraform.CreateOutput{}, fmt.Errorf("prepare workspace: %w", err)
return terraform.ApplyOutput{}, fmt.Errorf("prepare workspace: %w", err)
}
// Allow rollback of QEMU Terraform workspace from this point on
@ -301,7 +302,7 @@ func (c *Creator) createQEMU(ctx context.Context, cl terraformClient, lv libvirt
tfOutput, err = cl.CreateCluster(ctx, opts.TFLogLevel)
if err != nil {
return terraform.CreateOutput{}, fmt.Errorf("create cluster: %w", err)
return terraform.ApplyOutput{}, fmt.Errorf("create cluster: %w", err)
}
return tfOutput, nil

14
cli/internal/cloudcmd/terraform.go
View File

@ -64,6 +64,7 @@ func awsTerraformVars(conf *config.Config, imageRef string, controlPlaneCount, w
IAMProfileWorkerNodes: conf.Provider.AWS.IAMProfileWorkerNodes,
Debug: conf.IsDebugCluster(),
EnableSNP: conf.GetAttestationConfig().GetVariant().Equal(variant.AWSSEVSNP{}),
CustomEndpoint: conf.CustomEndpoint,
}
}
@ -98,6 +99,7 @@ func azureTerraformVars(conf *config.Config, imageRef string, controlPlaneCount,
SecureBoot: conf.Provider.Azure.SecureBoot,
UserAssignedIdentity: conf.Provider.Azure.UserAssignedIdentity,
ResourceGroup: conf.Provider.Azure.ResourceGroup,
CustomEndpoint: conf.CustomEndpoint,
}
vars = normalizeAzureURIs(vars)
@ -127,11 +129,12 @@ func gcpTerraformVars(conf *config.Config, imageRef string, controlPlaneCount, w
DiskType: conf.Provider.GCP.StateDiskType,
},
},
Project: conf.Provider.GCP.Project,
Region: conf.Provider.GCP.Region,
Zone: conf.Provider.GCP.Zone,
ImageID: imageRef,
Debug: conf.IsDebugCluster(),
Project: conf.Provider.GCP.Project,
Region: conf.Provider.GCP.Region,
Zone: conf.Provider.GCP.Zone,
ImageID: imageRef,
Debug: conf.IsDebugCluster(),
CustomEndpoint: conf.CustomEndpoint,
}
}
@ -165,6 +168,7 @@ func openStackTerraformVars(conf *config.Config, imageRef string, controlPlaneCo
StateDiskSizeGB: conf.StateDiskSizeGB,
},
},
CustomEndpoint: conf.CustomEndpoint,
}
}

7
cli/internal/clusterid/id.go
View File

@ -10,7 +10,9 @@ import (
"github.com/edgelesssys/constellation/v2/internal/cloud/cloudprovider"
)
// File contains identifying information about a cluster.
// File contains state information about a cluster.
// This information is accessible after the creation
// and can be used by further operations such as initialization and upgrades.
type File struct {
// ClusterID is the unique identifier of the cluster.
ClusterID string `json:"clusterID,omitempty"`
@ -22,6 +24,9 @@ type File struct {
CloudProvider cloudprovider.Provider `json:"cloudprovider,omitempty"`
// IP is the IP address the cluster can be reached at (often the load balancer).
IP string `json:"ip,omitempty"`
// APIServerCertSANs are subject alternative names (SAN) that are added to
// the TLS certificate of each apiserver instance.
APIServerCertSANs []string `json:"apiServerCertSANs,omitempty"`
// InitSecret is the secret the first Bootstrapper uses to verify the user.
InitSecret []byte `json:"initsecret,omitempty"`
// AttestationURL is the URL of the attestation service.

1
cli/internal/cmd/init.go
View File

@ -197,6 +197,7 @@ func (i *initCmd) initialize(cmd *cobra.Command, newDialer func(validator atls.V
ConformanceMode: flags.conformance,
InitSecret: idFile.InitSecret,
ClusterName: clusterName,
ApiserverCertSans: idFile.APIServerCertSANs,
}
i.log.Debugf("Sending initialization request")
resp, err := i.initCall(cmd.Context(), newDialer(validator), idFile.IP, req)

15
cli/internal/cmd/upgradeapply.go
View File

@ -113,6 +113,20 @@ func (u *upgradeApplyCmd) upgradeApply(cmd *cobra.Command, fileHandler file.Hand
if err := u.migrateTerraform(cmd, u.imageFetcher, conf, flags); err != nil {
return fmt.Errorf("performing Terraform migrations: %w", err)
}
// reload idFile after terraform migration
// it might have been updated by the migration
if err := fileHandler.ReadJSON(constants.ClusterIDsFileName, &idFile); err != nil {
return fmt.Errorf("reading updated cluster ID file: %w", err)
}
// extend the clusterConfig cert SANs with any of the supported endpoints:
// - (legacy) public IP
// - fallback endpoint
// - custom (user-provided) endpoint
sans := append([]string{idFile.IP, conf.CustomEndpoint}, idFile.APIServerCertSANs...)
if err := u.upgrader.ExtendClusterConfigCertSANs(cmd.Context(), sans); err != nil {
return fmt.Errorf("extending cert SANs: %w", err)
}
if conf.GetProvider() == cloudprovider.Azure || conf.GetProvider() == cloudprovider.GCP || conf.GetProvider() == cloudprovider.AWS {
var upgradeErr *compatibility.InvalidUpgradeError
@ -350,6 +364,7 @@ type cloudUpgrader interface {
UpgradeNodeVersion(ctx context.Context, conf *config.Config, force bool) error
UpgradeHelmServices(ctx context.Context, config *config.Config, timeout time.Duration, allowDestructive bool, force bool) error
UpdateAttestationConfig(ctx context.Context, newConfig config.AttestationCfg) error
ExtendClusterConfigCertSANs(ctx context.Context, alternativeNames []string) error
GetClusterAttestationConfig(ctx context.Context, variant variant.Variant) (config.AttestationCfg, *corev1.ConfigMap, error)
PlanTerraformMigrations(ctx context.Context, opts upgrade.TerraformUpgradeOptions) (bool, error)
ApplyTerraformMigrations(ctx context.Context, opts upgrade.TerraformUpgradeOptions) error

4
cli/internal/cmd/upgradeapply_test.go
View File

@ -197,6 +197,10 @@ func (u stubUpgrader) ApplyTerraformMigrations(context.Context, upgrade.Terrafor
return u.applyTerraformErr
}
func (u stubUpgrader) ExtendClusterConfigCertSANs(_ context.Context, _ []string) error {
return nil
}
// AddManualStateMigration is not used in this test.
// TODO(AB#3248): remove this method together with the definition in the interfaces.
func (u stubUpgrader) AddManualStateMigration(_ terraform.StateMigration) {

2
cli/internal/kubernetes/BUILD.bazel
View File

@ -37,6 +37,8 @@ go_library(
"@io_k8s_client_go//dynamic",
"@io_k8s_client_go//kubernetes",
"@io_k8s_client_go//tools/clientcmd",
"@io_k8s_kubernetes//cmd/kubeadm/app/apis/kubeadm/v1beta3",
"@io_k8s_sigs_yaml//:yaml",
],
)

66
cli/internal/kubernetes/upgrade.go
View File

@ -13,6 +13,7 @@ import (
"fmt"
"io"
"path/filepath"
"sort"
"strings"
"time"
@ -42,6 +43,8 @@ import (
"k8s.io/client-go/dynamic"
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/tools/clientcmd"
kubeadmv1beta3 "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta3"
"sigs.k8s.io/yaml"
)
// UpgradeCmdKind is the kind of the upgrade command (check, apply).
@ -343,6 +346,69 @@ func (u *Upgrader) GetClusterAttestationConfig(ctx context.Context, variant vari
return existingAttestationConfig, existingConf, nil
}
// ExtendClusterConfigCertSANs extends the ClusterConfig stored under "kube-system/kubeadm-config" with the given SANs.
// Existing SANs are preserved.
func (u *Upgrader) ExtendClusterConfigCertSANs(ctx context.Context, alternativeNames []string) error {
clusterConfiguration, kubeadmConfig, err := u.GetClusterConfiguration(ctx)
if err != nil {
return fmt.Errorf("getting ClusterConfig: %w", err)
}
existingSANs := make(map[string]struct{})
for _, existingSAN := range clusterConfiguration.APIServer.CertSANs {
existingSANs[existingSAN] = struct{}{}
}
var missingSANs []string
for _, san := range alternativeNames {
if _, ok := existingSANs[san]; !ok {
missingSANs = append(missingSANs, san)
}
}
if len(missingSANs) == 0 {
return nil
}
u.log.Debugf("Extending the cluster's apiserver SAN field with the following SANs: %s\n", strings.Join(missingSANs, ", "))
clusterConfiguration.APIServer.CertSANs = append(clusterConfiguration.APIServer.CertSANs, missingSANs...)
sort.Strings(clusterConfiguration.APIServer.CertSANs)
newConfigYAML, err := yaml.Marshal(clusterConfiguration)
if err != nil {
return fmt.Errorf("marshaling ClusterConfiguration: %w", err)
}
kubeadmConfig.Data[constants.ClusterConfigurationKey] = string(newConfigYAML)
u.log.Debugf("Triggering kubeadm config update now")
if _, err = u.stableInterface.UpdateConfigMap(ctx, kubeadmConfig); err != nil {
return fmt.Errorf("setting new kubeadm config: %w", err)
}
fmt.Fprintln(u.outWriter, "Successfully extended the cluster's apiserver SAN field")
return nil
}
// GetClusterConfiguration fetches the kubeadm-config configmap from the cluster, extracts the config
// and returns both the full configmap and the ClusterConfiguration.
func (u *Upgrader) GetClusterConfiguration(ctx context.Context) (kubeadmv1beta3.ClusterConfiguration, *corev1.ConfigMap, error) {
existingConf, err := u.stableInterface.GetCurrentConfigMap(ctx, constants.KubeadmConfigMap)
if err != nil {
return kubeadmv1beta3.ClusterConfiguration{}, nil, fmt.Errorf("retrieving current kubeadm-config: %w", err)
}
clusterConf, ok := existingConf.Data[constants.ClusterConfigurationKey]
if !ok {
return kubeadmv1beta3.ClusterConfiguration{}, nil, errors.New("ClusterConfiguration missing from kubeadm-config")
}
var existingClusterConfig kubeadmv1beta3.ClusterConfiguration
if err := yaml.Unmarshal([]byte(clusterConf), &existingClusterConfig); err != nil {
return kubeadmv1beta3.ClusterConfiguration{}, nil, fmt.Errorf("unmarshaling ClusterConfiguration: %w", err)
}
return existingClusterConfig, existingConf, nil
}
// applyComponentsCM applies the k8s components ConfigMap to the cluster.
func (u *Upgrader) applyComponentsCM(ctx context.Context, components *corev1.ConfigMap) error {
_, err := u.stableInterface.CreateConfigMap(ctx, components)

72
cli/internal/terraform/terraform.go
View File

@ -103,53 +103,66 @@ func (c *Client) PrepareUpgradeWorkspace(path, oldWorkingDir, newWorkingDir, bac
}
// CreateCluster creates a Constellation cluster using Terraform.
func (c *Client) CreateCluster(ctx context.Context, logLevel LogLevel) (CreateOutput, error) {
func (c *Client) CreateCluster(ctx context.Context, logLevel LogLevel) (ApplyOutput, error) {
if err := c.setLogLevel(logLevel); err != nil {
return CreateOutput{}, fmt.Errorf("set terraform log level %s: %w", logLevel.String(), err)
return ApplyOutput{}, fmt.Errorf("set terraform log level %s: %w", logLevel.String(), err)
}
if err := c.tf.Init(ctx); err != nil {
return CreateOutput{}, fmt.Errorf("terraform init: %w", err)
return ApplyOutput{}, fmt.Errorf("terraform init: %w", err)
}
if err := c.applyManualStateMigrations(ctx); err != nil {
return CreateOutput{}, fmt.Errorf("apply manual state migrations: %w", err)
return ApplyOutput{}, fmt.Errorf("apply manual state migrations: %w", err)
}
if err := c.tf.Apply(ctx); err != nil {
return CreateOutput{}, fmt.Errorf("terraform apply: %w", err)
return ApplyOutput{}, fmt.Errorf("terraform apply: %w", err)
}
tfState, err := c.tf.Show(ctx)
if err != nil {
return CreateOutput{}, fmt.Errorf("terraform show: %w", err)
return ApplyOutput{}, fmt.Errorf("terraform show: %w", err)
}
ipOutput, ok := tfState.Values.Outputs["ip"]
if !ok {
return CreateOutput{}, errors.New("no IP output found")
return ApplyOutput{}, errors.New("no IP output found")
}
ip, ok := ipOutput.Value.(string)
if !ok {
return CreateOutput{}, errors.New("invalid type in IP output: not a string")
return ApplyOutput{}, errors.New("invalid type in IP output: not a string")
}
apiServerCertSANsOutput, ok := tfState.Values.Outputs["api_server_cert_sans"]
if !ok {
return ApplyOutput{}, errors.New("no api_server_cert_sans output found")
}
apiServerCertSANsUntyped, ok := apiServerCertSANsOutput.Value.([]any)
if !ok {
return ApplyOutput{}, fmt.Errorf("invalid type in api_server_cert_sans output: %s is not a list of elements", apiServerCertSANsOutput.Type.FriendlyName())
}
apiServerCertSANs, err := toStringSlice(apiServerCertSANsUntyped)
if err != nil {
return ApplyOutput{}, fmt.Errorf("convert api_server_cert_sans output: %w", err)
}
secretOutput, ok := tfState.Values.Outputs["initSecret"]
if !ok {
return CreateOutput{}, errors.New("no initSecret output found")
return ApplyOutput{}, errors.New("no initSecret output found")
}
secret, ok := secretOutput.Value.(string)
if !ok {
return CreateOutput{}, errors.New("invalid type in initSecret output: not a string")
return ApplyOutput{}, errors.New("invalid type in initSecret output: not a string")
}
uidOutput, ok := tfState.Values.Outputs["uid"]
if !ok {
return CreateOutput{}, errors.New("no uid output found")
return ApplyOutput{}, errors.New("no uid output found")
}
uid, ok := uidOutput.Value.(string)
if !ok {
return CreateOutput{}, errors.New("invalid type in uid output: not a string")
return ApplyOutput{}, errors.New("invalid type in uid output: not a string")
}
var attestationURL string
@ -159,19 +172,22 @@ func (c *Client) CreateCluster(ctx context.Context, logLevel LogLevel) (CreateOu
}
}
return CreateOutput{
IP: ip,
Secret: secret,
UID: uid,
AttestationURL: attestationURL,
return ApplyOutput{
IP: ip,
APIServerCertSANs: apiServerCertSANs,
Secret: secret,
UID: uid,
AttestationURL: attestationURL,
}, nil
}
// CreateOutput contains the Terraform output values of a cluster creation.
type CreateOutput struct {
IP string
Secret string
UID string
// ApplyOutput contains the Terraform output values of a cluster creation
// or apply operation.
type ApplyOutput struct {
IP string
APIServerCertSANs []string
Secret string
UID string
// AttestationURL is the URL of the attestation provider.
// It is only set if the cluster is created on Azure.
AttestationURL string
@ -447,6 +463,18 @@ type StateMigration struct {
Hook func(ctx context.Context, tfClient TFMigrator) error
}
func toStringSlice(in []any) ([]string, error) {
out := make([]string, len(in))
for i, v := range in {
s, ok := v.(string)
if !ok {
return nil, fmt.Errorf("invalid type in list: item at index %v of list is not a string", i)
}
out[i] = s
}
return out, nil
}
type tfInterface interface {
Apply(context.Context, ...tfexec.ApplyOption) error
Destroy(context.Context, ...tfexec.DestroyOption) error

9
cli/internal/terraform/terraform/aws/main.tf
View File

@ -46,8 +46,13 @@ locals {
zones = distinct(sort([
for node_group in var.node_groups : node_group.zone
]))
// wildcard_lb_dns_name is the DNS name of the load balancer with a wildcard for the name.
// example: given "name-1234567890.region.elb.amazonaws.com" it will return "*.region.elb.amazonaws.com"
wildcard_lb_dns_name = replace(aws_lb.front_end.dns_name, "/^[^.]*\\./", "*.")
tags = { constellation-uid = local.uid }
tags = {
constellation-uid = local.uid,
}
}
resource "random_id" "uid" {
@ -83,7 +88,7 @@ resource "aws_eip" "lb" {
# control-plane.
for_each = toset([var.zone])
domain = "vpc"
tags = local.tags
tags = merge(local.tags, { "constellation-ip-endpoint" = each.key == var.zone ? "legacy-primary-zone" : "additional-zone" })
}
resource "aws_lb" "front_end" {

4
cli/internal/terraform/terraform/aws/outputs.tf
View File

@ -2,6 +2,10 @@ output "ip" {
value = aws_eip.lb[var.zone].public_ip
}
output "api_server_cert_sans" {
value = sort(concat([aws_eip.lb[var.zone].public_ip, local.wildcard_lb_dns_name], var.custom_endpoint == "" ? [] : [var.custom_endpoint]))
}
output "uid" {
value = local.uid
}

6
cli/internal/terraform/terraform/aws/variables.tf
View File

@ -63,3 +63,9 @@ variable "enable_snp" {
default = true
description = "Enable AMD SEV SNP. Setting this to true sets the cpu-option AmdSevSnp to enable."
}
variable "custom_endpoint" {
type = string
default = ""
description = "Custom endpoint to use for the Kubernetes apiserver. If not set, the default endpoint will be used."
}

14
cli/internal/terraform/terraform/azure/main.tf
View File

@ -20,10 +20,12 @@ provider "azurerm" {
}
locals {
uid = random_id.uid.hex
name = "${var.name}-${local.uid}"
initSecretHash = random_password.initSecret.bcrypt_hash
tags = { constellation-uid = local.uid }
uid = random_id.uid.hex
name = "${var.name}-${local.uid}"
initSecretHash = random_password.initSecret.bcrypt_hash
tags = {
constellation-uid = local.uid,
}
ports_node_range = "30000-32767"
ports_kubernetes = "6443"
ports_bootstrapper = "9000"
@ -33,6 +35,9 @@ locals {
ports_debugd = "4000"
cidr_vpc_subnet_nodes = "192.168.178.0/24"
cidr_vpc_subnet_pods = "10.10.0.0/16"
// wildcard_lb_dns_name is the DNS name of the load balancer with a wildcard for the name.
// example: given "name-1234567890.location.cloudapp.azure.com" it will return "*.location.cloudapp.azure.com"
wildcard_lb_dns_name = replace(azurerm_public_ip.loadbalancer_ip.fqdn, "/^[^.]*\\./", "*.")
}
resource "random_id" "uid" {
@ -72,6 +77,7 @@ resource "azurerm_application_insights" "insights" {
resource "azurerm_public_ip" "loadbalancer_ip" {
name = "${local.name}-lb"
domain_name_label = local.name
resource_group_name = var.resource_group
location = var.location
allocation_method = "Static"

4
cli/internal/terraform/terraform/azure/outputs.tf
View File

@ -2,6 +2,10 @@ output "ip" {
value = azurerm_public_ip.loadbalancer_ip.ip_address
}
output "api_server_cert_sans" {
value = sort(concat([azurerm_public_ip.loadbalancer_ip.ip_address, local.wildcard_lb_dns_name], var.custom_endpoint == "" ? [] : [var.custom_endpoint]))
}
output "uid" {
value = local.uid
}

6
cli/internal/terraform/terraform/azure/variables.tf
View File

@ -61,3 +61,9 @@ variable "user_assigned_identity" {
type = string
description = "The name of the user assigned identity to attache to the nodes of the cluster."
}
variable "custom_endpoint" {
type = string
default = ""
description = "Custom endpoint to use for the Kubernetes apiserver. If not set, the default endpoint will be used."
}

11
cli/internal/terraform/terraform/gcp/main.tf
View File

@ -30,10 +30,12 @@ provider "google-beta" {
}
locals {
uid = random_id.uid.hex
name = "${var.name}-${local.uid}"
initSecretHash = random_password.initSecret.bcrypt_hash
labels = { constellation-uid = local.uid }
uid = random_id.uid.hex
name = "${var.name}-${local.uid}"
initSecretHash = random_password.initSecret.bcrypt_hash
labels = {
constellation-uid = local.uid,
}
ports_node_range = "30000-32767"
ports_kubernetes = "6443"
ports_bootstrapper = "9000"
@ -170,6 +172,7 @@ module "instance_group" {
named_ports = each.value.role == "control-plane" ? local.control_plane_named_ports : []
labels = local.labels
init_secret_hash = local.initSecretHash
custom_endpoint = var.custom_endpoint
}
resource "google_compute_global_address" "loadbalancer_ip" {

5
cli/internal/terraform/terraform/gcp/modules/instance_group/variables.tf
View File

@ -94,3 +94,8 @@ variable "zone" {
type = string
description = "Zone to deploy the instance group in."
}
variable "custom_endpoint" {
type = string
description = "Custom endpoint to use for the Kubernetes apiserver. If not set, the default endpoint will be used."
}

8
cli/internal/terraform/terraform/gcp/outputs.tf
View File

@ -2,6 +2,14 @@ output "ip" {
value = google_compute_global_address.loadbalancer_ip.address
}
output "api_server_cert_sans" {
value = sort(concat([google_compute_global_address.loadbalancer_ip.address], var.custom_endpoint == "" ? [] : [var.custom_endpoint]))
}
output "fallback_endpoint" {
value = google_compute_global_address.loadbalancer_ip.address
}
output "uid" {
value = local.uid
}

6
cli/internal/terraform/terraform/gcp/variables.tf
View File

@ -45,3 +45,9 @@ variable "debug" {
default = false
description = "Enable debug mode. This opens up a debugd port that can be used to deploy a custom bootstrapper."
}
variable "custom_endpoint" {
type = string
default = ""
description = "Custom endpoint to use for the Kubernetes apiserver. If not set, the default endpoint will be used."
}

4
cli/internal/terraform/terraform/openstack/outputs.tf
View File

@ -2,6 +2,10 @@ output "ip" {
value = openstack_networking_floatingip_v2.public_ip.address
}
output "api_server_cert_sans" {
value = sort(concat([openstack_networking_floatingip_v2.public_ip.address], var.custom_endpoint == "" ? [] : [var.custom_endpoint]))
}
output "uid" {
value = local.uid
}

6
cli/internal/terraform/terraform/openstack/variables.tf
View File

@ -67,3 +67,9 @@ variable "debug" {
default = false
description = "Enable debug mode. This opens up a debugd port that can be used to deploy a custom bootstrapper."
}
variable "custom_endpoint" {
type = string
default = ""
description = "Custom endpoint to use for the Kubernetes apiserver. If not set, the default endpoint will be used."
}

4
cli/internal/terraform/terraform/qemu/outputs.tf
View File

@ -2,6 +2,10 @@ output "ip" {
value = module.node_group["control_plane_default"].instance_ips[0]
}
output "api_server_cert_sans" {
value = sort(concat([module.node_group["control_plane_default"].instance_ips[0]], var.custom_endpoint == "" ? [] : [var.custom_endpoint]))
}
output "uid" {
value = "qemu" // placeholder
}

6
cli/internal/terraform/terraform/qemu/variables.tf
View File

@ -96,3 +96,9 @@ variable "name" {
default = "constellation"
description = "name prefix of the cluster VMs"
}
variable "custom_endpoint" {
type = string
default = ""
description = "Custom endpoint to use for the Kubernetes apiserver. If not set, the default endpoint will be used."
}

6
cli/internal/terraform/terraform_test.go
View File

@ -221,6 +221,9 @@ func TestCreateCluster(t *testing.T) {
"uid": {
Value: "12345abc",
},
"api_server_cert_sans": {
Value: []any{"192.0.2.100"},
},
},
},
}
@ -242,6 +245,9 @@ func TestCreateCluster(t *testing.T) {
"attestationURL": {
Value: "https://12345.neu.attest.azure.net",
},
"api_server_cert_sans": {
Value: []any{"192.0.2.100"},
},
},
},
}

33
cli/internal/terraform/variables.go
View File

@ -30,29 +30,6 @@ type ClusterVariables interface {
GetCreateMAA() bool
}
// CommonVariables is user configuration for creating a cluster with Terraform.
type CommonVariables struct {
// Name of the cluster.
Name string
// CountControlPlanes is the number of control-plane nodes to create.
CountControlPlanes int
// CountWorkers is the number of worker nodes to create.
CountWorkers int
// StateDiskSizeGB is the size of the state disk to allocate to each node, in GB.
StateDiskSizeGB int
}
// String returns a string representation of the variables, formatted as Terraform variables.
func (v *CommonVariables) String() string {
b := &strings.Builder{}
writeLinef(b, "name = %q", v.Name)
writeLinef(b, "control_plane_count = %d", v.CountControlPlanes)
writeLinef(b, "worker_count = %d", v.CountWorkers)
writeLinef(b, "state_disk_size = %d", v.StateDiskSizeGB)
return b.String()
}
// AWSClusterVariables is user configuration for creating a cluster with Terraform on AWS.
type AWSClusterVariables struct {
// Name of the cluster.
@ -73,6 +50,8 @@ type AWSClusterVariables struct {
EnableSNP bool `hcl:"enable_snp" cty:"enable_snp"`
// NodeGroups is a map of node groups to create.
NodeGroups map[string]AWSNodeGroup `hcl:"node_groups" cty:"node_groups"`
// CustomEndpoint is the (optional) custom dns hostname for the kubernetes api server.
CustomEndpoint string `hcl:"custom_endpoint" cty:"custom_endpoint"`
}
// GetCreateMAA gets the CreateMAA variable.
@ -138,6 +117,8 @@ type GCPClusterVariables struct {
Debug bool `hcl:"debug" cty:"debug"`
// NodeGroups is a map of node groups to create.
NodeGroups map[string]GCPNodeGroup `hcl:"node_groups" cty:"node_groups"`
// CustomEndpoint is the (optional) custom dns hostname for the kubernetes api server.
CustomEndpoint string `hcl:"custom_endpoint" cty:"custom_endpoint"`
}
// GetCreateMAA gets the CreateMAA variable.
@ -212,6 +193,8 @@ type AzureClusterVariables struct {
SecureBoot *bool `hcl:"secure_boot" cty:"secure_boot"`
// NodeGroups is a map of node groups to create.
NodeGroups map[string]AzureNodeGroup `hcl:"node_groups" cty:"node_groups"`
// CustomEndpoint is the (optional) custom dns hostname for the kubernetes api server.
CustomEndpoint string `hcl:"custom_endpoint" cty:"custom_endpoint"`
}
// GetCreateMAA gets the CreateMAA variable.
@ -287,6 +270,8 @@ type OpenStackClusterVariables struct {
OpenstackPassword string `hcl:"openstack_password" cty:"openstack_password"`
// Debug is true if debug mode is enabled.
Debug bool `hcl:"debug" cty:"debug"`
// CustomEndpoint is the (optional) custom dns hostname for the kubernetes api server.
CustomEndpoint string `hcl:"custom_endpoint" cty:"custom_endpoint"`
}
// GetCreateMAA gets the CreateMAA variable.
@ -354,6 +339,8 @@ type QEMUVariables struct {
InitrdPath *string `hcl:"constellation_initrd" cty:"constellation_initrd"`
// KernelCmdline is the kernel command line.
KernelCmdline *string `hcl:"constellation_cmdline" cty:"constellation_cmdline"`
// CustomEndpoint is the (optional) custom dns hostname for the kubernetes api server.
CustomEndpoint string `hcl:"custom_endpoint" cty:"custom_endpoint"`
}
// GetCreateMAA gets the CreateMAA variable.

10
cli/internal/terraform/variables_test.go
View File

@ -43,6 +43,7 @@ func TestAWSClusterVariables(t *testing.T) {
IAMProfileWorkerNodes: "arn:aws:iam::123456789012:instance-profile/cluster-name-worker",
Debug: true,
EnableSNP: true,
CustomEndpoint: "example.com",
}
// test that the variables are correctly rendered
@ -72,6 +73,7 @@ node_groups = {
zone = "eu-central-1c"
}
}
custom_endpoint = "example.com"
`
got := vars.String()
assert.Equal(t, want, got)
@ -117,6 +119,7 @@ func TestGCPClusterVariables(t *testing.T) {
DiskType: "pd-ssd",
},
},
CustomEndpoint: "example.com",
}
// test that the variables are correctly rendered
@ -144,6 +147,7 @@ node_groups = {
zone = "eu-central-1b"
}
}
custom_endpoint = "example.com"
`
got := vars.String()
assert.Equal(t, want, got)
@ -186,6 +190,7 @@ func TestAzureClusterVariables(t *testing.T) {
CreateMAA: to.Ptr(true),
Debug: to.Ptr(true),
Location: "eu-central-1",
CustomEndpoint: "example.com",
}
// test that the variables are correctly rendered
@ -207,6 +212,7 @@ node_groups = {
zones = null
}
}
custom_endpoint = "example.com"
`
got := vars.String()
assert.Equal(t, want, got)
@ -249,6 +255,7 @@ func TestOpenStackClusterVariables(t *testing.T) {
StateDiskSizeGB: 30,
},
},
CustomEndpoint: "example.com",
}
// test that the variables are correctly rendered
@ -271,6 +278,7 @@ openstack_user_domain_name = "my-user-domain"
openstack_username = "my-username"
openstack_password = "my-password"
debug = true
custom_endpoint = "example.com"
`
got := vars.String()
assert.Equal(t, want, got)
@ -299,6 +307,7 @@ func TestQEMUClusterVariables(t *testing.T) {
NVRAM: "production",
InitrdPath: toPtr("/var/lib/libvirt/images/cluster-name-initrd"),
KernelCmdline: toPtr("console=ttyS0,115200n8"),
CustomEndpoint: "example.com",
}
// test that the variables are correctly rendered
@ -323,6 +332,7 @@ metadata_libvirt_uri = "qemu:///system"
nvram = "/usr/share/OVMF/constellation_vars.production.fd"
constellation_initrd = "/var/lib/libvirt/images/cluster-name-initrd"
constellation_cmdline = "console=ttyS0,115200n8"
custom_endpoint = "example.com"
`
got := vars.String()
assert.Equal(t, want, got)

13
cli/internal/upgrade/terraform.go
View File

@ -156,11 +156,12 @@ func (u *TerraformUpgrader) ApplyTerraformMigrations(ctx context.Context, opts T
}
outputFileContents := clusterid.File{
CloudProvider: opts.CSP,
InitSecret: []byte(tfOutput.Secret),
IP: tfOutput.IP,
UID: tfOutput.UID,
AttestationURL: tfOutput.AttestationURL,
CloudProvider: opts.CSP,
InitSecret: []byte(tfOutput.Secret),
IP: tfOutput.IP,
APIServerCertSANs: tfOutput.APIServerCertSANs,
UID: tfOutput.UID,
AttestationURL: tfOutput.AttestationURL,
}
if err := u.fileHandler.RemoveAll(constants.TerraformWorkingDir); err != nil {
@ -201,7 +202,7 @@ type tfClient interface {
PrepareUpgradeWorkspace(path, oldWorkingDir, newWorkingDir, upgradeID string, vars terraform.Variables) error
ShowPlan(ctx context.Context, logLevel terraform.LogLevel, planFilePath string, output io.Writer) error
Plan(ctx context.Context, logLevel terraform.LogLevel, planFile string) (bool, error)
CreateCluster(ctx context.Context, logLevel terraform.LogLevel) (terraform.CreateOutput, error)
CreateCluster(ctx context.Context, logLevel terraform.LogLevel) (terraform.ApplyOutput, error)
}
// policyPatcher interacts with the CSP (currently only applies for Azure) to update the attestation policy.

4
cli/internal/upgrade/terraform_test.go
View File

@ -371,8 +371,8 @@ func (u *stubTerraformClient) Plan(context.Context, terraform.LogLevel, string)
return u.hasDiff, u.planErr
}
func (u *stubTerraformClient) CreateCluster(context.Context, terraform.LogLevel) (terraform.CreateOutput, error) {
return terraform.CreateOutput{}, u.CreateClusterErr
func (u *stubTerraformClient) CreateCluster(context.Context, terraform.LogLevel) (terraform.ApplyOutput, error) {
return terraform.ApplyOutput{}, u.CreateClusterErr
}
type stubPolicyPatcher struct {

13
debugd/internal/debugd/metadata/cloudprovider/cloudprovider.go
View File

@ -10,7 +10,6 @@ package cloudprovider
import (
"context"
"fmt"
"net"
"github.com/edgelesssys/constellation/v2/internal/cloud/metadata"
"github.com/edgelesssys/constellation/v2/internal/role"
@ -22,7 +21,7 @@ type providerMetadata interface {
// Self retrieves the current instance.
Self(ctx context.Context) (metadata.InstanceMetadata, error)
// GetLoadBalancerEndpoint returns the endpoint of the load balancer.
GetLoadBalancerEndpoint(ctx context.Context) (string, error)
GetLoadBalancerEndpoint(ctx context.Context) (host, port string, err error)
// UID returns the UID of the current instance.
UID(ctx context.Context) (string, error)
}
@ -91,16 +90,10 @@ func (f *Fetcher) DiscoverDebugdIPs(ctx context.Context) ([]string, error) {
// DiscoverLoadbalancerIP gets load balancer IP from metadata API.
func (f *Fetcher) DiscoverLoadbalancerIP(ctx context.Context) (string, error) {
lbEndpoint, err := f.metaAPI.GetLoadBalancerEndpoint(ctx)
lbHost, _, err := f.metaAPI.GetLoadBalancerEndpoint(ctx)
if err != nil {
return "", fmt.Errorf("retrieving load balancer endpoint: %w", err)
}
// The port of the endpoint is not the port we need. We need to strip it off.
lbIP, _, err := net.SplitHostPort(lbEndpoint)
if err != nil {
return "", fmt.Errorf("parsing load balancer endpoint: %w", err)
}
return lbIP, nil
return lbHost, nil
}

27
debugd/internal/debugd/metadata/cloudprovider/cloudprovider_test.go
View File

@ -123,7 +123,6 @@ func TestDiscoverDebugIPs(t *testing.T) {
func TestDiscoverLoadbalancerIP(t *testing.T) {
ip := "192.0.2.1"
endpoint := ip + ":1234"
someErr := errors.New("failed")
testCases := map[string]struct {
@ -132,17 +131,13 @@ func TestDiscoverLoadbalancerIP(t *testing.T) {
wantErr bool
}{
"discovery works": {
metaAPI: &stubMetadata{getLBEndpointRes: endpoint},
metaAPI: &stubMetadata{getLBHostRes: ip},
wantIP: ip,
},
"get endpoint fails": {
metaAPI: &stubMetadata{getLBEndpointErr: someErr},
wantErr: true,
},
"invalid endpoint": {
metaAPI: &stubMetadata{getLBEndpointRes: "invalid"},
wantErr: true,
},
}
for name, tc := range testCases {
@ -166,14 +161,14 @@ func TestDiscoverLoadbalancerIP(t *testing.T) {
}
type stubMetadata struct {
listRes []metadata.InstanceMetadata
listErr error
selfRes metadata.InstanceMetadata
selfErr error
getLBEndpointRes string
getLBEndpointErr error
uid string
uidErr error
listRes []metadata.InstanceMetadata
listErr error
selfRes metadata.InstanceMetadata
selfErr error
getLBHostRes, getLBPortRes string
getLBEndpointErr error
uid string
uidErr error
}
func (m *stubMetadata) List(_ context.Context) ([]metadata.InstanceMetadata, error) {
@ -184,8 +179,8 @@ func (m *stubMetadata) Self(_ context.Context) (metadata.InstanceMetadata, error
return m.selfRes, m.selfErr
}
func (m *stubMetadata) GetLoadBalancerEndpoint(_ context.Context) (string, error) {
return m.getLBEndpointRes, m.getLBEndpointErr
func (m *stubMetadata) GetLoadBalancerEndpoint(_ context.Context) (string, string, error) {
return m.getLBHostRes, m.getLBPortRes, m.getLBEndpointErr
}
func (m *stubMetadata) UID(_ context.Context) (string, error) {

4
debugd/internal/debugd/metadata/fallback/fallback.go
View File

@ -32,8 +32,8 @@ func (fallbackMetadata) Self(context.Context) (metadata.InstanceMetadata, error)
}
// GetLoadBalancerEndpoint returns the endpoint of the load balancer.
func (fallbackMetadata) GetLoadBalancerEndpoint(context.Context) (string, error) {
return "", nil
func (fallbackMetadata) GetLoadBalancerEndpoint(context.Context) (string, string, error) {
return "", "", nil
}
// UID returns the UID of the current instance.

3
hack/go.mod
View File

@ -109,6 +109,7 @@ require (
github.com/benbjohnson/clock v1.3.0 // indirect
github.com/beorn7/perks v1.0.1 // indirect
github.com/blang/semver v3.5.1+incompatible // indirect
github.com/blang/semver/v4 v4.0.0 // indirect
github.com/cespare/xxhash/v2 v2.2.0 // indirect
github.com/chai2010/gettext-go v1.0.2 // indirect
github.com/cloudflare/circl v1.3.3 // indirect
@ -305,10 +306,12 @@ require (
k8s.io/apiserver v0.27.3 // indirect
k8s.io/cli-runtime v0.27.2 // indirect
k8s.io/client-go v0.27.3 // indirect
k8s.io/cluster-bootstrap v0.27.3 // indirect
k8s.io/component-base v0.27.3 // indirect
k8s.io/klog/v2 v2.100.1 // indirect
k8s.io/kube-openapi v0.0.0-20230501164219-8b0f38b5fd1f // indirect
k8s.io/kubectl v0.27.2 // indirect
k8s.io/kubernetes v1.27.3 // indirect
k8s.io/utils v0.0.0-20230505201702-9f6742963106 // indirect
oras.land/oras-go v1.2.3 // indirect
sigs.k8s.io/controller-runtime v0.15.0 // indirect

8
hack/go.sum
View File

@ -194,6 +194,8 @@ github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kB
github.com/bketelsen/crypt v0.0.4/go.mod h1:aI6NrJ0pMGgvZKL1iVgXLnfIFJtfV+bKCoqOes/6LfM=
github.com/blang/semver v3.5.1+incompatible h1:cQNTCjp13qL8KC3Nbxr/y2Bqb63oX6wdnnjpJbkM4JQ=
github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk=
github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM=
github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ=
github.com/bshuster-repo/logrus-logstash-hook v1.0.0 h1:e+C0SB5R1pu//O4MQ3f9cFuPGoOVeF2fE4Og9otCc70=
github.com/bugsnag/bugsnag-go v0.0.0-20141110184014-b1d153021fcd h1:rFt+Y/IK1aEZkEHchZRSq9OQbsSzIT/OrI8YFFmRIng=
github.com/bugsnag/osext v0.0.0-20130617224835-0dd3f918b21b h1:otBG+dV+YK+Soembjv71DPz3uX/V/6MMlSyD9JBQ6kQ=
@ -652,8 +654,8 @@ github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7V
github.com/k0kubun/go-ansi v0.0.0-20180517002512-3bf9e2903213/go.mod h1:vNUNkEQ1e29fT/6vq2aBdFsgNPmy8qMdSay1npru+Sw=
github.com/karrick/godirwalk v1.8.0/go.mod h1:H5KPZjojv4lE+QYImBI8xVtrBRgYrIVsaRPx4tDPEn4=
github.com/karrick/godirwalk v1.10.3/go.mod h1:RoGL9dQei4vP9ilrpETWE8CLOZ1kiN0LhBygSwrAsHA=
github.com/karrick/godirwalk v1.16.1 h1:DynhcF+bztK8gooS0+NDJFrdNZjJ3gzVzC545UNA9iw=
github.com/karrick/godirwalk v1.16.1/go.mod h1:j4mkqPuvaLI8mp1DroR3P6ad7cyYd4c1qeJ3RV7ULlk=
github.com/karrick/godirwalk v1.17.0 h1:b4kY7nqDdioR/6qnbHQyDvmA17u5G1cZ6J+CZXwSWoI=
github.com/kevinburke/ssh_config v1.2.0 h1:x584FjTGwHzMwvHx18PXxbBVzfnxogHaAReU4gf13a4=
github.com/kevinburke/ssh_config v1.2.0/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM=
github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=
@ -1549,6 +1551,8 @@ k8s.io/cli-runtime v0.27.2 h1:9HI8gfReNujKXt16tGOAnb8b4NZ5E+e0mQQHKhFGwYw=
k8s.io/cli-runtime v0.27.2/go.mod h1:9UecpyPDTkhiYY4d9htzRqN+rKomJgyb4wi0OfrmCjw=
k8s.io/client-go v0.27.3 h1:7dnEGHZEJld3lYwxvLl7WoehK6lAq7GvgjxpA3nv1E8=
k8s.io/client-go v0.27.3/go.mod h1:2MBEKuTo6V1lbKy3z1euEGnhPfGZLKTS9tiJ2xodM48=
k8s.io/cluster-bootstrap v0.27.3 h1:yk1XIWt/mbMgNHFdxd0HyVPq/rnJK7BS3oXj24gHClU=
k8s.io/cluster-bootstrap v0.27.3/go.mod h1:4/bxgDkpV7XPapJS1585P/nvy3FdBIoFssK4Z5oztrc=
k8s.io/component-base v0.27.3 h1:g078YmdcdTfrCE4fFobt7qmVXwS8J/3cI1XxRi/2+6k=
k8s.io/component-base v0.27.3/go.mod h1:JNiKYcGImpQ44iwSYs6dysxzR9SxIIgQalk4HaCNVUY=
k8s.io/klog/v2 v2.100.1 h1:7WCHKK6K8fNhTqfBhISHQ97KrnJNFZMcQvKp7gP/tmg=
@ -1557,6 +1561,8 @@ k8s.io/kube-openapi v0.0.0-20230501164219-8b0f38b5fd1f h1:2kWPakN3i/k81b0gvD5C5F
k8s.io/kube-openapi v0.0.0-20230501164219-8b0f38b5fd1f/go.mod h1:byini6yhqGC14c3ebc/QwanvYwhuMWF6yz2F8uwW8eg=
k8s.io/kubectl v0.27.2 h1:sSBM2j94MHBFRWfHIWtEXWCicViQzZsb177rNsKBhZg=
k8s.io/kubectl v0.27.2/go.mod h1:GCOODtxPcrjh+EC611MqREkU8RjYBh10ldQCQ6zpFKw=
k8s.io/kubernetes v1.27.3 h1:gwufSj7y6X18Q2Gl8v4Ev+AJHdzWkG7A8VNFffS9vu0=
k8s.io/kubernetes v1.27.3/go.mod h1:U8ZXeKBAPxeb4J4/HOaxjw1A9K6WfSH+fY2SS7CR6IM=
k8s.io/utils v0.0.0-20230505201702-9f6742963106 h1:EObNQ3TW2D+WptiYXlApGNLVy0zm/JIBVY9i+M4wpAU=
k8s.io/utils v0.0.0-20230505201702-9f6742963106/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0=
libvirt.org/go/libvirt v1.9004.0 h1:u+CHhs2OhVmu0MWzBDrlbLzQ5QB3ZfWtfT+lD3EaUIs=

2
internal/cloud/aws/BUILD.bazel
View File

@ -12,6 +12,7 @@ go_library(
deps = [
"//internal/cloud",
"//internal/cloud/metadata",
"//internal/constants",
"//internal/role",
"@com_github_aws_aws_sdk_go_v2//aws",
"@com_github_aws_aws_sdk_go_v2_config//:config",
@ -21,6 +22,7 @@ go_library(
"@com_github_aws_aws_sdk_go_v2_service_ec2//:ec2",
"@com_github_aws_aws_sdk_go_v2_service_ec2//types",
"@com_github_aws_aws_sdk_go_v2_service_elasticloadbalancingv2//:elasticloadbalancingv2",
"@com_github_aws_aws_sdk_go_v2_service_elasticloadbalancingv2//types",
"@com_github_aws_aws_sdk_go_v2_service_resourcegroupstaggingapi//:resourcegroupstaggingapi",
"@com_github_aws_aws_sdk_go_v2_service_resourcegroupstaggingapi//types",
"@io_k8s_utils//clock",

106
internal/cloud/aws/aws.go
View File

@ -19,6 +19,7 @@ import (
"context"
"errors"
"fmt"
"strconv"
"github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go-v2/config"
@ -26,10 +27,12 @@ import (
"github.com/aws/aws-sdk-go-v2/service/ec2"
ec2Types "github.com/aws/aws-sdk-go-v2/service/ec2/types"
"github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2"
elasticloadbalancingv2types "github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2/types"
"github.com/aws/aws-sdk-go-v2/service/resourcegroupstaggingapi"
tagType "github.com/aws/aws-sdk-go-v2/service/resourcegroupstaggingapi/types"
"github.com/edgelesssys/constellation/v2/internal/cloud"
"github.com/edgelesssys/constellation/v2/internal/cloud/metadata"
"github.com/edgelesssys/constellation/v2/internal/constants"
"github.com/edgelesssys/constellation/v2/internal/role"
)
@ -48,6 +51,7 @@ type loadbalancerAPI interface {
type ec2API interface {
DescribeInstances(context.Context, *ec2.DescribeInstancesInput, ...func(*ec2.Options)) (*ec2.DescribeInstancesOutput, error)
DescribeAddresses(context.Context, *ec2.DescribeAddressesInput, ...func(*ec2.Options)) (*ec2.DescribeAddressesOutput, error)
}
type imdsAPI interface {
@ -126,46 +130,100 @@ func (c *Cloud) InitSecretHash(ctx context.Context) ([]byte, error) {
}
// GetLoadBalancerEndpoint returns the endpoint of the load balancer.
func (c *Cloud) GetLoadBalancerEndpoint(ctx context.Context) (string, error) {
// TODO(malt3): remove old infrastructure code once we have migrated to using DNS as the load balancer endpoint.
func (c *Cloud) GetLoadBalancerEndpoint(ctx context.Context) (host, port string, err error) {
// try new architecture first
uid, err := c.readInstanceTag(ctx, cloud.TagUID)
if err != nil {
return "", fmt.Errorf("retrieving uid tag: %w", err)
return "", "", fmt.Errorf("retrieving uid tag: %w", err)
}
describeIPsOutput, err := c.ec2.DescribeAddresses(ctx, &ec2.DescribeAddressesInput{
Filters: []ec2Types.Filter{
{
Name: aws.String(cloud.TagUID),
Values: []string{uid},
},
{
Name: aws.String("constellation-ip-endpoint"),
Values: []string{"legacy-primary-zone"},
},
},
})
if err == nil && len(describeIPsOutput.Addresses) == 1 && describeIPsOutput.Addresses[0].PublicIp != nil {
return *describeIPsOutput.Addresses[0].PublicIp, strconv.FormatInt(constants.KubernetesPort, 10), nil
}
// fallback to old architecture
// this will be removed in the future
hostname, err := c.getLoadBalancerIPOldInfrastructure(ctx)
if err != nil {
return "", "", fmt.Errorf("retrieving load balancer ip: %w", err)
}
return hostname, strconv.FormatInt(constants.KubernetesPort, 10), nil
}
// getLoadBalancerIPOldInfrastructure returns the IP of the load balancer.
// This is only used for the old infrastructure.
// This will be removed in the future.
func (c *Cloud) getLoadBalancerIPOldInfrastructure(ctx context.Context) (string, error) {
loadbalancer, err := c.getLoadBalancer(ctx)
if err != nil {
return "", fmt.Errorf("finding Constellation load balancer: %w", err)
}
// TODO(malt3): Add support for multiple availability zones in the lb frontend.
// This can only be done after we have migrated to using DNS as the load balancer endpoint.
// At that point, we don't need to care about the number of availability zones anymore.
if len(loadbalancer.AvailabilityZones) != 1 {
return "", fmt.Errorf("%d availability zones found; expected 1", len(loadbalancer.AvailabilityZones))
}
if len(loadbalancer.AvailabilityZones[0].LoadBalancerAddresses) != 1 {
return "", fmt.Errorf("%d load balancer addresses found; expected 1", len(loadbalancer.AvailabilityZones[0].LoadBalancerAddresses))
}
if loadbalancer.AvailabilityZones[0].LoadBalancerAddresses[0].IpAddress == nil {
return "", errors.New("load balancer address is nil")
}
return *loadbalancer.AvailabilityZones[0].LoadBalancerAddresses[0].IpAddress, nil
}
/*
// TODO(malt3): uncomment and use as soon as we switch the primary endpoint to DNS.
func (c *Cloud) getLoadBalancerDNSName(ctx context.Context) (string, error) {
loadbalancer, err := c.getLoadBalancer(ctx)
if err != nil {
return "", fmt.Errorf("finding Constellation load balancer: %w", err)
}
if loadbalancer.DNSName == nil {
return "", errors.New("load balancer dns name missing")
}
return *loadbalancer.DNSName, nil
}
*/
func (c *Cloud) getLoadBalancer(ctx context.Context) (*elasticloadbalancingv2types.LoadBalancer, error) {
uid, err := c.readInstanceTag(ctx, cloud.TagUID)
if err != nil {
return nil, fmt.Errorf("retrieving uid tag: %w", err)
}
arns, err := c.getARNsByTag(ctx, uid, "elasticloadbalancing:loadbalancer")
if err != nil {
return "", fmt.Errorf("retrieving load balancer ARNs: %w", err)
return nil, fmt.Errorf("retrieving load balancer ARNs: %w", err)
}
if len(arns) != 1 {
return "", fmt.Errorf("%d load balancers found", len(arns))
return nil, fmt.Errorf("%d load balancers found", len(arns))
}
output, err := c.loadbalancer.DescribeLoadBalancers(ctx, &elasticloadbalancingv2.DescribeLoadBalancersInput{
LoadBalancerArns: arns,
})
if err != nil {
return "", fmt.Errorf("retrieving load balancer: %w", err)
return nil, fmt.Errorf("retrieving load balancer: %w", err)
}
if len(output.LoadBalancers) != 1 {
return "", fmt.Errorf("%d load balancers found; expected 1", len(output.LoadBalancers))
return nil, fmt.Errorf("%d load balancers found; expected 1", len(output.LoadBalancers))
}
// TODO(malt3): Add support for multiple availability zones in the lb frontend.
// This can only be done after we have migrated to using DNS as the load balancer endpoint.
// At that point, we don't need to care about the number of availability zones anymore.
if len(output.LoadBalancers[0].AvailabilityZones) != 1 {
return "", fmt.Errorf("%d availability zones found; expected 1", len(output.LoadBalancers[0].AvailabilityZones))
}
if len(output.LoadBalancers[0].AvailabilityZones[0].LoadBalancerAddresses) != 1 {
return "", fmt.Errorf("%d load balancer addresses found; expected 1", len(output.LoadBalancers[0].AvailabilityZones[0].LoadBalancerAddresses))
}
if output.LoadBalancers[0].AvailabilityZones[0].LoadBalancerAddresses[0].IpAddress == nil {
return "", errors.New("load balancer address is nil")
}
// TODO(malt3): ideally, we would use DNS here instead of IP addresses.
// Requires changes to the infrastructure.
return *output.LoadBalancers[0].AvailabilityZones[0].LoadBalancerAddresses[0].IpAddress, nil
return &output.LoadBalancers[0], nil
}
// getARNsByTag returns a list of ARNs that have the given tag.

68
internal/cloud/aws/aws_test.go
View File

@ -485,7 +485,7 @@ func TestGetLoadBalancerEndpoint(t *testing.T) {
ec2API *stubEC2
loadbalancer *stubLoadbalancer
resourceapi *stubResourceGroupTagging
wantAddr string
wantHost string
wantErr bool
}{
"success retrieving loadbalancer endpoint": {
@ -514,6 +514,47 @@ func TestGetLoadBalancerEndpoint(t *testing.T) {
},
},
},
describeAddressesResp: &ec2.DescribeAddressesOutput{
Addresses: []ec2Types.Address{
{
Tags: []ec2Types.Tag{
{Key: aws.String(cloud.TagUID), Value: aws.String("uid")},
{Key: aws.String("constellation-ip-endpoint"), Value: aws.String("legacy-primary-zone")},
},
PublicIp: aws.String(lbAddr),
},
},
},
},
wantHost: lbAddr,
},
"success retrieving loadbalancer endpoint legacy": {
imds: &stubIMDS{
instanceDocumentResp: &imds.GetInstanceIdentityDocumentOutput{
InstanceIdentityDocument: imds.InstanceIdentityDocument{
InstanceID: "test-instance-id",
},
},
},
ec2API: &stubEC2{
selfInstance: &ec2.DescribeInstancesOutput{
Reservations: []ec2Types.Reservation{
{
Instances: []ec2Types.Instance{
{
InstanceId: aws.String("test-instance-id"),
Tags: []ec2Types.Tag{
{
Key: aws.String(cloud.TagUID),
Value: aws.String("uid"),
},
},
},
},
},
},
},
describeAddressesErr: errors.New("using legacy infrastructure"),
},
loadbalancer: &stubLoadbalancer{
describeLoadBalancersOut: &elasticloadbalancingv2.DescribeLoadBalancersOutput{
@ -541,9 +582,9 @@ func TestGetLoadBalancerEndpoint(t *testing.T) {
},
},
},
wantAddr: lbAddr,
wantHost: lbAddr,
},
"too many ARNs": {
"too many ARNs legacy": {
imds: &stubIMDS{
instanceDocumentResp: &imds.GetInstanceIdentityDocumentOutput{
InstanceIdentityDocument: imds.InstanceIdentityDocument{
@ -569,6 +610,7 @@ func TestGetLoadBalancerEndpoint(t *testing.T) {
},
},
},
describeAddressesErr: errors.New("using legacy infrastructure"),
},
loadbalancer: &stubLoadbalancer{
describeLoadBalancersOut: &elasticloadbalancingv2.DescribeLoadBalancersOutput{
@ -601,7 +643,7 @@ func TestGetLoadBalancerEndpoint(t *testing.T) {
},
wantErr: true,
},
"too many ARNs (paged)": {
"too many ARNs (paged) legacy": {
imds: &stubIMDS{
instanceDocumentResp: &imds.GetInstanceIdentityDocumentOutput{
InstanceIdentityDocument: imds.InstanceIdentityDocument{
@ -627,6 +669,7 @@ func TestGetLoadBalancerEndpoint(t *testing.T) {
},
},
},
describeAddressesErr: errors.New("using legacy infrastructure"),
},
loadbalancer: &stubLoadbalancer{
describeLoadBalancersOut: &elasticloadbalancingv2.DescribeLoadBalancersOutput{
@ -664,7 +707,7 @@ func TestGetLoadBalancerEndpoint(t *testing.T) {
},
wantErr: true,
},
"loadbalancer has no availability zones": {
"loadbalancer has no availability zones legacy": {
imds: &stubIMDS{
instanceDocumentResp: &imds.GetInstanceIdentityDocumentOutput{
InstanceIdentityDocument: imds.InstanceIdentityDocument{
@ -690,6 +733,7 @@ func TestGetLoadBalancerEndpoint(t *testing.T) {
},
},
},
describeAddressesErr: errors.New("using legacy infrastructure"),
},
loadbalancer: &stubLoadbalancer{
describeLoadBalancersOut: &elasticloadbalancingv2.DescribeLoadBalancersOutput{
@ -711,7 +755,7 @@ func TestGetLoadBalancerEndpoint(t *testing.T) {
},
wantErr: true,
},
"failure to get resources by tag": {
"failure to get resources by tag legacy": {
imds: &stubIMDS{
instanceDocumentResp: &imds.GetInstanceIdentityDocumentOutput{
InstanceIdentityDocument: imds.InstanceIdentityDocument{
@ -737,6 +781,7 @@ func TestGetLoadBalancerEndpoint(t *testing.T) {
},
},
},
describeAddressesErr: errors.New("using legacy infrastructure"),
},
loadbalancer: &stubLoadbalancer{
describeLoadBalancersOut: &elasticloadbalancingv2.DescribeLoadBalancersOutput{
@ -772,14 +817,15 @@ func TestGetLoadBalancerEndpoint(t *testing.T) {
resourceapiClient: tc.resourceapi,
}
endpoint, err := m.GetLoadBalancerEndpoint(context.Background())
gotHost, gotPort, err := m.GetLoadBalancerEndpoint(context.Background())
if tc.wantErr {
assert.Error(err)
return
}
assert.NoError(err)
assert.Equal(tc.wantAddr, endpoint)
assert.Equal(tc.wantHost, gotHost)
assert.Equal("6443", gotPort)
})
}
}
@ -973,6 +1019,8 @@ type stubEC2 struct {
selfInstance *ec2.DescribeInstancesOutput
describeInstancesResp1 *ec2.DescribeInstancesOutput
describeInstancesResp2 *ec2.DescribeInstancesOutput
describeAddressesErr error
describeAddressesResp *ec2.DescribeAddressesOutput
}
func (s *stubEC2) DescribeInstances(_ context.Context, in *ec2.DescribeInstancesInput, _ ...func(*ec2.Options)) (*ec2.DescribeInstancesOutput, error) {
@ -985,6 +1033,10 @@ func (s *stubEC2) DescribeInstances(_ context.Context, in *ec2.DescribeInstances
return s.describeInstancesResp2, s.describeInstancesErr
}
func (s *stubEC2) DescribeAddresses(context.Context, *ec2.DescribeAddressesInput, ...func(*ec2.Options)) (*ec2.DescribeAddressesOutput, error) {
return s.describeAddressesResp, s.describeAddressesErr
}
type stubLoadbalancer struct {
describeLoadBalancersErr error
describeLoadBalancersOut *elasticloadbalancingv2.DescribeLoadBalancersOutput

1
internal/cloud/azure/BUILD.bazel
View File

@ -15,6 +15,7 @@ go_library(
"//internal/cloud",
"//internal/cloud/azureshared",
"//internal/cloud/metadata",
"//internal/constants",
"//internal/role",
"@com_github_azure_azure_sdk_for_go_sdk_azcore//runtime",
"@com_github_azure_azure_sdk_for_go_sdk_azidentity//:azidentity",

119
internal/cloud/azure/azure.go
View File

@ -21,6 +21,7 @@ import (
"errors"
"fmt"
"path"
"strconv"
"github.com/Azure/azure-sdk-for-go/sdk/azidentity"
"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v4"
@ -28,6 +29,7 @@ import (
"github.com/edgelesssys/constellation/v2/internal/cloud"
"github.com/edgelesssys/constellation/v2/internal/cloud/azureshared"
"github.com/edgelesssys/constellation/v2/internal/cloud/metadata"
"github.com/edgelesssys/constellation/v2/internal/constants"
"github.com/edgelesssys/constellation/v2/internal/role"
)
@ -156,41 +158,12 @@ func (c *Cloud) GetCCMConfig(ctx context.Context, providerID string, cloudServic
//
// The returned string is an IP address without a port, but the method name needs to satisfy the
// metadata interface.
func (c *Cloud) GetLoadBalancerEndpoint(ctx context.Context) (string, error) {
resourceGroup, err := c.imds.resourceGroup(ctx)
func (c *Cloud) GetLoadBalancerEndpoint(ctx context.Context) (host, port string, err error) {
hostname, err := c.getLoadBalancerPublicIP(ctx)
if err != nil {
return "", fmt.Errorf("retrieving resource group: %w", err)
return "", "", fmt.Errorf("retrieving load balancer public IP: %w", err)
}
uid, err := c.imds.uid(ctx)
if err != nil {
return "", fmt.Errorf("retrieving instance UID: %w", err)
}
lb, err := c.getLoadBalancer(ctx, resourceGroup, uid)
if err != nil {
return "", fmt.Errorf("retrieving load balancer: %w", err)
}
if lb == nil || lb.Properties == nil {
return "", errors.New("could not dereference load balancer IP configuration")
}
var pubIP string
for _, fipConf := range lb.Properties.FrontendIPConfigurations {
if fipConf == nil || fipConf.Properties == nil || fipConf.Properties.PublicIPAddress == nil || fipConf.Properties.PublicIPAddress.ID == nil {
continue
}
pubIP = path.Base(*fipConf.Properties.PublicIPAddress.ID)
break
}
resp, err := c.pubIPAPI.Get(ctx, resourceGroup, pubIP, nil)
if err != nil {
return "", fmt.Errorf("retrieving load balancer public IP address: %w", err)
}
if resp.Properties == nil || resp.Properties.IPAddress == nil {
return "", fmt.Errorf("could not resolve public IP address reference for load balancer")
}
return *resp.Properties.IPAddress, nil
return hostname, strconv.FormatInt(constants.KubernetesPort, 10), nil
}
// List retrieves all instances belonging to the current constellation.
@ -409,6 +382,86 @@ func (c *Cloud) getVMInterfaces(ctx context.Context, vm armcompute.VirtualMachin
return networkInterfaces, nil
}
// getLoadBalancerPublicIP retrieves the first load balancer IP from cloud provider metadata.
func (c *Cloud) getLoadBalancerPublicIP(ctx context.Context) (string, error) {
resourceGroup, err := c.imds.resourceGroup(ctx)
if err != nil {
return "", fmt.Errorf("retrieving resource group: %w", err)
}
uid, err := c.imds.uid(ctx)
if err != nil {
return "", fmt.Errorf("retrieving instance UID: %w", err)
}
lb, err := c.getLoadBalancer(ctx, resourceGroup, uid)
if err != nil {
return "", fmt.Errorf("retrieving load balancer: %w", err)
}
if lb == nil || lb.Properties == nil {
return "", errors.New("could not dereference load balancer IP configuration")
}
var pubIP string
for _, fipConf := range lb.Properties.FrontendIPConfigurations {
if fipConf == nil || fipConf.Properties == nil || fipConf.Properties.PublicIPAddress == nil || fipConf.Properties.PublicIPAddress.ID == nil {
continue
}
pubIP = path.Base(*fipConf.Properties.PublicIPAddress.ID)
break
}
resp, err := c.pubIPAPI.Get(ctx, resourceGroup, pubIP, nil)
if err != nil {
return "", fmt.Errorf("retrieving load balancer public IP address: %w", err)
}
if resp.Properties == nil || resp.Properties.IPAddress == nil {
return "", fmt.Errorf("could not resolve public IP address reference for load balancer")
}
return *resp.Properties.IPAddress, nil
}
/*
// TODO(malt3): uncomment and use as soon as we switch the primary endpoint to DNS.
// getLoadBalancerDNSName retrieves the dns name of the load balancer.
// On Azure, the DNS name is the DNS name of the public IP address of the load balancer.
func (c *Cloud) getLoadBalancerDNSName(ctx context.Context) (string, error) {
resourceGroup, err := c.imds.resourceGroup(ctx)
if err != nil {
return "", fmt.Errorf("retrieving resource group: %w", err)
}
uid, err := c.imds.uid(ctx)
if err != nil {
return "", fmt.Errorf("retrieving instance UID: %w", err)
}
lb, err := c.getLoadBalancer(ctx, resourceGroup, uid)
if err != nil {
return "", fmt.Errorf("retrieving load balancer: %w", err)
}
if lb == nil || lb.Properties == nil {
return "", errors.New("could not dereference load balancer IP configuration")
}
var pubIP string
for _, fipConf := range lb.Properties.FrontendIPConfigurations {
if fipConf == nil || fipConf.Properties == nil || fipConf.Properties.PublicIPAddress == nil || fipConf.Properties.PublicIPAddress.ID == nil {
continue
}
pubIP = path.Base(*fipConf.Properties.PublicIPAddress.ID)
break
}
resp, err := c.pubIPAPI.Get(ctx, resourceGroup, pubIP, nil)
if err != nil {
return "", fmt.Errorf("retrieving load balancer public IP address: %w", err)
}
if resp.Properties == nil || resp.Properties.DNSSettings == nil || resp.Properties.DNSSettings.Fqdn == nil {
return "", fmt.Errorf("could not resolve public IP address fqdn for load balancer")
}
return *resp.Properties.DNSSettings.Fqdn, nil
}
*/
type cloudConfig struct {
Cloud string `json:"cloud,omitempty"`
TenantID string `json:"tenantId,omitempty"`

5
internal/cloud/azure/azure_test.go
View File

@ -1039,13 +1039,14 @@ func TestGetLoadBalancerEndpoint(t *testing.T) {
loadBalancerAPI: tc.loadBalancerAPI,
pubIPAPI: tc.publicIPAddressesAPI,
}
loadbalancerName, err := metadata.GetLoadBalancerEndpoint(context.Background())
gotHost, gotPort, err := metadata.GetLoadBalancerEndpoint(context.Background())
if tc.wantErr {
assert.Error(err)
return
}
require.NoError(err)
assert.Equal(tc.wantIP, loadbalancerName)
assert.Equal(tc.wantIP, gotHost)
assert.Equal("6443", gotPort)
})
}
}

3
internal/cloud/cloud.go
View File

@ -33,4 +33,7 @@ const (
TagUID = "constellation-uid"
// TagInitSecretHash is the tag/label key used to identify the hash of the init secret.
TagInitSecretHash = "constellation-init-secret-hash"
// TagCustomEndpoint is the tag/label key used to identify the custom endpoint
// or dns name that should be added to tls cert SANs.
TagCustomEndpoint = "constellation-custom-endpoint"
)

44
internal/cloud/gcp/gcp.go
View File

@ -19,7 +19,6 @@ import (
"context"
"errors"
"fmt"
"net"
"path"
"regexp"
"strings"
@ -124,43 +123,44 @@ func (c *Cloud) Close() {
}
// GetLoadBalancerEndpoint returns the endpoint of the load balancer.
func (c *Cloud) GetLoadBalancerEndpoint(ctx context.Context) (string, error) {
func (c *Cloud) GetLoadBalancerEndpoint(ctx context.Context) (host, port string, err error) {
project, zone, instanceName, err := c.retrieveInstanceInfo()
if err != nil {
return "", err
return "", "", err
}
uid, err := c.uid(ctx, project, zone, instanceName)
if err != nil {
return "", err
return "", "", err
}
// First try to find a global forwarding rule.
endpoint, err := c.getGlobalForwardingRule(ctx, project, uid)
host, port, err = c.getGlobalForwardingRule(ctx, project, uid)
if err != nil && !errors.Is(err, errNoForwardingRule) {
return "", fmt.Errorf("getting global forwarding rule: %w", err)
return "", "", fmt.Errorf("getting global forwarding rule: %w", err)
} else if err == nil {
return endpoint, nil
return host, port, nil
}
// If no global forwarding rule was found, try to find a regional forwarding rule.
region := zoneFromRegionRegex.FindString(zone)
if region == "" {
return "", fmt.Errorf("invalid zone %s", zone)
return "", "", fmt.Errorf("invalid zone %s", zone)
}
endpoint, err = c.getRegionalForwardingRule(ctx, project, uid, region)
host, port, err = c.getRegionalForwardingRule(ctx, project, uid, region)
if err != nil && !errors.Is(err, errNoForwardingRule) {
return "", fmt.Errorf("getting regional forwarding rule: %w", err)
} else if err == nil {
return endpoint, nil
return "", "", fmt.Errorf("getting regional forwarding rule: %w", err)
} else if err != nil {
return "", "", fmt.Errorf("kubernetes load balancer with UID %s not found: %w", uid, err)
}
return "", fmt.Errorf("kubernetes load balancer with UID %s not found: %w", uid, err)
return host, port, nil
}
// getGlobalForwardingRule returns the endpoint of the load balancer if it is a global load balancer.
// It returns the host, port and optionally an error.
// This functions returns ErrNoForwardingRule if no forwarding rule was found.
func (c *Cloud) getGlobalForwardingRule(ctx context.Context, project, uid string) (string, error) {
func (c *Cloud) getGlobalForwardingRule(ctx context.Context, project, uid string) (string, string, error) {
var resp *computepb.ForwardingRule
var err error
iter := c.globalForwardingRulesAPI.List(ctx, &computepb.ListGlobalForwardingRulesRequest{
@ -175,19 +175,19 @@ func (c *Cloud) getGlobalForwardingRule(ctx context.Context, project, uid string
continue
}
portRange := strings.Split(*resp.PortRange, "-")
return net.JoinHostPort(*resp.IPAddress, portRange[0]), nil
return *resp.IPAddress, portRange[0], nil
}
if err != iterator.Done {
return "", fmt.Errorf("error listing global forwarding rules with UID %s: %w", uid, err)
return "", "", fmt.Errorf("error listing global forwarding rules with UID %s: %w", uid, err)
}
return "", errNoForwardingRule
return "", "", errNoForwardingRule
}
// getRegionalForwardingRule returns the endpoint of the load balancer if it is a regional load balancer.
// It returns the host, port and optionally an error.
// This functions returns ErrNoForwardingRule if no forwarding rule was found.
func (c *Cloud) getRegionalForwardingRule(ctx context.Context, project, uid, region string) (string, error) {
func (c *Cloud) getRegionalForwardingRule(ctx context.Context, project, uid, region string) (host string, port string, err error) {
var resp *computepb.ForwardingRule
var err error
iter := c.regionalForwardingRulesAPI.List(ctx, &computepb.ListForwardingRulesRequest{
Project: project,
Region: region,
@ -204,12 +204,12 @@ func (c *Cloud) getRegionalForwardingRule(ctx context.Context, project, uid, reg
continue
}
portRange := strings.Split(*resp.PortRange, "-")
return net.JoinHostPort(*resp.IPAddress, portRange[0]), nil
return *resp.IPAddress, portRange[0], nil
}
if err != iterator.Done {
return "", fmt.Errorf("error listing global forwarding rules with UID %s: %w", uid, err)
return "", "", fmt.Errorf("error listing global forwarding rules with UID %s: %w", uid, err)
}
return "", errNoForwardingRule
return "", "", errNoForwardingRule
}
// List retrieves all instances belonging to the current constellation.

11
internal/cloud/gcp/gcp_test.go
View File

@ -211,7 +211,7 @@ func TestGetLoadbalancerEndpoint(t *testing.T) {
instanceAPI stubInstanceAPI
globalForwardingRulesAPI stubGlobalForwardingRulesAPI
regionalForwardingRulesAPI stubRegionalForwardingRulesAPI
wantEndpoint string
wantHost string
wantErr bool
}{
"success global forwarding rule": {
@ -236,7 +236,7 @@ func TestGetLoadbalancerEndpoint(t *testing.T) {
regionalForwardingRulesAPI: stubRegionalForwardingRulesAPI{
iterator: &stubForwardingRulesIterator{},
},
wantEndpoint: "192.0.2.255:6443",
wantHost: "192.0.2.255",
},
"success regional forwarding rule": {
imds: stubIMDS{
@ -261,7 +261,7 @@ func TestGetLoadbalancerEndpoint(t *testing.T) {
},
},
},
wantEndpoint: "192.0.2.255:6443",
wantHost: "192.0.2.255",
},
"regional forwarding rule has no region": {
imds: stubIMDS{
@ -473,13 +473,14 @@ func TestGetLoadbalancerEndpoint(t *testing.T) {
regionalForwardingRulesAPI: &tc.regionalForwardingRulesAPI,
}
endpoint, err := cloud.GetLoadBalancerEndpoint(context.Background())
gotHost, gotPort, err := cloud.GetLoadBalancerEndpoint(context.Background())
if tc.wantErr {
assert.Error(err)
return
}
assert.NoError(err)
assert.Equal(tc.wantEndpoint, endpoint)
assert.Equal(tc.wantHost, gotHost)
assert.Equal("6443", gotPort)
})
}
}

1
internal/cloud/openstack/BUILD.bazel
View File

@ -16,6 +16,7 @@ go_library(
deps = [
"//internal/cloud",
"//internal/cloud/metadata",
"//internal/constants",
"//internal/role",
"@com_github_gophercloud_gophercloud//:gophercloud",
"@com_github_gophercloud_gophercloud//openstack/compute/v2/servers",

12
internal/cloud/openstack/openstack.go
View File

@ -12,9 +12,11 @@ import (
"fmt"
"net/http"
"net/netip"
"strconv"
"strings"
"github.com/edgelesssys/constellation/v2/internal/cloud/metadata"
"github.com/edgelesssys/constellation/v2/internal/constants"
"github.com/edgelesssys/constellation/v2/internal/role"
"github.com/gophercloud/gophercloud/openstack/compute/v2/servers"
"github.com/gophercloud/gophercloud/openstack/networking/v2/subnets"
@ -230,7 +232,15 @@ func (c *Cloud) InitSecretHash(ctx context.Context) ([]byte, error) {
// For OpenStack, the load balancer is a floating ip attached to
// a control plane node.
// TODO(malt3): Rewrite to use real load balancer once it is available.
func (c *Cloud) GetLoadBalancerEndpoint(ctx context.Context) (string, error) {
func (c *Cloud) GetLoadBalancerEndpoint(ctx context.Context) (host, port string, err error) {
host, err = c.getLoadBalancerHost(ctx)
if err != nil {
return "", "", fmt.Errorf("getting load balancer host: %w", err)
}
return host, strconv.FormatInt(constants.KubernetesPort, 10), nil
}
func (c *Cloud) getLoadBalancerHost(ctx context.Context) (string, error) {
uid, err := c.imds.uid(ctx)
if err != nil {
return "", fmt.Errorf("getting uid: %w", err)

17
internal/cloud/openstack/openstack_test.go
View File

@ -487,10 +487,10 @@ func TestGetLoadBalancerEndpoint(t *testing.T) {
}
testCases := map[string]struct {
imds *stubIMDSClient
api *stubServersClient
want string
wantErr bool
imds *stubIMDSClient
api *stubServersClient
wantHost string
wantErr bool
}{
"error returned from IMDS client": {
imds: &stubIMDSClient{uidErr: errors.New("failed")},
@ -613,7 +613,7 @@ func TestGetLoadBalancerEndpoint(t *testing.T) {
},
}, nil),
},
want: "198.51.100.0",
wantHost: "198.51.100.0",
},
"first valid endpoint returned from server addresses not in subnet CIDR": {
imds: &stubIMDSClient{},
@ -628,7 +628,7 @@ func TestGetLoadBalancerEndpoint(t *testing.T) {
},
}, nil),
},
want: "198.51.100.0",
wantHost: "198.51.100.0",
},
}
@ -641,13 +641,14 @@ func TestGetLoadBalancerEndpoint(t *testing.T) {
api: tc.api,
}
got, err := c.GetLoadBalancerEndpoint(context.Background())
gotHost, gotPort, err := c.GetLoadBalancerEndpoint(context.Background())
if tc.wantErr {
assert.Error(err)
} else {
assert.NoError(err)
assert.Equal(tc.want, got)
assert.Equal(tc.wantHost, gotHost)
assert.Equal("6443", gotPort)
}
})
}

5
internal/cloud/qemu/BUILD.bazel
View File

@ -8,5 +8,8 @@ go_library(
],
importpath = "github.com/edgelesssys/constellation/v2/internal/cloud/qemu",
visibility = ["//:__subpackages__"],
deps = ["//internal/cloud/metadata"],
deps = [
"//internal/cloud/metadata",
"//internal/constants",
],
)

12
internal/cloud/qemu/qemu.go
View File

@ -16,8 +16,10 @@ import (
"io"
"net/http"
"net/url"
"strconv"
"github.com/edgelesssys/constellation/v2/internal/cloud/metadata"
"github.com/edgelesssys/constellation/v2/internal/constants"
)
const qemuMetadataEndpoint = "10.42.0.1:8080"
@ -56,7 +58,15 @@ func (c *Cloud) Self(ctx context.Context) (metadata.InstanceMetadata, error) {
// GetLoadBalancerEndpoint returns the endpoint of the load balancer.
// For QEMU, the load balancer is the first control plane node returned by the metadata API.
func (c *Cloud) GetLoadBalancerEndpoint(ctx context.Context) (string, error) {
func (c *Cloud) GetLoadBalancerEndpoint(ctx context.Context) (host, port string, err error) {
host, err = c.getLoadBalancerHost(ctx)
if err != nil {
return "", "", fmt.Errorf("getting load balancer host: %w", err)
}
return host, strconv.FormatInt(constants.KubernetesPort, 10), nil
}
func (c *Cloud) getLoadBalancerHost(ctx context.Context) (string, error) {
endpointRaw, err := c.retrieveMetadata(ctx, "/endpoint")
if err != nil {
return "", err

6
internal/config/config.go
View File

@ -85,6 +85,12 @@ type Config struct {
// description: |
// Configuration for attestation validation. This configuration provides sensible defaults for the Constellation version it was created for.\nSee the docs for an overview on attestation: https://docs.edgeless.systems/constellation/architecture/attestation
Attestation AttestationConfig `yaml:"attestation" validate:"dive"`
// description: |
// Optional custom endpoint (DNS name) for the Constellation API server.
// This can be used to point a custom dns name at the Constellation API server
// and is added to the Subject Alternative Name (SAN) field of the TLS certificate used by the API server.
// A fallback to DNS name is always available.
CustomEndpoint string `yaml:"customEndpoint" validate:"omitempty,hostname_rfc1123"`
}
// ProviderConfig are cloud-provider specific configuration values used by the CLI.

7
internal/config/config_doc.go
View File

@ -34,7 +34,7 @@ func init() {
ConfigDoc.Type = "Config"
ConfigDoc.Comments[encoder.LineComment] = "Config defines configuration used by CLI."
ConfigDoc.Description = "Config defines configuration used by CLI."
ConfigDoc.Fields = make([]encoder.Doc, 9)
ConfigDoc.Fields = make([]encoder.Doc, 10)
ConfigDoc.Fields[0].Name = "version"
ConfigDoc.Fields[0].Type = "string"
ConfigDoc.Fields[0].Note = ""
@ -80,6 +80,11 @@ func init() {
ConfigDoc.Fields[8].Note = ""
ConfigDoc.Fields[8].Description = "Configuration for attestation validation. This configuration provides sensible defaults for the Constellation version it was created for.\nSee the docs for an overview on attestation: https://docs.edgeless.systems/constellation/architecture/attestation"
ConfigDoc.Fields[8].Comments[encoder.LineComment] = "Configuration for attestation validation. This configuration provides sensible defaults for the Constellation version it was created for.\nSee the docs for an overview on attestation: https://docs.edgeless.systems/constellation/architecture/attestation"
ConfigDoc.Fields[9].Name = "customEndpoint"
ConfigDoc.Fields[9].Type = "string"
ConfigDoc.Fields[9].Note = ""
ConfigDoc.Fields[9].Description = "Optional custom endpoint (DNS name) for the Constellation API server.\nThis can be used to point a custom dns name at the Constellation API server\nand is added to the Subject Alternative Name (SAN) field of the TLS certificate used by the API server.\nA fallback to DNS name is always available."
ConfigDoc.Fields[9].Comments[encoder.LineComment] = "Optional custom endpoint (DNS name) for the Constellation API server."
ProviderConfigDoc.Type = "ProviderConfig"
ProviderConfigDoc.Comments[encoder.LineComment] = "ProviderConfig are cloud-provider specific configuration values used by the CLI."

5
internal/constants/constants.go
View File

@ -175,6 +175,11 @@ const (
JoinConfigMap = "join-config"
// InternalConfigMap k8s config map with internal Constellation config.
InternalConfigMap = "internal-config"
// KubeadmConfigMap k8s config map with kubeadm config
// (holds ClusterConfiguration).
KubeadmConfigMap = "kubeadm-config"
// ClusterConfigurationKey key in kubeadm config map with ClusterConfiguration.
ClusterConfigurationKey = "ClusterConfiguration"
//
// Helm.