package core import ( "context" "strings" "time" "github.com/edgelesssys/constellation/coordinator/kubernetes" "github.com/edgelesssys/constellation/coordinator/kubernetes/k8sapi/resources" "github.com/edgelesssys/constellation/coordinator/role" "github.com/edgelesssys/constellation/internal/constants" "go.uber.org/zap" kubeadm "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta3" ) // GetK8sJoinArgs returns the args needed by a Node to join the cluster. func (c *Core) GetK8sJoinArgs() (*kubeadm.BootstrapTokenDiscovery, error) { return c.kube.GetJoinToken(constants.KubernetesJoinTokenTTL) } // GetK8SCertificateKey returns the key needed by a Coordinator to join the cluster. func (c *Core) GetK8SCertificateKey() (string, error) { return c.kube.GetKubeadmCertificateKey() } // InitCluster initializes the cluster, stores the join args, and returns the kubeconfig. func (c *Core) InitCluster(autoscalingNodeGroups []string, cloudServiceAccountURI string) ([]byte, error) { var nodeName string var providerID string var instance Instance var ccmConfigMaps resources.ConfigMaps var ccmSecrets resources.Secrets var caSecrets resources.Secrets var err error nodeIP := coordinatorVPNIP.String() if c.metadata.Supported() { instance, err = c.metadata.Self(context.TODO()) if err != nil { c.zaplogger.Error("Retrieving own instance metadata failed", zap.Error(err)) return nil, err } nodeName = instance.Name providerID = instance.ProviderID if len(instance.IPs) > 0 { nodeIP = instance.IPs[0] } } else { nodeName = coordinatorVPNIP.String() } if c.cloudControllerManager.Supported() && c.metadata.Supported() { c.zaplogger.Info("Preparing node for cloud-controller-manager") if err := PrepareInstanceForCCM(context.TODO(), c.metadata, c.cloudControllerManager, coordinatorVPNIP.String()); err != nil { c.zaplogger.Error("Preparing node for CCM failed", zap.Error(err)) return nil, err } ccmConfigMaps, err = c.cloudControllerManager.ConfigMaps(instance) if err != nil { c.zaplogger.Error("Defining ConfigMaps for CCM failed", zap.Error(err)) return nil, err } ccmSecrets, err = c.cloudControllerManager.Secrets(instance, cloudServiceAccountURI) if err != nil { c.zaplogger.Error("Defining Secrets for CCM failed", zap.Error(err)) return nil, err } } if c.clusterAutoscaler.Supported() { caSecrets, err = c.clusterAutoscaler.Secrets(instance, cloudServiceAccountURI) if err != nil { c.zaplogger.Error("Defining Secrets for cluster-autoscaler failed", zap.Error(err)) return nil, err } } c.zaplogger.Info("Initializing cluster") if err := c.kube.InitCluster(kubernetes.InitClusterInput{ APIServerAdvertiseIP: coordinatorVPNIP.String(), NodeIP: nodeIP, NodeName: k8sCompliantHostname(nodeName), ProviderID: providerID, SupportClusterAutoscaler: c.clusterAutoscaler.Supported(), AutoscalingCloudprovider: c.clusterAutoscaler.Name(), AutoscalingSecrets: caSecrets, AutoscalingVolumes: c.clusterAutoscaler.Volumes(), AutoscalingVolumeMounts: c.clusterAutoscaler.VolumeMounts(), AutoscalingEnv: c.clusterAutoscaler.Env(), AutoscalingNodeGroups: autoscalingNodeGroups, SupportsCloudControllerManager: c.cloudControllerManager.Supported(), CloudControllerManagerName: c.cloudControllerManager.Name(), CloudControllerManagerImage: c.cloudControllerManager.Image(), CloudControllerManagerPath: c.cloudControllerManager.Path(), CloudControllerManagerExtraArgs: c.cloudControllerManager.ExtraArgs(), CloudControllerManagerConfigMaps: ccmConfigMaps, CloudControllerManagerSecrets: ccmSecrets, CloudControllerManagerVolumes: c.cloudControllerManager.Volumes(), CloudControllerManagerVolumeMounts: c.cloudControllerManager.VolumeMounts(), CloudControllerManagerEnv: c.cloudControllerManager.Env(), SupportsCloudNodeManager: c.cloudNodeManager.Supported(), CloudNodeManagerImage: c.cloudNodeManager.Image(), CloudNodeManagerPath: c.cloudNodeManager.Path(), CloudNodeManagerExtraArgs: c.cloudNodeManager.ExtraArgs(), }); err != nil { c.zaplogger.Error("Initializing cluster failed", zap.Error(err)) return nil, err } kubeconfig, err := c.kube.GetKubeconfig() if err != nil { return nil, err } if err := c.data().PutKubernetesConfig(kubeconfig); err != nil { return nil, err } // set role in cloud provider metadata for autoconfiguration if c.metadata.Supported() { if err := c.metadata.SignalRole(context.TODO(), role.Coordinator); err != nil { c.zaplogger.Info("unable to update role in cloud provider metadata", zap.Error(err)) } } return kubeconfig, nil } // JoinCluster lets a Node join the cluster. func (c *Core) JoinCluster(args *kubeadm.BootstrapTokenDiscovery, certKey string, peerRole role.Role) error { c.zaplogger.Info("Joining Kubernetes cluster") nodeVPNIP, err := c.vpn.GetInterfaceIP() if err != nil { c.zaplogger.Error("Retrieving vpn ip failed", zap.Error(err)) return err } var nodeName string var providerID string nodeIP := nodeVPNIP if c.metadata.Supported() { instance, err := c.metadata.Self(context.TODO()) if err != nil { c.zaplogger.Error("Retrieving own instance metadata failed", zap.Error(err)) return err } providerID = instance.ProviderID nodeName = instance.Name if len(instance.IPs) > 0 { nodeIP = instance.IPs[0] } } else { nodeName = nodeVPNIP } if c.cloudControllerManager.Supported() && c.metadata.Supported() { c.zaplogger.Info("Preparing node for cloud-controller-manager") if err := PrepareInstanceForCCM(context.TODO(), c.metadata, c.cloudControllerManager, nodeVPNIP); err != nil { c.zaplogger.Error("Preparing node for CCM failed", zap.Error(err)) return err } } c.zaplogger.Info("k8s Join data", zap.String("nodename", nodeName), zap.String("nodeIP", nodeIP), zap.String("nodeVPNIP", nodeVPNIP), zap.String("provid", providerID)) // we need to pass the VPNIP for another control-plane, otherwise etcd will bind itself to the wrong IP address and fails if err := c.kube.JoinCluster(args, k8sCompliantHostname(nodeName), nodeIP, nodeVPNIP, providerID, certKey, c.cloudControllerManager.Supported(), peerRole); err != nil { c.zaplogger.Error("Joining Kubernetes cluster failed", zap.Error(err)) return err } c.zaplogger.Info("Joined Kubernetes cluster") // set role in cloud provider metadata for autoconfiguration if c.metadata.Supported() { if err := c.metadata.SignalRole(context.TODO(), peerRole); err != nil { c.zaplogger.Info("unable to update role in cloud provider metadata", zap.Error(err)) } } return nil } // Cluster manages the overall cluster lifecycle (init, join). type Cluster interface { // InitCluster bootstraps a new cluster with the current node being the master, returning the arguments required to join the cluster. InitCluster(kubernetes.InitClusterInput) error // JoinCluster will join the current node to an existing cluster. JoinCluster(args *kubeadm.BootstrapTokenDiscovery, nodeName, nodeIP, nodeVPNIP, providerID, certKey string, ccmSupported bool, peerRole role.Role) error // GetKubeconfig reads the kubeconfig from the filesystem. Only succeeds after cluster is initialized. GetKubeconfig() ([]byte, error) // GetKubeadmCertificateKey returns the 64-byte hex string key needed to join the cluster as control-plane. This function must be executed on a control-plane. GetKubeadmCertificateKey() (string, error) // GetJoinToken returns a bootstrap (join) token. GetJoinToken(ttl time.Duration) (*kubeadm.BootstrapTokenDiscovery, error) // StartKubelet starts the kubelet service. StartKubelet() error } // ClusterFake behaves like a real cluster, but does not actually initialize or join Kubernetes. type ClusterFake struct{} // InitCluster fakes bootstrapping a new cluster with the current node being the master, returning the arguments required to join the cluster. func (c *ClusterFake) InitCluster(kubernetes.InitClusterInput) error { return nil } // JoinCluster will fake joining the current node to an existing cluster. func (c *ClusterFake) JoinCluster(args *kubeadm.BootstrapTokenDiscovery, nodeName, nodeIP, nodeVPNIP, providerID, certKey string, _ bool, _ role.Role) error { return nil } // GetKubeconfig fakes reading the kubeconfig from the filesystem. Only succeeds after cluster is initialized. func (c *ClusterFake) GetKubeconfig() ([]byte, error) { return []byte("kubeconfig"), nil } // GetKubeadmCertificateKey fakes generating a certificateKey. func (c *ClusterFake) GetKubeadmCertificateKey() (string, error) { return "controlPlaneCertficateKey", nil } // GetJoinToken returns a bootstrap (join) token. func (c *ClusterFake) GetJoinToken(_ time.Duration) (*kubeadm.BootstrapTokenDiscovery, error) { return &kubeadm.BootstrapTokenDiscovery{ APIServerEndpoint: "0.0.0.0", Token: "kube-fake-token", CACertHashes: []string{"sha256:a60ebe9b0879090edd83b40a4df4bebb20506bac1e51d518ff8f4505a721930f"}, }, nil } // StartKubelet starts the kubelet service. func (c *ClusterFake) StartKubelet() error { return nil } // k8sCompliantHostname transforms a hostname to an RFC 1123 compliant, lowercase subdomain as required by Kubernetes node names. // The following regex is used by k8s for validation: /^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$/ . // Only a simple heuristic is used for now (to lowercase, replace underscores). func k8sCompliantHostname(in string) string { hostname := strings.ToLower(in) hostname = strings.ReplaceAll(hostname, "_", "-") return hostname }