monorepo

Co-authored-by: Malte Poll <mp@edgeless.systems>
Co-authored-by: katexochen <katexochen@users.noreply.github.com>
Co-authored-by: Daniel Weiße <dw@edgeless.systems>
Co-authored-by: Thomas Tendyck <tt@edgeless.systems>
Co-authored-by: Benedict Schlueter <bs@edgeless.systems>
Co-authored-by: leongross <leon.gross@rub.de>
Co-authored-by: Moritz Eckert <m1gh7ym0@gmail.com>

coordinator/kubernetes/k8sapi/util.go

@@ -0,0 +1,134 @@
+package k8sapi
+
+import (
+	"errors"
+	"fmt"
+	"os"
+	"os/exec"
+	"strings"
+
+	"github.com/edgelesssys/constellation/coordinator/kubernetes/k8sapi/resources"
+	kubeadm "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta3"
+)
+
+// kubeConfig is the path to the kubernetes admin config (used for authentication).
+const kubeConfig = "/etc/kubernetes/admin.conf"
+
+// Client provides the functionality of `kubectl apply`.
+type Client interface {
+	Apply(resources resources.Marshaler, forceConflicts bool) error
+	SetKubeconfig(kubeconfig []byte)
+	// TODO: add tolerations
+}
+
+type ClusterUtil interface {
+	InitCluster(initConfig []byte) (*kubeadm.BootstrapTokenDiscovery, error)
+	JoinCluster(joinConfig []byte) error
+	SetupPodNetwork(kubectl Client, podNetworkConfiguration resources.Marshaler) error
+	SetupAutoscaling(kubectl Client, clusterAutoscalerConfiguration resources.Marshaler) error
+	SetupCloudControllerManager(kubectl Client, cloudControllerManagerConfiguration resources.Marshaler) error
+	RestartKubelet() error
+}
+
+type KubernetesUtil struct{}
+
+func (k *KubernetesUtil) InitCluster(initConfig []byte) (*kubeadm.BootstrapTokenDiscovery, error) {
+	initConfigFile, err := os.CreateTemp("", "kubeadm-init.*.yaml")
+	if err != nil {
+		return nil, fmt.Errorf("failed to create init config file %v: %w", initConfigFile.Name(), err)
+	}
+	defer os.Remove(initConfigFile.Name())
+
+	if _, err := initConfigFile.Write(initConfig); err != nil {
+		return nil, fmt.Errorf("writing kubeadm init yaml config %v failed: %w", initConfigFile.Name(), err)
+	}
+
+	cmd := exec.Command("kubeadm", "init", "--config", initConfigFile.Name())
+	stdout, err := cmd.Output()
+	if err != nil {
+		var exitErr *exec.ExitError
+		if errors.As(err, &exitErr) {
+			return nil, fmt.Errorf("kubeadm init failed (code %v) with: %s", exitErr.ExitCode(), exitErr.Stderr)
+		}
+		return nil, fmt.Errorf("kubeadm init failed: %w", err)
+	}
+
+	stdoutStr := string(stdout)
+	indexKubeadmJoin := strings.Index(stdoutStr, "kubeadm join")
+	if indexKubeadmJoin < 0 {
+		return nil, errors.New("kubeadm init did not return join command")
+	}
+
+	joinCommand := strings.ReplaceAll(stdoutStr[indexKubeadmJoin:], "\\\n", " ")
+	// `kubeadm init` returns the two join commands, each broken up into two lines with backslash + newline in between.
+	// The following functions assume that stdoutStr[indexKubeadmJoin:] look like the following string.
+
+	// -----------------------------------------------------------------------------------------------
+	// --- When modifying the kubeadm.InitConfiguration make sure that this assumption still holds ---
+	// -----------------------------------------------------------------------------------------------
+
+	// "kubeadm join 127.0.0.1:16443 --token vlhjr4.9l6lhek0b9v65m67 \
+	//	--discovery-token-ca-cert-hash sha256:2b5343a162e31b70602e3cab3d87189dc10431e869633c4db63c3bfcd038dee6 \
+	//	--control-plane
+	//
+	// Then you can join any number of worker nodes by running the following on each as root:
+	//
+	// kubeadm join 127.0.0.1:16443 --token vlhjr4.9l6lhek0b9v65m67 \
+	//  --discovery-token-ca-cert-hash sha256:2b5343a162e31b70602e3cab3d87189dc10431e869633c4db63c3bfcd038dee6"
+
+	// Splits the string into a slice, where earch slice-element contains one line from the previous string
+	splittedJoinCommand := strings.SplitN(joinCommand, "\n", 2)
+
+	return ParseJoinCommand(splittedJoinCommand[0])
+}
+
+// SetupPodNetwork sets up the flannel pod network.
+func (k *KubernetesUtil) SetupPodNetwork(kubectl Client, podNetworkConfiguration resources.Marshaler) error {
+	if err := kubectl.Apply(podNetworkConfiguration, true); err != nil {
+		return err
+	}
+
+	// allow coredns to run on uninitialized nodes (required by cloud-controller-manager)
+	err := exec.Command("kubectl", "--kubeconfig", kubeConfig, "-n", "kube-system", "patch", "deployment", "coredns", "--type", "json", "-p", "[{\"op\":\"add\",\"path\":\"/spec/template/spec/tolerations/-\",\"value\":{\"key\":\"node.cloudprovider.kubernetes.io/uninitialized\",\"value\":\"true\",\"effect\":\"NoSchedule\"}}]").Run()
+	if err != nil {
+		return err
+	}
+	return exec.Command("kubectl", "--kubeconfig", kubeConfig, "-n", "kube-system", "patch", "deployment", "coredns", "--type", "json", "-p", "[{\"op\":\"add\",\"path\":\"/spec/template/spec/tolerations/-\",\"value\":{\"key\":\"node.kubernetes.io/network-unavailable\",\"value\":\"\",\"effect\":\"NoSchedule\"}}]").Run()
+}
+
+func (k *KubernetesUtil) SetupAutoscaling(kubectl Client, clusterAutoscalerConfiguration resources.Marshaler) error {
+	return kubectl.Apply(clusterAutoscalerConfiguration, true)
+}
+
+func (k *KubernetesUtil) SetupCloudControllerManager(kubectl Client, cloudControllerManagerConfiguration resources.Marshaler) error {
+	return kubectl.Apply(cloudControllerManagerConfiguration, true)
+}
+
+// JoinCluster joins existing kubernetes cluster using kubeadm join.
+func (k *KubernetesUtil) JoinCluster(joinConfig []byte) error {
+	joinConfigFile, err := os.CreateTemp("", "kubeadm-join.*.yaml")
+	if err != nil {
+		return fmt.Errorf("failed to create join config file %v: %w", joinConfigFile.Name(), err)
+	}
+	defer os.Remove(joinConfigFile.Name())
+
+	if _, err := joinConfigFile.Write(joinConfig); err != nil {
+		return fmt.Errorf("writing kubeadm init yaml config %v failed: %w", joinConfigFile.Name(), err)
+	}
+
+	// run `kubeadm join` to join a worker node to an existing kubernetes cluster
+	cmd := exec.Command("kubeadm", "join", "--config", joinConfigFile.Name())
+	if _, err := cmd.Output(); err != nil {
+		var exitErr *exec.ExitError
+		if errors.As(err, &exitErr) {
+			return fmt.Errorf("kubeadm join failed (code %v) with: %s", exitErr.ExitCode(), exitErr.Stderr)
+		}
+		return fmt.Errorf("kubeadm join failed: %w", err)
+	}
+	return nil
+}
+
+// RestartKubelet restarts a kubelet.
+func (k *KubernetesUtil) RestartKubelet() error {
+	return RestartSystemdUnit("kubelet.service")
+}