From c6ccee1250e0019f5731fec6357993afed6b999a Mon Sep 17 00:00:00 2001 From: Otto Bittner Date: Tue, 18 Oct 2022 13:15:54 +0200 Subject: [PATCH] AB#2490: deploy KMS via Helm * Bundle helm-install related code in speparate package * Move cilium installation to new helm package --- bootstrapper/cmd/bootstrapper/main.go | 12 +- bootstrapper/cmd/bootstrapper/test.go | 4 +- bootstrapper/internal/helm/client.go | 174 ++++++++++++ bootstrapper/internal/helm/kms.go | 13 + .../internal/initserver/initserver.go | 14 +- .../internal/initserver/initserver_test.go | 4 +- .../internal/kubernetes/k8sapi/constants.go | 1 - .../kubernetes/k8sapi/{util.go => k8sutil.go} | 152 +--------- .../kubernetes/k8sapi/resources/kms.go | 259 ------------------ .../kubernetes/k8sapi/resources/kms_test.go | 28 -- bootstrapper/internal/kubernetes/k8sutil.go | 14 +- .../internal/kubernetes/kubernetes.go | 25 +- .../internal/kubernetes/kubernetes_test.go | 38 ++- cli/internal/helm/loader.go | 45 ++- internal/constants/constants.go | 7 +- internal/deploy/helm/helm.go | 14 +- kms/cmd/main.go | 2 +- 17 files changed, 311 insertions(+), 495 deletions(-) create mode 100644 bootstrapper/internal/helm/client.go create mode 100644 bootstrapper/internal/helm/kms.go rename bootstrapper/internal/kubernetes/k8sapi/{util.go => k8sutil.go} (77%) delete mode 100644 bootstrapper/internal/kubernetes/k8sapi/resources/kms.go delete mode 100644 bootstrapper/internal/kubernetes/k8sapi/resources/kms_test.go diff --git a/bootstrapper/cmd/bootstrapper/main.go b/bootstrapper/cmd/bootstrapper/main.go index cd6837fee..cebba2be6 100644 --- a/bootstrapper/cmd/bootstrapper/main.go +++ b/bootstrapper/cmd/bootstrapper/main.go @@ -14,6 +14,7 @@ import ( "os" "strconv" + "github.com/edgelesssys/constellation/v2/bootstrapper/internal/helm" "github.com/edgelesssys/constellation/v2/bootstrapper/internal/initserver" "github.com/edgelesssys/constellation/v2/bootstrapper/internal/kubernetes" "github.com/edgelesssys/constellation/v2/bootstrapper/internal/kubernetes/k8sapi" @@ -69,6 +70,11 @@ func main() { var openTPM vtpm.TPMOpenFunc var fs afero.Fs + helmClient, err := helm.New(log) + if err != nil { + log.With(zap.Error(err)).Fatalf("Helm client could not be initialized") + } + switch cloudprovider.FromString(os.Getenv(constellationCSP)) { case cloudprovider.AWS: panic("AWS cloud provider currently unsupported") @@ -104,7 +110,7 @@ func main() { } clusterInitJoiner = kubernetes.New( "gcp", k8sapi.NewKubernetesUtil(), &k8sapi.CoreOSConfiguration{}, kubectl.New(), cloudControllerManager, - &gcpcloud.CloudNodeManager{}, &gcpcloud.Autoscaler{}, metadata, pcrsJSON, + &gcpcloud.CloudNodeManager{}, &gcpcloud.Autoscaler{}, metadata, pcrsJSON, helmClient, ) openTPM = vtpm.OpenVTPM fs = afero.NewOsFs() @@ -137,7 +143,7 @@ func main() { } clusterInitJoiner = kubernetes.New( "azure", k8sapi.NewKubernetesUtil(), &k8sapi.CoreOSConfiguration{}, kubectl.New(), azurecloud.NewCloudControllerManager(metadata), - &azurecloud.CloudNodeManager{}, &azurecloud.Autoscaler{}, metadata, pcrsJSON, + &azurecloud.CloudNodeManager{}, &azurecloud.Autoscaler{}, metadata, pcrsJSON, helmClient, ) openTPM = vtpm.OpenVTPM @@ -158,7 +164,7 @@ func main() { } clusterInitJoiner = kubernetes.New( "qemu", k8sapi.NewKubernetesUtil(), &k8sapi.CoreOSConfiguration{}, kubectl.New(), &qemucloud.CloudControllerManager{}, - &qemucloud.CloudNodeManager{}, &qemucloud.Autoscaler{}, metadata, pcrsJSON, + &qemucloud.CloudNodeManager{}, &qemucloud.Autoscaler{}, metadata, pcrsJSON, helmClient, ) metadataAPI = metadata diff --git a/bootstrapper/cmd/bootstrapper/test.go b/bootstrapper/cmd/bootstrapper/test.go index 2d6be9d2e..1cc82d677 100644 --- a/bootstrapper/cmd/bootstrapper/test.go +++ b/bootstrapper/cmd/bootstrapper/test.go @@ -9,7 +9,7 @@ package main import ( "context" - "github.com/edgelesssys/constellation/v2/bootstrapper/internal/kubernetes/k8sapi/resources" + "github.com/edgelesssys/constellation/v2/bootstrapper/internal/helm" "github.com/edgelesssys/constellation/v2/internal/cloud/metadata" "github.com/edgelesssys/constellation/v2/internal/logger" "github.com/edgelesssys/constellation/v2/internal/role" @@ -22,7 +22,7 @@ type clusterFake struct{} // InitCluster fakes bootstrapping a new cluster with the current node being the master, returning the arguments required to join the cluster. func (c *clusterFake) InitCluster( context.Context, string, string, []byte, []uint32, bool, []byte, bool, - resources.KMSConfig, map[string]string, []byte, bool, *logger.Logger, + helm.KMSConfig, map[string]string, []byte, bool, *logger.Logger, ) ([]byte, error) { return []byte{}, nil } diff --git a/bootstrapper/internal/helm/client.go b/bootstrapper/internal/helm/client.go new file mode 100644 index 000000000..6a05a0304 --- /dev/null +++ b/bootstrapper/internal/helm/client.go @@ -0,0 +1,174 @@ +/* +Copyright (c) Edgeless Systems GmbH + +SPDX-License-Identifier: AGPL-3.0-only +*/ + +package helm + +import ( + "context" + "encoding/base64" + "errors" + "fmt" + "net" + "os/exec" + "strconv" + "time" + + "github.com/edgelesssys/constellation/v2/bootstrapper/internal/kubernetes/k8sapi" + "github.com/edgelesssys/constellation/v2/internal/constants" + "github.com/edgelesssys/constellation/v2/internal/deploy/helm" + "github.com/edgelesssys/constellation/v2/internal/logger" + "helm.sh/helm/v3/pkg/action" + "helm.sh/helm/v3/pkg/cli" + corev1 "k8s.io/api/core/v1" +) + +const ( + // timeout is the maximum time given to the helm client. + timeout = 5 * time.Minute +) + +// Client is used to install microservice during cluster initialization. It is a wrapper for a helm install action. +type Client struct { + *action.Install +} + +// New creates a new client with the given logger. +func New(log *logger.Logger) (*Client, error) { + settings := cli.New() + settings.KubeConfig = constants.CoreOSAdminConfFilename + + actionConfig := &action.Configuration{} + if err := actionConfig.Init(settings.RESTClientGetter(), constants.HelmNamespace, + "secret", log.Infof); err != nil { + return nil, err + } + + return &Client{ + action.NewInstall(actionConfig), + }, nil +} + +// InstallCilium sets up the cilium pod network. +func (h *Client) InstallCilium(ctx context.Context, kubectl k8sapi.Client, release helm.Release, in k8sapi.SetupPodNetworkInput) error { + h.Namespace = constants.HelmNamespace + h.ReleaseName = release.ReleaseName + h.Wait = release.Wait + h.Timeout = timeout + + switch in.CloudProvider { + case "gcp": + return h.installlCiliumGCP(ctx, kubectl, release, in.NodeName, in.FirstNodePodCIDR, in.SubnetworkPodCIDR, in.LoadBalancerEndpoint) + case "azure": + return h.installCiliumAzure(ctx, release, in.LoadBalancerEndpoint) + case "qemu": + return h.installCiliumQEMU(ctx, release, in.SubnetworkPodCIDR, in.LoadBalancerEndpoint) + default: + return fmt.Errorf("unsupported cloud provider %q", in.CloudProvider) + } +} + +func (h *Client) installCiliumAzure(ctx context.Context, release helm.Release, kubeAPIEndpoint string) error { + host := kubeAPIEndpoint + release.Values["k8sServiceHost"] = host + release.Values["k8sServicePort"] = strconv.Itoa(constants.KubernetesPort) + + _, err := h.RunWithContext(ctx, release.Chart, release.Values) + if err != nil { + return fmt.Errorf("installing cilium: %w", err) + } + return nil +} + +func (h *Client) installlCiliumGCP(ctx context.Context, kubectl k8sapi.Client, release helm.Release, nodeName, nodePodCIDR, subnetworkPodCIDR, kubeAPIEndpoint string) error { + out, err := exec.CommandContext(ctx, constants.KubectlPath, "--kubeconfig", constants.CoreOSAdminConfFilename, "patch", "node", nodeName, "-p", "{\"spec\":{\"podCIDR\": \""+nodePodCIDR+"\"}}").CombinedOutput() + if err != nil { + err = errors.New(string(out)) + return err + } + + timeoutS := int64(10) + // allow coredns to run on uninitialized nodes (required by cloud-controller-manager) + tolerations := []corev1.Toleration{ + { + Key: "node.cloudprovider.kubernetes.io/uninitialized", + Value: "true", + Effect: corev1.TaintEffectNoSchedule, + }, + { + Key: "node.kubernetes.io/unreachable", + Operator: corev1.TolerationOpExists, + Effect: corev1.TaintEffectNoExecute, + TolerationSeconds: &timeoutS, + }, + } + if err = kubectl.AddTolerationsToDeployment(ctx, tolerations, "coredns", "kube-system"); err != nil { + return err + } + selectors := map[string]string{ + "node-role.kubernetes.io/control-plane": "", + } + if err = kubectl.AddNodeSelectorsToDeployment(ctx, selectors, "coredns", "kube-system"); err != nil { + return err + } + + host, port, err := net.SplitHostPort(kubeAPIEndpoint) + if err != nil { + return err + } + + // configure pod network CIDR + release.Values["ipv4NativeRoutingCIDR"] = subnetworkPodCIDR + release.Values["strictModeCIDR"] = subnetworkPodCIDR + release.Values["k8sServiceHost"] = host + if port != "" { + release.Values["k8sServicePort"] = port + } + + _, err = h.RunWithContext(ctx, release.Chart, release.Values) + if err != nil { + return fmt.Errorf("helm install cilium: %w", err) + } + + return nil +} + +func (h *Client) installCiliumQEMU(ctx context.Context, release helm.Release, subnetworkPodCIDR, kubeAPIEndpoint string) error { + // configure pod network CIDR + release.Values["ipam"] = map[string]interface{}{ + "operator": map[string]interface{}{ + "clusterPoolIPv4PodCIDRList": []interface{}{ + subnetworkPodCIDR, + }, + }, + } + + release.Values["k8sServiceHost"] = kubeAPIEndpoint + release.Values["k8sServicePort"] = strconv.Itoa(constants.KubernetesPort) + + _, err := h.RunWithContext(ctx, release.Chart, release.Values) + if err != nil { + return fmt.Errorf("helm install cilium: %w", err) + } + return nil +} + +// InstallKMS deploys the KMS deployment. +func (h *Client) InstallKMS(ctx context.Context, release helm.Release, kmsConfig KMSConfig) error { + h.Namespace = constants.HelmNamespace + h.ReleaseName = release.ReleaseName + h.Wait = release.Wait + h.Timeout = timeout + + release.Values["masterSecret"] = base64.StdEncoding.EncodeToString(kmsConfig.MasterSecret[:]) + release.Values["salt"] = base64.StdEncoding.EncodeToString(kmsConfig.Salt[:]) + + _, err := h.RunWithContext(ctx, release.Chart, release.Values) + if err != nil { + return fmt.Errorf("helm install kms: %w", err) + } + + return nil +} diff --git a/bootstrapper/internal/helm/kms.go b/bootstrapper/internal/helm/kms.go new file mode 100644 index 000000000..89cf7e096 --- /dev/null +++ b/bootstrapper/internal/helm/kms.go @@ -0,0 +1,13 @@ +/* +Copyright (c) Edgeless Systems GmbH + +SPDX-License-Identifier: AGPL-3.0-only +*/ + +package helm + +// KMSConfig is the configuration needed to set up Constellation's key management service. +type KMSConfig struct { + MasterSecret []byte + Salt []byte +} diff --git a/bootstrapper/internal/initserver/initserver.go b/bootstrapper/internal/initserver/initserver.go index fd30a45e2..3a010d904 100644 --- a/bootstrapper/internal/initserver/initserver.go +++ b/bootstrapper/internal/initserver/initserver.go @@ -15,7 +15,7 @@ import ( "github.com/edgelesssys/constellation/v2/bootstrapper/initproto" "github.com/edgelesssys/constellation/v2/bootstrapper/internal/diskencryption" - "github.com/edgelesssys/constellation/v2/bootstrapper/internal/kubernetes/k8sapi/resources" + "github.com/edgelesssys/constellation/v2/bootstrapper/internal/helm" "github.com/edgelesssys/constellation/v2/internal/atls" "github.com/edgelesssys/constellation/v2/internal/attestation" "github.com/edgelesssys/constellation/v2/internal/cloud/vmtype" @@ -132,13 +132,9 @@ func (s *Server) Init(ctx context.Context, req *initproto.InitRequest) (*initpro req.EnforceIdkeydigest, s.issuerWrapper.IDKeyDigest(), s.issuerWrapper.VMType() == vmtype.AzureCVM, - resources.KMSConfig{ - MasterSecret: req.MasterSecret, - Salt: req.Salt, - KMSURI: req.KmsUri, - StorageURI: req.StorageUri, - KeyEncryptionKeyID: req.KeyEncryptionKeyId, - UseExistingKEK: req.UseExistingKek, + helm.KMSConfig{ + MasterSecret: req.MasterSecret, + Salt: req.Salt, }, sshProtoKeysToMap(req.SshUserKeys), req.HelmDeployments, @@ -240,7 +236,7 @@ type ClusterInitializer interface { enforceIDKeyDigest bool, idKeyDigest []byte, azureCVM bool, - kmsConfig resources.KMSConfig, + kmsConfig helm.KMSConfig, sshUserKeys map[string]string, helmDeployments []byte, conformanceMode bool, diff --git a/bootstrapper/internal/initserver/initserver_test.go b/bootstrapper/internal/initserver/initserver_test.go index ff7c2beff..36d1fc964 100644 --- a/bootstrapper/internal/initserver/initserver_test.go +++ b/bootstrapper/internal/initserver/initserver_test.go @@ -16,7 +16,7 @@ import ( "time" "github.com/edgelesssys/constellation/v2/bootstrapper/initproto" - "github.com/edgelesssys/constellation/v2/bootstrapper/internal/kubernetes/k8sapi/resources" + "github.com/edgelesssys/constellation/v2/bootstrapper/internal/helm" "github.com/edgelesssys/constellation/v2/internal/crypto/testvector" "github.com/edgelesssys/constellation/v2/internal/file" "github.com/edgelesssys/constellation/v2/internal/logger" @@ -290,7 +290,7 @@ type stubClusterInitializer struct { func (i *stubClusterInitializer) InitCluster( context.Context, string, string, []byte, []uint32, bool, []byte, bool, - resources.KMSConfig, map[string]string, []byte, bool, *logger.Logger, + helm.KMSConfig, map[string]string, []byte, bool, *logger.Logger, ) ([]byte, error) { return i.initClusterKubeconfig, i.initClusterErr } diff --git a/bootstrapper/internal/kubernetes/k8sapi/constants.go b/bootstrapper/internal/kubernetes/k8sapi/constants.go index baf3e0a76..779bebc6f 100644 --- a/bootstrapper/internal/kubernetes/k8sapi/constants.go +++ b/bootstrapper/internal/kubernetes/k8sapi/constants.go @@ -12,7 +12,6 @@ const ( binDir = "/run/state/bin" kubeadmPath = "/run/state/bin/kubeadm" kubeletPath = "/run/state/bin/kubelet" - kubectlPath = "/run/state/bin/kubectl" kubeletServiceEtcPath = "/etc/systemd/system/kubelet.service" kubeletServiceStatePath = "/run/state/systemd/system/kubelet.service" kubeadmConfEtcPath = "/etc/systemd/system/kubelet.service.d/10-kubeadm.conf" diff --git a/bootstrapper/internal/kubernetes/k8sapi/util.go b/bootstrapper/internal/kubernetes/k8sapi/k8sutil.go similarity index 77% rename from bootstrapper/internal/kubernetes/k8sapi/util.go rename to bootstrapper/internal/kubernetes/k8sapi/k8sutil.go index 79ec33a70..88ad4de34 100644 --- a/bootstrapper/internal/kubernetes/k8sapi/util.go +++ b/bootstrapper/internal/kubernetes/k8sapi/k8sutil.go @@ -10,7 +10,6 @@ import ( "context" "crypto/rand" "crypto/x509" - "encoding/json" "encoding/pem" "errors" "fmt" @@ -32,7 +31,6 @@ import ( kubeconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" "github.com/edgelesssys/constellation/v2/internal/crypto" - "github.com/edgelesssys/constellation/v2/internal/deploy/helm" "github.com/edgelesssys/constellation/v2/internal/file" "github.com/edgelesssys/constellation/v2/internal/logger" "github.com/edgelesssys/constellation/v2/internal/versions" @@ -40,20 +38,14 @@ import ( "github.com/spf13/afero" "go.uber.org/zap" "golang.org/x/text/transform" - "helm.sh/helm/v3/pkg/action" - "helm.sh/helm/v3/pkg/cli" corev1 "k8s.io/api/core/v1" ) const ( - // kubeConfig is the path to the Kubernetes admin config (used for authentication). - kubeConfig = "/etc/kubernetes/admin.conf" // kubeletStartTimeout is the maximum time given to the kubelet service to (re)start. kubeletStartTimeout = 10 * time.Minute // crdTimeout is the maximum time given to the CRDs to be created. crdTimeout = 30 * time.Second - // helmTimeout is the maximum time given to the helm client. - helmTimeout = 5 * time.Minute ) // Client provides the functions to talk to the k8s API. @@ -122,7 +114,7 @@ func (k *KubernetesUtil) InstallComponents(ctx context.Context, version versions return fmt.Errorf("installing kubeadm: %w", err) } if err := k.inst.Install( - ctx, versionConf.KubectlURL, []string{kubectlPath}, executablePerm, false, + ctx, versionConf.KubectlURL, []string{constants.KubectlPath}, executablePerm, false, ); err != nil { return fmt.Errorf("installing kubectl: %w", err) } @@ -235,19 +227,19 @@ func (k *KubernetesUtil) prepareControlPlaneForKonnectivity(ctx context.Context, return fmt.Errorf("generating konnectivity server certificate: %w", err) } - if out, err := exec.CommandContext(ctx, kubectlPath, "config", "set-credentials", "--kubeconfig", "/etc/kubernetes/konnectivity-server.conf", "system:konnectivity-server", + if out, err := exec.CommandContext(ctx, constants.KubectlPath, "config", "set-credentials", "--kubeconfig", "/etc/kubernetes/konnectivity-server.conf", "system:konnectivity-server", "--client-certificate", "/etc/kubernetes/konnectivity.crt", "--client-key", "/etc/kubernetes/konnectivity.key", "--embed-certs=true").CombinedOutput(); err != nil { return fmt.Errorf("konnectivity kubeconfig set-credentials: %w, %s", err, string(out)) } - if out, err := exec.CommandContext(ctx, kubectlPath, "--kubeconfig", "/etc/kubernetes/konnectivity-server.conf", "config", "set-cluster", "kubernetes", "--server", "https://"+loadBalancerEndpoint, + if out, err := exec.CommandContext(ctx, constants.KubectlPath, "--kubeconfig", "/etc/kubernetes/konnectivity-server.conf", "config", "set-cluster", "kubernetes", "--server", "https://"+loadBalancerEndpoint, "--certificate-authority", "/etc/kubernetes/pki/ca.crt", "--embed-certs=true").CombinedOutput(); err != nil { return fmt.Errorf("konnectivity kubeconfig set-cluster: %w, %s", err, string(out)) } - if out, err := exec.CommandContext(ctx, kubectlPath, "--kubeconfig", "/etc/kubernetes/konnectivity-server.conf", "config", "set-context", "system:konnectivity-server@kubernetes", + if out, err := exec.CommandContext(ctx, constants.KubectlPath, "--kubeconfig", "/etc/kubernetes/konnectivity-server.conf", "config", "set-context", "system:konnectivity-server@kubernetes", "--cluster", "kubernetes", "--user", "system:konnectivity-server").CombinedOutput(); err != nil { return fmt.Errorf("konnectivity kubeconfig set-context: %w, %s", err, string(out)) } - if out, err := exec.CommandContext(ctx, kubectlPath, "--kubeconfig", "/etc/kubernetes/konnectivity-server.conf", "config", "use-context", "system:konnectivity-server@kubernetes").CombinedOutput(); err != nil { + if out, err := exec.CommandContext(ctx, constants.KubectlPath, "--kubeconfig", "/etc/kubernetes/konnectivity-server.conf", "config", "use-context", "system:konnectivity-server@kubernetes").CombinedOutput(); err != nil { return fmt.Errorf("konnectivity kubeconfig use-context: %w, %s", err, string(out)) } // cleanup @@ -264,33 +256,6 @@ func (k *KubernetesUtil) SetupKonnectivity(kubectl Client, konnectivityAgentsDae return kubectl.Apply(konnectivityAgentsDaemonSet, true) } -func (k *KubernetesUtil) SetupHelmDeployments(ctx context.Context, kubectl Client, helmDeployments []byte, in SetupPodNetworkInput, log *logger.Logger) error { - var helmDeploy helm.Deployments - if err := json.Unmarshal(helmDeployments, &helmDeploy); err != nil { - return fmt.Errorf("unmarshalling helm deployments: %w", err) - } - settings := cli.New() - settings.KubeConfig = kubeConfig - - actionConfig := new(action.Configuration) - if err := actionConfig.Init(settings.RESTClientGetter(), constants.HelmNamespace, - "secret", log.Infof); err != nil { - return err - } - - helmClient := action.NewInstall(actionConfig) - helmClient.Namespace = constants.HelmNamespace - helmClient.ReleaseName = "cilium" - helmClient.Wait = true - helmClient.Timeout = helmTimeout - - if err := k.deployCilium(ctx, in, helmClient, helmDeploy.Cilium, kubectl); err != nil { - return fmt.Errorf("deploying cilium: %w", err) - } - - return nil -} - type SetupPodNetworkInput struct { CloudProvider string NodeName string @@ -300,85 +265,6 @@ type SetupPodNetworkInput struct { LoadBalancerEndpoint string } -// deployCilium sets up the cilium pod network. -func (k *KubernetesUtil) deployCilium(ctx context.Context, in SetupPodNetworkInput, helmClient *action.Install, ciliumDeployment helm.Deployment, kubectl Client) error { - switch in.CloudProvider { - case "gcp": - return k.deployCiliumGCP(ctx, helmClient, kubectl, ciliumDeployment, in.NodeName, in.FirstNodePodCIDR, in.SubnetworkPodCIDR, in.LoadBalancerEndpoint) - case "azure": - return k.deployCiliumAzure(ctx, helmClient, ciliumDeployment, in.LoadBalancerEndpoint) - case "qemu": - return k.deployCiliumQEMU(ctx, helmClient, ciliumDeployment, in.SubnetworkPodCIDR, in.LoadBalancerEndpoint) - default: - return fmt.Errorf("unsupported cloud provider %q", in.CloudProvider) - } -} - -func (k *KubernetesUtil) deployCiliumAzure(ctx context.Context, helmClient *action.Install, ciliumDeployment helm.Deployment, kubeAPIEndpoint string) error { - host := kubeAPIEndpoint - ciliumDeployment.Values["k8sServiceHost"] = host - ciliumDeployment.Values["k8sServicePort"] = strconv.Itoa(constants.KubernetesPort) - - _, err := helmClient.RunWithContext(ctx, ciliumDeployment.Chart, ciliumDeployment.Values) - if err != nil { - return fmt.Errorf("installing cilium: %w", err) - } - return nil -} - -func (k *KubernetesUtil) deployCiliumGCP(ctx context.Context, helmClient *action.Install, kubectl Client, ciliumDeployment helm.Deployment, nodeName, nodePodCIDR, subnetworkPodCIDR, kubeAPIEndpoint string) error { - out, err := exec.CommandContext(ctx, kubectlPath, "--kubeconfig", kubeConfig, "patch", "node", nodeName, "-p", "{\"spec\":{\"podCIDR\": \""+nodePodCIDR+"\"}}").CombinedOutput() - if err != nil { - err = errors.New(string(out)) - return err - } - - timeoutS := int64(10) - // allow coredns to run on uninitialized nodes (required by cloud-controller-manager) - tolerations := []corev1.Toleration{ - { - Key: "node.cloudprovider.kubernetes.io/uninitialized", - Value: "true", - Effect: corev1.TaintEffectNoSchedule, - }, - { - Key: "node.kubernetes.io/unreachable", - Operator: corev1.TolerationOpExists, - Effect: corev1.TaintEffectNoExecute, - TolerationSeconds: &timeoutS, - }, - } - if err = kubectl.AddTolerationsToDeployment(ctx, tolerations, "coredns", "kube-system"); err != nil { - return err - } - selectors := map[string]string{ - "node-role.kubernetes.io/control-plane": "", - } - if err = kubectl.AddNodeSelectorsToDeployment(ctx, selectors, "coredns", "kube-system"); err != nil { - return err - } - - host, port, err := net.SplitHostPort(kubeAPIEndpoint) - if err != nil { - return err - } - - // configure pod network CIDR - ciliumDeployment.Values["ipv4NativeRoutingCIDR"] = subnetworkPodCIDR - ciliumDeployment.Values["strictModeCIDR"] = subnetworkPodCIDR - ciliumDeployment.Values["k8sServiceHost"] = host - if port != "" { - ciliumDeployment.Values["k8sServicePort"] = port - } - - _, err = helmClient.RunWithContext(ctx, ciliumDeployment.Chart, ciliumDeployment.Values) - if err != nil { - return fmt.Errorf("installing cilium: %w", err) - } - - return nil -} - // FixCilium fixes https://github.com/cilium/cilium/issues/19958 but instead of a rollout restart of // the cilium daemonset, it only restarts the local cilium pod. func (k *KubernetesUtil) FixCilium(log *logger.Logger) { @@ -423,26 +309,6 @@ func (k *KubernetesUtil) FixCilium(log *logger.Logger) { } } -func (k *KubernetesUtil) deployCiliumQEMU(ctx context.Context, helmClient *action.Install, ciliumDeployment helm.Deployment, subnetworkPodCIDR, kubeAPIEndpoint string) error { - // configure pod network CIDR - ciliumDeployment.Values["ipam"] = map[string]interface{}{ - "operator": map[string]interface{}{ - "clusterPoolIPv4PodCIDRList": []interface{}{ - subnetworkPodCIDR, - }, - }, - } - - ciliumDeployment.Values["k8sServiceHost"] = kubeAPIEndpoint - ciliumDeployment.Values["k8sServicePort"] = strconv.Itoa(constants.KubernetesPort) - - _, err := helmClient.RunWithContext(ctx, ciliumDeployment.Chart, ciliumDeployment.Values) - if err != nil { - return fmt.Errorf("installing cilium: %w", err) - } - return nil -} - // SetupAutoscaling deploys the k8s cluster autoscaler. func (k *KubernetesUtil) SetupAutoscaling(kubectl Client, clusterAutoscalerConfiguration kubernetes.Marshaler, secrets kubernetes.Marshaler) error { if err := kubectl.Apply(secrets, true); err != nil { @@ -485,14 +351,6 @@ func (k *KubernetesUtil) SetupAccessManager(kubectl Client, accessManagerConfigu return kubectl.Apply(accessManagerConfiguration, true) } -// SetupKMS deploys the KMS deployment. -func (k *KubernetesUtil) SetupKMS(kubectl Client, kmsConfiguration kubernetes.Marshaler) error { - if err := kubectl.Apply(kmsConfiguration, true); err != nil { - return fmt.Errorf("applying KMS configuration: %w", err) - } - return nil -} - // SetupVerificationService deploys the verification service. func (k *KubernetesUtil) SetupVerificationService(kubectl Client, verificationServiceConfiguration kubernetes.Marshaler) error { return kubectl.Apply(verificationServiceConfiguration, true) diff --git a/bootstrapper/internal/kubernetes/k8sapi/resources/kms.go b/bootstrapper/internal/kubernetes/k8sapi/resources/kms.go deleted file mode 100644 index 0a339b997..000000000 --- a/bootstrapper/internal/kubernetes/k8sapi/resources/kms.go +++ /dev/null @@ -1,259 +0,0 @@ -/* -Copyright (c) Edgeless Systems GmbH - -SPDX-License-Identifier: AGPL-3.0-only -*/ - -package resources - -import ( - "fmt" - - "github.com/edgelesssys/constellation/v2/internal/constants" - "github.com/edgelesssys/constellation/v2/internal/kubernetes" - "github.com/edgelesssys/constellation/v2/internal/versions" - apps "k8s.io/api/apps/v1" - k8s "k8s.io/api/core/v1" - rbac "k8s.io/api/rbac/v1" - meta "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/util/intstr" -) - -const kmsNamespace = "kube-system" - -type KMSDeployment struct { - ServiceAccount k8s.ServiceAccount - Service k8s.Service - ClusterRole rbac.ClusterRole - ClusterRoleBinding rbac.ClusterRoleBinding - Deployment apps.DaemonSet - MasterSecret k8s.Secret -} - -// KMSConfig is the configuration needed to set up Constellation's key management service. -type KMSConfig struct { - MasterSecret []byte - Salt []byte - KMSURI string - StorageURI string - KeyEncryptionKeyID string - UseExistingKEK bool -} - -// NewKMSDeployment creates a new *kmsDeployment to use as the key management system inside Constellation. -func NewKMSDeployment(csp string, config KMSConfig) *KMSDeployment { - return &KMSDeployment{ - ServiceAccount: k8s.ServiceAccount{ - TypeMeta: meta.TypeMeta{ - APIVersion: "v1", - Kind: "ServiceAccount", - }, - ObjectMeta: meta.ObjectMeta{ - Name: "kms", - Namespace: kmsNamespace, - }, - }, - Service: k8s.Service{ - TypeMeta: meta.TypeMeta{ - APIVersion: "v1", - Kind: "Service", - }, - ObjectMeta: meta.ObjectMeta{ - Name: "kms", - Namespace: kmsNamespace, - }, - Spec: k8s.ServiceSpec{ - Type: k8s.ServiceTypeClusterIP, - Ports: []k8s.ServicePort{ - { - Name: "grpc", - Protocol: k8s.ProtocolTCP, - Port: constants.KMSPort, - TargetPort: intstr.FromInt(constants.KMSPort), - }, - }, - Selector: map[string]string{ - "k8s-app": "kms", - }, - }, - }, - ClusterRole: rbac.ClusterRole{ - TypeMeta: meta.TypeMeta{ - APIVersion: "rbac.authorization.k8s.io/v1", - Kind: "ClusterRole", - }, - ObjectMeta: meta.ObjectMeta{ - Name: "kms", - Labels: map[string]string{ - "k8s-app": "kms", - }, - }, - Rules: []rbac.PolicyRule{ - { - APIGroups: []string{""}, - Resources: []string{"secrets"}, - Verbs: []string{"get"}, - }, - }, - }, - ClusterRoleBinding: rbac.ClusterRoleBinding{ - TypeMeta: meta.TypeMeta{ - APIVersion: "rbac.authorization.k8s.io/v1", - Kind: "ClusterRoleBinding", - }, - ObjectMeta: meta.ObjectMeta{ - Name: "kms", - }, - RoleRef: rbac.RoleRef{ - APIGroup: "rbac.authorization.k8s.io", - Kind: "ClusterRole", - Name: "kms", - }, - Subjects: []rbac.Subject{ - { - Kind: "ServiceAccount", - Name: "kms", - Namespace: kmsNamespace, - }, - }, - }, - Deployment: apps.DaemonSet{ - TypeMeta: meta.TypeMeta{ - APIVersion: "apps/v1", - Kind: "DaemonSet", - }, - ObjectMeta: meta.ObjectMeta{ - Labels: map[string]string{ - "k8s-app": "kms", - "component": "kms", - "kubernetes.io/cluster-service": "true", - }, - Name: "kms", - Namespace: kmsNamespace, - }, - Spec: apps.DaemonSetSpec{ - Selector: &meta.LabelSelector{ - MatchLabels: map[string]string{ - "k8s-app": "kms", - }, - }, - Template: k8s.PodTemplateSpec{ - ObjectMeta: meta.ObjectMeta{ - Labels: map[string]string{ - "k8s-app": "kms", - }, - }, - Spec: k8s.PodSpec{ - PriorityClassName: "system-cluster-critical", - Tolerations: []k8s.Toleration{ - { - Key: "CriticalAddonsOnly", - Operator: k8s.TolerationOpExists, - }, - { - Key: "node-role.kubernetes.io/master", - Operator: k8s.TolerationOpEqual, - Value: "true", - Effect: k8s.TaintEffectNoSchedule, - }, - { - Key: "node-role.kubernetes.io/control-plane", - Operator: k8s.TolerationOpExists, - Effect: k8s.TaintEffectNoSchedule, - }, - { - Operator: k8s.TolerationOpExists, - Effect: k8s.TaintEffectNoExecute, - }, - { - Operator: k8s.TolerationOpExists, - Effect: k8s.TaintEffectNoSchedule, - }, - }, - // Only run on control plane nodes - NodeSelector: map[string]string{ - "node-role.kubernetes.io/control-plane": "", - }, - Volumes: []k8s.Volume{ - { - Name: "config", - VolumeSource: k8s.VolumeSource{ - Projected: &k8s.ProjectedVolumeSource{ - Sources: []k8s.VolumeProjection{ - { - ConfigMap: &k8s.ConfigMapProjection{ - LocalObjectReference: k8s.LocalObjectReference{ - Name: "join-config", - }, - Items: []k8s.KeyToPath{ - { - Key: constants.MeasurementsFilename, - Path: constants.MeasurementsFilename, - }, - }, - }, - }, - { - Secret: &k8s.SecretProjection{ - LocalObjectReference: k8s.LocalObjectReference{ - Name: constants.ConstellationMasterSecretStoreName, - }, - Items: []k8s.KeyToPath{ - { - Key: constants.ConstellationMasterSecretKey, - Path: constants.ConstellationMasterSecretKey, - }, - { - Key: constants.ConstellationMasterSecretSalt, - Path: constants.ConstellationMasterSecretSalt, - }, - }, - }, - }, - }, - }, - }, - }, - }, - ServiceAccountName: "kms", - Containers: []k8s.Container{ - { - Name: "kms", - Image: versions.KmsImage, - Args: []string{ - fmt.Sprintf("--port=%d", constants.KMSPort), - }, - VolumeMounts: []k8s.VolumeMount{ - { - Name: "config", - ReadOnly: true, - MountPath: constants.ServiceBasePath, - }, - }, - }, - }, - }, - }, - }, - }, - MasterSecret: k8s.Secret{ - TypeMeta: meta.TypeMeta{ - APIVersion: "v1", - Kind: "Secret", - }, - ObjectMeta: meta.ObjectMeta{ - Name: constants.ConstellationMasterSecretStoreName, - Namespace: kmsNamespace, - }, - Data: map[string][]byte{ - constants.ConstellationMasterSecretKey: config.MasterSecret, - constants.ConstellationMasterSecretSalt: config.Salt, - }, - Type: "Opaque", - }, - } -} - -func (c *KMSDeployment) Marshal() ([]byte, error) { - return kubernetes.MarshalK8SResources(c) -} diff --git a/bootstrapper/internal/kubernetes/k8sapi/resources/kms_test.go b/bootstrapper/internal/kubernetes/k8sapi/resources/kms_test.go deleted file mode 100644 index afe493274..000000000 --- a/bootstrapper/internal/kubernetes/k8sapi/resources/kms_test.go +++ /dev/null @@ -1,28 +0,0 @@ -/* -Copyright (c) Edgeless Systems GmbH - -SPDX-License-Identifier: AGPL-3.0-only -*/ - -package resources - -import ( - "testing" - - "github.com/edgelesssys/constellation/v2/internal/kubernetes" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestKMSMarshalUnmarshal(t *testing.T) { - require := require.New(t) - assert := assert.New(t) - - kmsDepl := NewKMSDeployment("test", KMSConfig{MasterSecret: []byte{0x0, 0x1, 0x2}, Salt: []byte{0x3, 0x4, 0x5}}) - data, err := kmsDepl.Marshal() - require.NoError(err) - - var recreated KMSDeployment - require.NoError(kubernetes.UnmarshalK8SResources(data, &recreated)) - assert.Equal(kmsDepl, &recreated) -} diff --git a/bootstrapper/internal/kubernetes/k8sutil.go b/bootstrapper/internal/kubernetes/k8sutil.go index 367edc64d..82d4066f0 100644 --- a/bootstrapper/internal/kubernetes/k8sutil.go +++ b/bootstrapper/internal/kubernetes/k8sutil.go @@ -10,7 +10,9 @@ import ( "context" "net" + helmClient "github.com/edgelesssys/constellation/v2/bootstrapper/internal/helm" "github.com/edgelesssys/constellation/v2/bootstrapper/internal/kubernetes/k8sapi" + "github.com/edgelesssys/constellation/v2/internal/deploy/helm" "github.com/edgelesssys/constellation/v2/internal/kubernetes" "github.com/edgelesssys/constellation/v2/internal/logger" "github.com/edgelesssys/constellation/v2/internal/role" @@ -21,19 +23,25 @@ type clusterUtil interface { InstallComponents(ctx context.Context, version versions.ValidK8sVersion) error InitCluster(ctx context.Context, initConfig []byte, nodeName string, ips []net.IP, controlPlaneEndpoint string, conformanceMode bool, log *logger.Logger) error JoinCluster(ctx context.Context, joinConfig []byte, peerRole role.Role, controlPlaneEndpoint string, log *logger.Logger) error - SetupHelmDeployments(ctx context.Context, client k8sapi.Client, helmDeployments []byte, in k8sapi.SetupPodNetworkInput, log *logger.Logger) error SetupAccessManager(kubectl k8sapi.Client, sshUsers kubernetes.Marshaler) error SetupAutoscaling(kubectl k8sapi.Client, clusterAutoscalerConfiguration kubernetes.Marshaler, secrets kubernetes.Marshaler) error SetupJoinService(kubectl k8sapi.Client, joinServiceConfiguration kubernetes.Marshaler) error SetupCloudControllerManager(kubectl k8sapi.Client, cloudControllerManagerConfiguration kubernetes.Marshaler, configMaps kubernetes.Marshaler, secrets kubernetes.Marshaler) error SetupCloudNodeManager(kubectl k8sapi.Client, cloudNodeManagerConfiguration kubernetes.Marshaler) error SetupKonnectivity(kubectl k8sapi.Client, konnectivityAgentsDaemonSet kubernetes.Marshaler) error - SetupKMS(kubectl k8sapi.Client, kmsConfiguration kubernetes.Marshaler) error SetupVerificationService(kubectl k8sapi.Client, verificationServiceConfiguration kubernetes.Marshaler) error SetupGCPGuestAgent(kubectl k8sapi.Client, gcpGuestAgentConfiguration kubernetes.Marshaler) error SetupOperatorLifecycleManager(ctx context.Context, kubectl k8sapi.Client, olmCRDs, olmConfiguration kubernetes.Marshaler, crdNames []string) error SetupNodeMaintenanceOperator(kubectl k8sapi.Client, nodeMaintenanceOperatorConfiguration kubernetes.Marshaler) error SetupNodeOperator(ctx context.Context, kubectl k8sapi.Client, nodeOperatorConfiguration kubernetes.Marshaler) error - StartKubelet() error FixCilium(log *logger.Logger) + StartKubelet() error +} + +// HelmUtil bundles functions related to microservice deployment. Only microservices that can be deployed purely via Helm are deployed with this interface. +// Currently only a subset of microservices is deployed via Helm. +// Naming is inspired by Helm. +type HelmUtil interface { + InstallCilium(context.Context, k8sapi.Client, helm.Release, k8sapi.SetupPodNetworkInput) error + InstallKMS(context.Context, helm.Release, helmClient.KMSConfig) error } diff --git a/bootstrapper/internal/kubernetes/kubernetes.go b/bootstrapper/internal/kubernetes/kubernetes.go index fcd45b0eb..64b1e5157 100644 --- a/bootstrapper/internal/kubernetes/kubernetes.go +++ b/bootstrapper/internal/kubernetes/kubernetes.go @@ -15,10 +15,12 @@ import ( "strconv" "strings" + helmClient "github.com/edgelesssys/constellation/v2/bootstrapper/internal/helm" "github.com/edgelesssys/constellation/v2/bootstrapper/internal/kubernetes/k8sapi" "github.com/edgelesssys/constellation/v2/bootstrapper/internal/kubernetes/k8sapi/resources" "github.com/edgelesssys/constellation/v2/internal/cloud/metadata" "github.com/edgelesssys/constellation/v2/internal/constants" + "github.com/edgelesssys/constellation/v2/internal/deploy/helm" "github.com/edgelesssys/constellation/v2/internal/logger" "github.com/edgelesssys/constellation/v2/internal/role" "github.com/edgelesssys/constellation/v2/internal/versions" @@ -44,6 +46,7 @@ type configurationProvider interface { type KubeWrapper struct { cloudProvider string clusterUtil clusterUtil + helmUtil HelmUtil configProvider configurationProvider client k8sapi.Client kubeconfigReader configReader @@ -57,11 +60,12 @@ type KubeWrapper struct { // New creates a new KubeWrapper with real values. func New(cloudProvider string, clusterUtil clusterUtil, configProvider configurationProvider, client k8sapi.Client, cloudControllerManager CloudControllerManager, - cloudNodeManager CloudNodeManager, clusterAutoscaler ClusterAutoscaler, providerMetadata ProviderMetadata, initialMeasurementsJSON []byte, + cloudNodeManager CloudNodeManager, clusterAutoscaler ClusterAutoscaler, providerMetadata ProviderMetadata, initialMeasurementsJSON []byte, helmUtil HelmUtil, ) *KubeWrapper { return &KubeWrapper{ cloudProvider: cloudProvider, clusterUtil: clusterUtil, + helmUtil: helmUtil, configProvider: configProvider, client: client, kubeconfigReader: &KubeconfigReader{fs: afero.Afero{Fs: afero.NewOsFs()}}, @@ -77,8 +81,8 @@ func New(cloudProvider string, clusterUtil clusterUtil, configProvider configura // InitCluster initializes a new Kubernetes cluster and applies pod network provider. func (k *KubeWrapper) InitCluster( ctx context.Context, cloudServiceAccountURI, versionString string, measurementSalt []byte, enforcedPCRs []uint32, - enforceIDKeyDigest bool, idKeyDigest []byte, azureCVM bool, kmsConfig resources.KMSConfig, sshUsers map[string]string, - helmDeployments []byte, conformanceMode bool, log *logger.Logger, + enforceIDKeyDigest bool, idKeyDigest []byte, azureCVM bool, kmsConfig helmClient.KMSConfig, sshUsers map[string]string, + helmReleasesRaw []byte, conformanceMode bool, log *logger.Logger, ) ([]byte, error) { k8sVersion, err := versions.NewValidK8sVersion(versionString) if err != nil { @@ -174,8 +178,14 @@ func (k *KubeWrapper) InitCluster( SubnetworkPodCIDR: subnetworkPodCIDR, LoadBalancerEndpoint: controlPlaneEndpoint, } - if err = k.clusterUtil.SetupHelmDeployments(ctx, k.client, helmDeployments, setupPodNetworkInput, log); err != nil { - return nil, fmt.Errorf("setting up pod network: %w", err) + + var helmReleases helm.Releases + if err := json.Unmarshal(helmReleasesRaw, &helmReleases); err != nil { + return nil, fmt.Errorf("unmarshalling helm releases: %w", err) + } + + if err = k.helmUtil.InstallCilium(ctx, k.client, helmReleases.Cilium, setupPodNetworkInput); err != nil { + return nil, fmt.Errorf("installing pod network: %w", err) } var controlPlaneIP string @@ -191,9 +201,8 @@ func (k *KubeWrapper) InitCluster( return nil, fmt.Errorf("setting up konnectivity: %w", err) } - kms := resources.NewKMSDeployment(k.cloudProvider, kmsConfig) - if err = k.clusterUtil.SetupKMS(k.client, kms); err != nil { - return nil, fmt.Errorf("setting up kms: %w", err) + if err = k.helmUtil.InstallKMS(ctx, helmReleases.KMS, kmsConfig); err != nil { + return nil, fmt.Errorf("installing kms: %w", err) } if err := k.setupInternalConfigMap(ctx, strconv.FormatBool(azureCVM)); err != nil { diff --git a/bootstrapper/internal/kubernetes/kubernetes_test.go b/bootstrapper/internal/kubernetes/kubernetes_test.go index 211fbde90..c75b1296f 100644 --- a/bootstrapper/internal/kubernetes/kubernetes_test.go +++ b/bootstrapper/internal/kubernetes/kubernetes_test.go @@ -14,10 +14,11 @@ import ( "strconv" "testing" + helmClient "github.com/edgelesssys/constellation/v2/bootstrapper/internal/helm" "github.com/edgelesssys/constellation/v2/bootstrapper/internal/kubernetes/k8sapi" - "github.com/edgelesssys/constellation/v2/bootstrapper/internal/kubernetes/k8sapi/resources" "github.com/edgelesssys/constellation/v2/internal/cloud/metadata" "github.com/edgelesssys/constellation/v2/internal/constants" + "github.com/edgelesssys/constellation/v2/internal/deploy/helm" "github.com/edgelesssys/constellation/v2/internal/kubernetes" "github.com/edgelesssys/constellation/v2/internal/logger" "github.com/edgelesssys/constellation/v2/internal/role" @@ -47,6 +48,7 @@ func TestInitCluster(t *testing.T) { testCases := map[string]struct { clusterUtil stubClusterUtil + helmUtil stubHelmClient kubectl stubKubectl providerMetadata ProviderMetadata CloudControllerManager CloudControllerManager @@ -178,8 +180,9 @@ func TestInitCluster(t *testing.T) { wantErr: true, k8sVersion: versions.Default, }, - "kubeadm init fails when deploying helm charts": { - clusterUtil: stubClusterUtil{setupHelmDeploymentsErr: someErr}, + "kubeadm init fails when deploying cilium": { + clusterUtil: stubClusterUtil{}, + helmUtil: stubHelmClient{ciliumError: someErr}, kubeconfigReader: &stubKubeconfigReader{ Kubeconfig: []byte("someKubeconfig"), }, @@ -251,7 +254,8 @@ func TestInitCluster(t *testing.T) { k8sVersion: versions.Default, }, "kubeadm init fails when setting up the kms": { - clusterUtil: stubClusterUtil{setupKMSError: someErr}, + clusterUtil: stubClusterUtil{}, + helmUtil: stubHelmClient{kmsError: someErr}, kubeconfigReader: &stubKubeconfigReader{ Kubeconfig: []byte("someKubeconfig"), }, @@ -307,6 +311,7 @@ func TestInitCluster(t *testing.T) { kube := KubeWrapper{ clusterUtil: &tc.clusterUtil, + helmUtil: &tc.helmUtil, providerMetadata: tc.providerMetadata, cloudControllerManager: tc.CloudControllerManager, cloudNodeManager: tc.CloudNodeManager, @@ -319,7 +324,7 @@ func TestInitCluster(t *testing.T) { _, err := kube.InitCluster( context.Background(), serviceAccountURI, string(tc.k8sVersion), - nil, nil, false, nil, true, resources.KMSConfig{MasterSecret: masterSecret}, nil, nil, false, logger.NewTest(t), + nil, nil, false, nil, true, helmClient.KMSConfig{MasterSecret: masterSecret}, nil, []byte("{}"), false, logger.NewTest(t), ) if tc.wantErr { @@ -528,13 +533,11 @@ func TestK8sCompliantHostname(t *testing.T) { type stubClusterUtil struct { installComponentsErr error initClusterErr error - setupHelmDeploymentsErr error setupAutoscalingError error setupJoinServiceError error setupCloudControllerManagerError error setupCloudNodeManagerError error setupKonnectivityError error - setupKMSError error setupAccessManagerError error setupVerificationServiceErr error setupGCPGuestAgentErr error @@ -561,10 +564,6 @@ func (s *stubClusterUtil) InitCluster(ctx context.Context, initConfig []byte, no return s.initClusterErr } -func (s *stubClusterUtil) SetupHelmDeployments(context.Context, k8sapi.Client, []byte, k8sapi.SetupPodNetworkInput, *logger.Logger) error { - return s.setupHelmDeploymentsErr -} - func (s *stubClusterUtil) SetupAutoscaling(kubectl k8sapi.Client, clusterAutoscalerConfiguration kubernetes.Marshaler, secrets kubernetes.Marshaler) error { return s.setupAutoscalingError } @@ -581,10 +580,6 @@ func (s *stubClusterUtil) SetupCloudControllerManager(kubectl k8sapi.Client, clo return s.setupCloudControllerManagerError } -func (s *stubClusterUtil) SetupKMS(kubectl k8sapi.Client, kmsDeployment kubernetes.Marshaler) error { - return s.setupKMSError -} - func (s *stubClusterUtil) SetupAccessManager(kubectl k8sapi.Client, accessManagerConfiguration kubernetes.Marshaler) error { return s.setupAccessManagerError } @@ -685,3 +680,16 @@ type stubKubeconfigReader struct { func (s *stubKubeconfigReader) ReadKubeconfig() ([]byte, error) { return s.Kubeconfig, s.ReadErr } + +type stubHelmClient struct { + ciliumError error + kmsError error +} + +func (s *stubHelmClient) InstallCilium(ctx context.Context, kubectl k8sapi.Client, release helm.Release, in k8sapi.SetupPodNetworkInput) error { + return s.ciliumError +} + +func (s *stubHelmClient) InstallKMS(ctx context.Context, release helm.Release, kmsConfig helmClient.KMSConfig) error { + return s.kmsError +} diff --git a/cli/internal/helm/loader.go b/cli/internal/helm/loader.go index dea6bdeab..3b2ae6f32 100644 --- a/cli/internal/helm/loader.go +++ b/cli/internal/helm/loader.go @@ -16,7 +16,9 @@ import ( "strings" "github.com/edgelesssys/constellation/v2/internal/cloud/cloudprovider" + "github.com/edgelesssys/constellation/v2/internal/constants" "github.com/edgelesssys/constellation/v2/internal/deploy/helm" + "github.com/edgelesssys/constellation/v2/internal/versions" "github.com/pkg/errors" "helm.sh/helm/pkg/ignore" "helm.sh/helm/v3/pkg/chart" @@ -26,28 +28,33 @@ import ( // Run `go generate` to deterministically create the patched Helm deployment for cilium //go:generate ./generateCilium.sh -//go:embed all:charts/cilium/* +//go:embed all:charts/* var HelmFS embed.FS type ChartLoader struct{} func (i *ChartLoader) Load(csp cloudprovider.Provider, conformanceMode bool) ([]byte, error) { - ciliumDeployment, err := i.loadCilium(csp, conformanceMode) + ciliumRelease, err := i.loadCilium(csp, conformanceMode) if err != nil { return nil, err } - deployments := helm.Deployments{Cilium: ciliumDeployment} - depl, err := json.Marshal(deployments) + + kmsRelease, err := i.loadKMS() if err != nil { return nil, err } - return depl, nil + releases := helm.Releases{Cilium: ciliumRelease, KMS: kmsRelease} + rel, err := json.Marshal(releases) + if err != nil { + return nil, err + } + return rel, nil } -func (i *ChartLoader) loadCilium(csp cloudprovider.Provider, conformanceMode bool) (helm.Deployment, error) { +func (i *ChartLoader) loadCilium(csp cloudprovider.Provider, conformanceMode bool) (helm.Release, error) { chart, err := loadChartsDir(HelmFS, "charts/cilium") if err != nil { - return helm.Deployment{}, err + return helm.Release{}, err } var ciliumVals map[string]interface{} switch csp { @@ -58,7 +65,7 @@ func (i *ChartLoader) loadCilium(csp cloudprovider.Provider, conformanceMode boo case cloudprovider.QEMU: ciliumVals = qemuVals default: - return helm.Deployment{}, fmt.Errorf("unknown csp: %s", csp) + return helm.Release{}, fmt.Errorf("unknown csp: %s", csp) } if conformanceMode { ciliumVals["kubeProxyReplacementHealthzBindAddr"] = "" @@ -70,7 +77,27 @@ func (i *ChartLoader) loadCilium(csp cloudprovider.Provider, conformanceMode boo } - return helm.Deployment{Chart: chart, Values: ciliumVals}, nil + return helm.Release{Chart: chart, Values: ciliumVals, ReleaseName: "cilium", Wait: true}, nil +} + +func (i *ChartLoader) loadKMS() (helm.Release, error) { + chart, err := loadChartsDir(HelmFS, "charts/edgeless/kms") + if err != nil { + return helm.Release{}, err + } + kmsVals := map[string]interface{}{ + "namespace": constants.ConstellationNamespace, + "kmsPort": constants.KMSPort, + "joinConfigCMName": constants.JoinConfigMap, + "serviceBasePath": constants.ServiceBasePath, + "kmsImage": versions.KmsImage, + "masterSecretName": constants.ConstellationMasterSecretStoreName, + "masterSecretKeyName": constants.ConstellationMasterSecretKey, + "saltKeyName": constants.ConstellationSaltKey, + "measurementsFilename": constants.MeasurementsFilename, + } + + return helm.Release{Chart: chart, Values: kmsVals, ReleaseName: "kms", Wait: true}, nil } // taken from loader.LoadDir from the helm go module diff --git a/internal/constants/constants.go b/internal/constants/constants.go index a720c1148..7f9a32fe0 100644 --- a/internal/constants/constants.go +++ b/internal/constants/constants.go @@ -23,10 +23,10 @@ const ( ConstellationNameLength = 37 // ConstellationMasterSecretStoreName is the name for the Constellation secrets in Kubernetes. ConstellationMasterSecretStoreName = "constellation-mastersecret" - // ConstellationMasterSecretKey is the name of the key for master secret in the master secret store secret. + // ConstellationMasterSecretKey is the name of the key for the master secret in the master secret kubernetes secret. ConstellationMasterSecretKey = "mastersecret" - // ConstellationMasterSecretSalt is the name of the key for salt in the master secret store secret. - ConstellationMasterSecretSalt = "salt" + // ConstellationSaltKey is the name of the key for the salt in the master secret kubernetes secret. + ConstellationSaltKey = "salt" // // Ports. @@ -67,6 +67,7 @@ const ( WGQuickConfigFilename = "wg0.conf" CoreOSAdminConfFilename = "/etc/kubernetes/admin.conf" KubeadmCertificateDir = "/etc/kubernetes/pki" + KubectlPath = "/run/state/bin/kubectl" // // Filenames for Constellation's micro services. diff --git a/internal/deploy/helm/helm.go b/internal/deploy/helm/helm.go index 6ae8d27fb..2e6c7ec2d 100644 --- a/internal/deploy/helm/helm.go +++ b/internal/deploy/helm/helm.go @@ -8,11 +8,15 @@ package helm import "helm.sh/helm/v3/pkg/chart" -type Deployment struct { - Chart *chart.Chart - Values map[string]interface{} +// Release bundles all information necessary to create a helm release. +type Release struct { + Chart *chart.Chart + Values map[string]interface{} + ReleaseName string + Wait bool } -type Deployments struct { - Cilium Deployment +type Releases struct { + Cilium Release + KMS Release } diff --git a/kms/cmd/main.go b/kms/cmd/main.go index 479567ffa..f980aed2c 100644 --- a/kms/cmd/main.go +++ b/kms/cmd/main.go @@ -28,7 +28,7 @@ import ( func main() { port := flag.String("port", strconv.Itoa(constants.KMSPort), "Port gRPC server listens on") masterSecretPath := flag.String("master-secret", filepath.Join(constants.ServiceBasePath, constants.ConstellationMasterSecretKey), "Path to the Constellation master secret") - saltPath := flag.String("salt", filepath.Join(constants.ServiceBasePath, constants.ConstellationMasterSecretSalt), "Path to the Constellation salt") + saltPath := flag.String("salt", filepath.Join(constants.ServiceBasePath, constants.ConstellationSaltKey), "Path to the Constellation salt") verbosity := flag.Int("v", 0, logger.CmdLineVerbosityDescription) flag.Parse()