AB#2504: Deploy join-service via helm (#358)

bootstrapper/internal/helm/client.go

@@ -53,7 +53,7 @@ func New(log *logger.Logger) (*Client, error) {
 }
 
 // InstallConstellationServices installs the constellation-services chart. In the future this chart should bundle all microservices.
-func (h *Client) InstallConstellationServices(ctx context.Context, release helm.Release) error {
+func (h *Client) InstallConstellationServices(ctx context.Context, release helm.Release, extraVals map[string]interface{}) error {
 	h.Namespace = constants.HelmNamespace
 	h.ReleaseName = release.ReleaseName
 	h.Wait = release.Wait
@@ -61,7 +61,7 @@ func (h *Client) InstallConstellationServices(ctx context.Context, release helm.
 	// update dependencies - unsure if necessary for local deps.
 	h.DependencyUpdate = true
 
-	// TODO: Possibly fetch metadata to extend values here.
+	mergedVals := mergeMaps(release.Values, extraVals)
 
 	reader := bytes.NewReader(release.Chart)
 	chart, err := loader.LoadArchive(reader)
@@ -69,7 +69,7 @@ func (h *Client) InstallConstellationServices(ctx context.Context, release helm.
 		return fmt.Errorf("helm load archive: %w", err)
 	}
 
-	_, err = h.RunWithContext(ctx, chart, release.Values)
+	_, err = h.RunWithContext(ctx, chart, mergedVals)
 	if err != nil {
 		return fmt.Errorf("helm install services: %w", err)
 	}
@@ -77,6 +77,27 @@ func (h *Client) InstallConstellationServices(ctx context.Context, release helm.
 	return nil
 }
 
+// mergeMaps returns a new map that is the merger of it's inputs.
+// Taken from: https://github.com/helm/helm/blob/dbc6d8e20fe1d58d50e6ed30f09a04a77e4c68db/pkg/cli/values/options.go#L91-L108.
+func mergeMaps(a, b map[string]interface{}) map[string]interface{} {
+	out := make(map[string]interface{}, len(a))
+	for k, v := range a {
+		out[k] = v
+	}
+	for k, v := range b {
+		if v, ok := v.(map[string]interface{}); ok {
+			if bv, ok := out[k]; ok {
+				if bv, ok := bv.(map[string]interface{}); ok {
+					out[k] = mergeMaps(bv, v)
+					continue
+				}
+			}
+		}
+		out[k] = v
+	}
+	return out
+}
+
 // InstallCilium sets up the cilium pod network.
 func (h *Client) InstallCilium(ctx context.Context, kubectl k8sapi.Client, release helm.Release, in k8sapi.SetupPodNetworkInput) error {
 	h.Namespace = constants.HelmNamespace

bootstrapper/internal/helm/client_test.go

@@ -0,0 +1,92 @@
+/*
+Copyright (c) Edgeless Systems GmbH
+
+SPDX-License-Identifier: AGPL-3.0-only
+*/
+
+package helm
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestMe(t *testing.T) {
+	testCases := map[string]struct {
+		vals      map[string]interface{}
+		extraVals map[string]interface{}
+		expected  map[string]interface{}
+	}{
+		"equal": {
+			vals: map[string]interface{}{
+				"join-service": map[string]interface{}{
+					"key1": "foo",
+					"key2": "bar",
+				},
+			},
+			extraVals: map[string]interface{}{
+				"join-service": map[string]interface{}{
+					"extraKey1": "extraFoo",
+					"extraKey2": "extraBar",
+				},
+			},
+			expected: map[string]interface{}{
+				"join-service": map[string]interface{}{
+					"key1":      "foo",
+					"key2":      "bar",
+					"extraKey1": "extraFoo",
+					"extraKey2": "extraBar",
+				},
+			},
+		},
+		"missing join-service extraVals": {
+			vals: map[string]interface{}{
+				"join-service": map[string]interface{}{
+					"key1": "foo",
+					"key2": "bar",
+				},
+			},
+			extraVals: map[string]interface{}{
+				"extraKey1": "extraFoo",
+				"extraKey2": "extraBar",
+			},
+			expected: map[string]interface{}{
+				"join-service": map[string]interface{}{
+					"key1": "foo",
+					"key2": "bar",
+				},
+				"extraKey1": "extraFoo",
+				"extraKey2": "extraBar",
+			},
+		},
+		"missing join-service vals": {
+			vals: map[string]interface{}{
+				"key1": "foo",
+				"key2": "bar",
+			},
+			extraVals: map[string]interface{}{
+				"join-service": map[string]interface{}{
+					"extraKey1": "extraFoo",
+					"extraKey2": "extraBar",
+				},
+			},
+			expected: map[string]interface{}{
+				"key1": "foo",
+				"key2": "bar",
+				"join-service": map[string]interface{}{
+					"extraKey1": "extraFoo",
+					"extraKey2": "extraBar",
+				},
+			},
+		},
+	}
+
+	for name, tc := range testCases {
+		t.Run(name, func(t *testing.T) {
+			assert := assert.New(t)
+			newVals := mergeMaps(tc.vals, tc.extraVals)
+			assert.Equal(tc.expected, newVals)
+		})
+	}
+}

bootstrapper/internal/kubernetes/k8sapi/constants.go

@@ -12,7 +12,6 @@ const (
 	binDir                  = "/run/state/bin"
 	kubeadmPath             = "/run/state/bin/kubeadm"
 	kubeletPath             = "/run/state/bin/kubelet"
-	kubectlPath             = "/run/state/bin/kubectl"
 	kubeletServiceEtcPath   = "/run/systemd/system/kubelet.service"
 	kubeletServiceStatePath = "/run/state/systemd/system/kubelet.service"
 	kubeadmConfEtcPath      = "/run/systemd/system/kubelet.service.d/10-kubeadm.conf"

bootstrapper/internal/kubernetes/k8sapi/k8sutil.go

@@ -334,11 +334,6 @@ func (k *KubernetesUtil) SetupAutoscaling(kubectl Client, clusterAutoscalerConfi
 	return kubectl.Apply(clusterAutoscalerConfiguration, true)
 }
 
-// SetupJoinService deploys the Constellation node join service.
-func (k *KubernetesUtil) SetupJoinService(kubectl Client, joinServiceConfiguration kubernetes.Marshaler) error {
-	return kubectl.Apply(joinServiceConfiguration, true)
-}
-
 // SetupGCPGuestAgent deploys the GCP guest agent daemon set.
 func (k *KubernetesUtil) SetupGCPGuestAgent(kubectl Client, guestAgentDaemonset kubernetes.Marshaler) error {
 	return kubectl.Apply(guestAgentDaemonset, true)

bootstrapper/internal/kubernetes/k8sapi/resources/joinservice.go

@@ -1,277 +0,0 @@
-/*
-Copyright (c) Edgeless Systems GmbH
-
-SPDX-License-Identifier: AGPL-3.0-only
-*/
-
-package resources
-
-import (
-	"fmt"
-
-	"github.com/edgelesssys/constellation/v2/internal/cloud/cloudprovider"
-	"github.com/edgelesssys/constellation/v2/internal/constants"
-	"github.com/edgelesssys/constellation/v2/internal/kubernetes"
-	"github.com/edgelesssys/constellation/v2/internal/versions"
-	apps "k8s.io/api/apps/v1"
-	k8s "k8s.io/api/core/v1"
-	rbac "k8s.io/api/rbac/v1"
-	meta "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"k8s.io/apimachinery/pkg/util/intstr"
-)
-
-type JoinServiceDaemonset struct {
-	ClusterRole        rbac.ClusterRole
-	ClusterRoleBinding rbac.ClusterRoleBinding
-	ConfigMap          k8s.ConfigMap
-	DaemonSet          apps.DaemonSet
-	ServiceAccount     k8s.ServiceAccount
-	Service            k8s.Service
-}
-
-// NewJoinServiceDaemonset returns a daemonset for the join service.
-func NewJoinServiceDaemonset(csp, measurementsJSON, enforcedPCRsJSON, initialIDKeyDigest, enforceIDKeyDigest string, measurementSalt []byte) *JoinServiceDaemonset {
-	joinConfigData := map[string]string{
-		constants.MeasurementsFilename: measurementsJSON,
-		constants.EnforcedPCRsFilename: enforcedPCRsJSON,
-	}
-	if cloudprovider.FromString(csp) == cloudprovider.Azure {
-		joinConfigData[constants.EnforceIDKeyDigestFilename] = enforceIDKeyDigest
-		joinConfigData[constants.IDKeyDigestFilename] = initialIDKeyDigest
-	}
-
-	return &JoinServiceDaemonset{
-		ClusterRole: rbac.ClusterRole{
-			TypeMeta: meta.TypeMeta{
-				APIVersion: "rbac.authorization.k8s.io/v1",
-				Kind:       "ClusterRole",
-			},
-			ObjectMeta: meta.ObjectMeta{
-				Name: "join-service",
-				Labels: map[string]string{
-					"k8s-app": "join-service",
-				},
-			},
-			Rules: []rbac.PolicyRule{
-				{
-					APIGroups: []string{""},
-					Resources: []string{"secrets"},
-					Verbs:     []string{"get", "list", "create", "update"},
-				},
-				{
-					APIGroups: []string{"rbac.authorization.k8s.io"},
-					Resources: []string{"roles", "rolebindings"},
-					Verbs:     []string{"create", "update"},
-				},
-			},
-		},
-		ClusterRoleBinding: rbac.ClusterRoleBinding{
-			TypeMeta: meta.TypeMeta{
-				APIVersion: "rbac.authorization.k8s.io/v1",
-				Kind:       "ClusterRoleBinding",
-			},
-			ObjectMeta: meta.ObjectMeta{
-				Name: "join-service",
-			},
-			RoleRef: rbac.RoleRef{
-				APIGroup: "rbac.authorization.k8s.io",
-				Kind:     "ClusterRole",
-				Name:     "join-service",
-			},
-			Subjects: []rbac.Subject{
-				{
-					Kind:      "ServiceAccount",
-					Name:      "join-service",
-					Namespace: constants.ConstellationNamespace,
-				},
-			},
-		},
-		DaemonSet: apps.DaemonSet{
-			TypeMeta: meta.TypeMeta{
-				APIVersion: "apps/v1",
-				Kind:       "DaemonSet",
-			},
-			ObjectMeta: meta.ObjectMeta{
-				Name:      "join-service",
-				Namespace: constants.ConstellationNamespace,
-				Labels: map[string]string{
-					"k8s-app":                       "join-service",
-					"component":                     "join-service",
-					"kubernetes.io/cluster-service": "true",
-				},
-			},
-			Spec: apps.DaemonSetSpec{
-				Selector: &meta.LabelSelector{
-					MatchLabels: map[string]string{
-						"k8s-app": "join-service",
-					},
-				},
-				Template: k8s.PodTemplateSpec{
-					ObjectMeta: meta.ObjectMeta{
-						Labels: map[string]string{
-							"k8s-app": "join-service",
-						},
-					},
-					Spec: k8s.PodSpec{
-						PriorityClassName:  "system-cluster-critical",
-						ServiceAccountName: "join-service",
-						Tolerations: []k8s.Toleration{
-							{
-								Key:      "CriticalAddonsOnly",
-								Operator: k8s.TolerationOpExists,
-							},
-							{
-								Key:      "node-role.kubernetes.io/master",
-								Operator: k8s.TolerationOpEqual,
-								Value:    "true",
-								Effect:   k8s.TaintEffectNoSchedule,
-							},
-							{
-								Key:      "node-role.kubernetes.io/control-plane",
-								Operator: k8s.TolerationOpExists,
-								Effect:   k8s.TaintEffectNoSchedule,
-							},
-							{
-								Operator: k8s.TolerationOpExists,
-								Effect:   k8s.TaintEffectNoExecute,
-							},
-							{
-								Operator: k8s.TolerationOpExists,
-								Effect:   k8s.TaintEffectNoSchedule,
-							},
-						},
-						// Only run on control plane nodes
-						NodeSelector: map[string]string{
-							"node-role.kubernetes.io/control-plane": "",
-						},
-						Containers: []k8s.Container{
-							{
-								Name:  "join-service",
-								Image: versions.JoinImage,
-								Ports: []k8s.ContainerPort{
-									{
-										ContainerPort: constants.JoinServicePort,
-										Name:          "tcp",
-									},
-								},
-								SecurityContext: &k8s.SecurityContext{
-									Privileged: func(b bool) *bool { return &b }(true),
-								},
-								Args: []string{
-									fmt.Sprintf("--cloud-provider=%s", csp),
-									fmt.Sprintf("--kms-endpoint=kms.kube-system:%d", constants.KMSPort),
-								},
-								VolumeMounts: []k8s.VolumeMount{
-									{
-										Name:      "config",
-										ReadOnly:  true,
-										MountPath: constants.ServiceBasePath,
-									},
-									{
-										Name:      "kubeadm",
-										ReadOnly:  true,
-										MountPath: "/etc/kubernetes",
-									},
-								},
-							},
-						},
-						Volumes: []k8s.Volume{
-							{
-								Name: "config",
-								VolumeSource: k8s.VolumeSource{
-									Projected: &k8s.ProjectedVolumeSource{
-										Sources: []k8s.VolumeProjection{
-											{
-												ConfigMap: &k8s.ConfigMapProjection{
-													LocalObjectReference: k8s.LocalObjectReference{
-														Name: constants.JoinConfigMap,
-													},
-												},
-											},
-											{
-												ConfigMap: &k8s.ConfigMapProjection{
-													LocalObjectReference: k8s.LocalObjectReference{
-														Name: constants.K8sVersion,
-													},
-												},
-											},
-											{
-												ConfigMap: &k8s.ConfigMapProjection{
-													LocalObjectReference: k8s.LocalObjectReference{
-														Name: constants.InternalConfigMap,
-													},
-												},
-											},
-										},
-									},
-								},
-							},
-							{
-								Name: "kubeadm",
-								VolumeSource: k8s.VolumeSource{
-									HostPath: &k8s.HostPathVolumeSource{
-										Path: "/etc/kubernetes",
-									},
-								},
-							},
-						},
-					},
-				},
-			},
-		},
-		ServiceAccount: k8s.ServiceAccount{
-			TypeMeta: meta.TypeMeta{
-				APIVersion: "v1",
-				Kind:       "ServiceAccount",
-			},
-			ObjectMeta: meta.ObjectMeta{
-				Name:      "join-service",
-				Namespace: constants.ConstellationNamespace,
-			},
-		},
-		Service: k8s.Service{
-			TypeMeta: meta.TypeMeta{
-				APIVersion: "v1",
-				Kind:       "Service",
-			},
-			ObjectMeta: meta.ObjectMeta{
-				Name:      "join-service",
-				Namespace: constants.ConstellationNamespace,
-			},
-			Spec: k8s.ServiceSpec{
-				Type: k8s.ServiceTypeNodePort,
-				Ports: []k8s.ServicePort{
-					{
-						Name:       "grpc",
-						Protocol:   k8s.ProtocolTCP,
-						Port:       constants.JoinServicePort,
-						TargetPort: intstr.IntOrString{IntVal: constants.JoinServicePort},
-						NodePort:   constants.JoinServiceNodePort,
-					},
-				},
-				Selector: map[string]string{
-					"k8s-app": "join-service",
-				},
-			},
-		},
-		ConfigMap: k8s.ConfigMap{
-			TypeMeta: meta.TypeMeta{
-				APIVersion: "v1",
-				Kind:       "ConfigMap",
-			},
-			ObjectMeta: meta.ObjectMeta{
-				Name:      constants.JoinConfigMap,
-				Namespace: constants.ConstellationNamespace,
-			},
-			Data: joinConfigData,
-			BinaryData: map[string][]byte{
-				constants.MeasurementSaltFilename: measurementSalt,
-			},
-		},
-	}
-}
-
-// Marshal the daemonset using the Kubernetes resource marshaller.
-func (a *JoinServiceDaemonset) Marshal() ([]byte, error) {
-	return kubernetes.MarshalK8SResources(a)
-}

bootstrapper/internal/kubernetes/k8sapi/resources/joinservice_test.go

@@ -1,25 +0,0 @@
-/*
-Copyright (c) Edgeless Systems GmbH
-
-SPDX-License-Identifier: AGPL-3.0-only
-*/
-
-package resources
-
-import (
-	"testing"
-
-	"github.com/edgelesssys/constellation/v2/internal/kubernetes"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-)
-
-func TestNewJoinServiceDaemonset(t *testing.T) {
-	deployment := NewJoinServiceDaemonset("csp", "measurementsJSON", "enforcedPCRsJSON", "deadbeef", "true", []byte{0x0, 0x1, 0x2})
-	deploymentYAML, err := deployment.Marshal()
-	require.NoError(t, err)
-
-	var recreated JoinServiceDaemonset
-	require.NoError(t, kubernetes.UnmarshalK8SResources(deploymentYAML, &recreated))
-	assert.Equal(t, deployment, &recreated)
-}

bootstrapper/internal/kubernetes/k8sutil.go

@@ -24,7 +24,6 @@ type clusterUtil interface {
 	JoinCluster(ctx context.Context, joinConfig []byte, peerRole role.Role, controlPlaneEndpoint string, log *logger.Logger) error
 	SetupAccessManager(kubectl k8sapi.Client, sshUsers kubernetes.Marshaler) error
 	SetupAutoscaling(kubectl k8sapi.Client, clusterAutoscalerConfiguration kubernetes.Marshaler, secrets kubernetes.Marshaler) error
-	SetupJoinService(kubectl k8sapi.Client, joinServiceConfiguration kubernetes.Marshaler) error
 	SetupCloudControllerManager(kubectl k8sapi.Client, cloudControllerManagerConfiguration kubernetes.Marshaler, configMaps kubernetes.Marshaler, secrets kubernetes.Marshaler) error
 	SetupCloudNodeManager(kubectl k8sapi.Client, cloudNodeManagerConfiguration kubernetes.Marshaler) error
 	SetupKonnectivity(kubectl k8sapi.Client, konnectivityAgentsDaemonSet kubernetes.Marshaler) error
@@ -42,5 +41,5 @@ type clusterUtil interface {
 // Naming is inspired by Helm.
 type helmClient interface {
 	InstallCilium(context.Context, k8sapi.Client, helm.Release, k8sapi.SetupPodNetworkInput) error
-	InstallConstellationServices(ctx context.Context, release helm.Release) error
+	InstallConstellationServices(ctx context.Context, release helm.Release, extraVals map[string]interface{}) error
 }

bootstrapper/internal/kubernetes/kubernetes.go

@@ -8,6 +8,7 @@ package kubernetes
 
 import (
 	"context"
+	"encoding/base64"
 	"encoding/hex"
 	"encoding/json"
 	"fmt"
@@ -200,18 +201,16 @@ func (k *KubeWrapper) InitCluster(
 		return nil, fmt.Errorf("setting up konnectivity: %w", err)
 	}
 
-	if err = k.helmClient.InstallConstellationServices(ctx, helmReleases.ConstellationServices); err != nil {
-		return nil, fmt.Errorf("installing kms: %w", err)
+	extraVals := setupExtraVals(k.initialMeasurementsJSON, idKeyDigest, measurementSalt)
+
+	if err = k.helmClient.InstallConstellationServices(ctx, helmReleases.ConstellationServices, extraVals); err != nil {
+		return nil, fmt.Errorf("installing constellation-services: %w", err)
 	}
 
 	if err := k.setupInternalConfigMap(ctx, strconv.FormatBool(azureCVM)); err != nil {
 		return nil, fmt.Errorf("failed to setup internal ConfigMap: %w", err)
 	}
 
-	if err := k.setupJoinService(k.cloudProvider, k.initialMeasurementsJSON, measurementSalt, enforcedPCRs, idKeyDigest, enforceIDKeyDigest); err != nil {
-		return nil, fmt.Errorf("setting up join service failed: %w", err)
-	}
-
 	if err := k.setupCCM(ctx, subnetworkPodCIDR, cloudServiceAccountURI, instance, k8sVersion); err != nil {
 		return nil, fmt.Errorf("setting up cloud controller manager: %w", err)
 	}
@@ -329,21 +328,6 @@ func (k *KubeWrapper) GetKubeconfig() ([]byte, error) {
 	return k.kubeconfigReader.ReadKubeconfig()
 }
 
-func (k *KubeWrapper) setupJoinService(
-	csp string, measurementsJSON, measurementSalt []byte, enforcedPCRs []uint32, initialIDKeyDigest []byte, enforceIDKeyDigest bool,
-) error {
-	enforcedPCRsJSON, err := json.Marshal(enforcedPCRs)
-	if err != nil {
-		return fmt.Errorf("marshaling enforcedPCRs: %w", err)
-	}
-
-	joinConfiguration := resources.NewJoinServiceDaemonset(
-		csp, string(measurementsJSON), string(enforcedPCRsJSON), hex.EncodeToString(initialIDKeyDigest), strconv.FormatBool(enforceIDKeyDigest), measurementSalt,
-	)
-
-	return k.clusterUtil.SetupJoinService(k.client, joinConfiguration)
-}
-
 func (k *KubeWrapper) setupCCM(ctx context.Context, subnetworkPodCIDR, cloudServiceAccountURI string, instance metadata.InstanceMetadata, k8sVersion versions.ValidK8sVersion) error {
 	if !k.cloudControllerManager.Supported() {
 		return nil
@@ -511,3 +495,15 @@ func getIPAddr() (string, error) {
 
 	return localAddr.IP.String(), nil
 }
+
+// setupExtraVals create a helm values map for consumption by helm-install.
+// Will move to a more dedicated place once that place becomes apparent.
+func setupExtraVals(initialMeasurementsJSON []byte, idkeydigest []byte, measurementSalt []byte) map[string]interface{} {
+	return map[string]interface{}{
+		"join-service": map[string]interface{}{
+			"measurements":    string(initialMeasurementsJSON),
+			"idkeydigest":     hex.EncodeToString(idkeydigest),
+			"measurementSalt": base64.StdEncoding.EncodeToString(measurementSalt),
+		},
+	}
+}

bootstrapper/internal/kubernetes/kubernetes_test.go

@@ -192,7 +192,8 @@ func TestInitCluster(t *testing.T) {
 			k8sVersion:             versions.Default,
 		},
 		"kubeadm init fails when setting up the join service": {
-			clusterUtil: stubClusterUtil{setupJoinServiceError: someErr},
+			clusterUtil: stubClusterUtil{},
+			helmClient:  stubHelmClient{servicesError: someErr},
 			kubeconfigReader: &stubKubeconfigReader{
 				Kubeconfig: []byte("someKubeconfig"),
 			},
@@ -532,7 +533,6 @@ type stubClusterUtil struct {
 	installComponentsErr             error
 	initClusterErr                   error
 	setupAutoscalingError            error
-	setupJoinServiceError            error
 	setupCloudControllerManagerError error
 	setupCloudNodeManagerError       error
 	setupKonnectivityError           error
@@ -566,10 +566,6 @@ func (s *stubClusterUtil) SetupAutoscaling(kubectl k8sapi.Client, clusterAutosca
 	return s.setupAutoscalingError
 }
 
-func (s *stubClusterUtil) SetupJoinService(kubectl k8sapi.Client, joinServiceConfiguration kubernetes.Marshaler) error {
-	return s.setupJoinServiceError
-}
-
 func (s *stubClusterUtil) SetupGCPGuestAgent(kubectl k8sapi.Client, gcpGuestAgentConfiguration kubernetes.Marshaler) error {
 	return s.setupGCPGuestAgentErr
 }
@@ -688,6 +684,6 @@ func (s *stubHelmClient) InstallCilium(ctx context.Context, kubectl k8sapi.Clien
 	return s.ciliumError
 }
 
-func (s *stubHelmClient) InstallConstellationServices(ctx context.Context, release helm.Release) error {
+func (s *stubHelmClient) InstallConstellationServices(ctx context.Context, release helm.Release, extraVals map[string]interface{}) error {
 	return s.servicesError
 }