Refactor Helm deployments (#341)

* Wrap KMS deployment in one main chart that
deploys all other services. Other services will follow.
* Use .tgz via helm-package as serialization format
* Change Release type to carry chart as byte slice
* Remove KMSConfig
* Use json-schema to validate values
* Extend release.md to mention updating helm charts

bootstrapper/cmd/bootstrapper/test.go

@@ -9,7 +9,6 @@ package main
 import (
 	"context"
 
-	"github.com/edgelesssys/constellation/v2/bootstrapper/internal/helm"
 	"github.com/edgelesssys/constellation/v2/internal/cloud/metadata"
 	"github.com/edgelesssys/constellation/v2/internal/logger"
 	"github.com/edgelesssys/constellation/v2/internal/role"
@@ -22,7 +21,7 @@ type clusterFake struct{}
 // InitCluster fakes bootstrapping a new cluster with the current node being the master, returning the arguments required to join the cluster.
 func (c *clusterFake) InitCluster(
 	context.Context, string, string, []byte, []uint32, bool, []byte, bool,
-	helm.KMSConfig, map[string]string, []byte, bool, *logger.Logger,
+	map[string]string, []byte, bool, *logger.Logger,
 ) ([]byte, error) {
 	return []byte{}, nil
 }

bootstrapper/internal/helm/client.go

@@ -7,8 +7,8 @@ SPDX-License-Identifier: AGPL-3.0-only
 package helm
 
 import (
+	"bytes"
 	"context"
-	"encoding/base64"
 	"errors"
 	"fmt"
 	"net"
@@ -21,6 +21,7 @@ import (
 	"github.com/edgelesssys/constellation/v2/internal/deploy/helm"
 	"github.com/edgelesssys/constellation/v2/internal/logger"
 	"helm.sh/helm/v3/pkg/action"
+	"helm.sh/helm/v3/pkg/chart/loader"
 	"helm.sh/helm/v3/pkg/cli"
 	corev1 "k8s.io/api/core/v1"
 )
@@ -51,6 +52,31 @@ func New(log *logger.Logger) (*Client, error) {
 	}, nil
 }
 
+// InstallConstellationServices installs the constellation-services chart. In the future this chart should bundle all microservices.
+func (h *Client) InstallConstellationServices(ctx context.Context, release helm.Release) error {
+	h.Namespace = constants.HelmNamespace
+	h.ReleaseName = release.ReleaseName
+	h.Wait = release.Wait
+	h.Timeout = timeout
+	// update dependencies - unsure if necessary for local deps.
+	h.DependencyUpdate = true
+
+	// TODO: Possibly fetch metadata to extend values here.
+
+	reader := bytes.NewReader(release.Chart)
+	chart, err := loader.LoadArchive(reader)
+	if err != nil {
+		return fmt.Errorf("helm load archive: %w", err)
+	}
+
+	_, err = h.RunWithContext(ctx, chart, release.Values)
+	if err != nil {
+		return fmt.Errorf("helm install services: %w", err)
+	}
+
+	return nil
+}
+
 // InstallCilium sets up the cilium pod network.
 func (h *Client) InstallCilium(ctx context.Context, kubectl k8sapi.Client, release helm.Release, in k8sapi.SetupPodNetworkInput) error {
 	h.Namespace = constants.HelmNamespace
@@ -75,7 +101,13 @@ func (h *Client) installCiliumAzure(ctx context.Context, release helm.Release, k
 	release.Values["k8sServiceHost"] = host
 	release.Values["k8sServicePort"] = strconv.Itoa(constants.KubernetesPort)
 
-	_, err := h.RunWithContext(ctx, release.Chart, release.Values)
+	reader := bytes.NewReader(release.Chart)
+	chart, err := loader.LoadArchive(reader)
+	if err != nil {
+		return fmt.Errorf("helm load archive: %w", err)
+	}
+
+	_, err = h.RunWithContext(ctx, chart, release.Values)
 	if err != nil {
 		return fmt.Errorf("installing cilium: %w", err)
 	}
@@ -127,7 +159,13 @@ func (h *Client) installlCiliumGCP(ctx context.Context, kubectl k8sapi.Client, r
 		release.Values["k8sServicePort"] = port
 	}
 
-	_, err = h.RunWithContext(ctx, release.Chart, release.Values)
+	reader := bytes.NewReader(release.Chart)
+	chart, err := loader.LoadArchive(reader)
+	if err != nil {
+		return fmt.Errorf("helm load archive: %w", err)
+	}
+
+	_, err = h.RunWithContext(ctx, chart, release.Values)
 	if err != nil {
 		return fmt.Errorf("helm install cilium: %w", err)
 	}
@@ -148,27 +186,15 @@ func (h *Client) installCiliumQEMU(ctx context.Context, release helm.Release, su
 	release.Values["k8sServiceHost"] = kubeAPIEndpoint
 	release.Values["k8sServicePort"] = strconv.Itoa(constants.KubernetesPort)
 
-	_, err := h.RunWithContext(ctx, release.Chart, release.Values)
+	reader := bytes.NewReader(release.Chart)
+	chart, err := loader.LoadArchive(reader)
+	if err != nil {
+		return fmt.Errorf("helm load archive: %w", err)
+	}
+
+	_, err = h.RunWithContext(ctx, chart, release.Values)
 	if err != nil {
 		return fmt.Errorf("helm install cilium: %w", err)
 	}
 	return nil
 }
-
-// InstallKMS deploys the KMS deployment.
-func (h *Client) InstallKMS(ctx context.Context, release helm.Release, kmsConfig KMSConfig) error {
-	h.Namespace = constants.HelmNamespace
-	h.ReleaseName = release.ReleaseName
-	h.Wait = release.Wait
-	h.Timeout = timeout
-
-	release.Values["masterSecret"] = base64.StdEncoding.EncodeToString(kmsConfig.MasterSecret[:])
-	release.Values["salt"] = base64.StdEncoding.EncodeToString(kmsConfig.Salt[:])
-
-	_, err := h.RunWithContext(ctx, release.Chart, release.Values)
-	if err != nil {
-		return fmt.Errorf("helm install kms: %w", err)
-	}
-
-	return nil
-}

bootstrapper/internal/helm/kms.go

@@ -1,13 +0,0 @@
-/*
-Copyright (c) Edgeless Systems GmbH
-
-SPDX-License-Identifier: AGPL-3.0-only
-*/
-
-package helm
-
-// KMSConfig is the configuration needed to set up Constellation's key management service.
-type KMSConfig struct {
-	MasterSecret []byte
-	Salt         []byte
-}

bootstrapper/internal/initserver/initserver.go

@@ -15,7 +15,6 @@ import (
 
 	"github.com/edgelesssys/constellation/v2/bootstrapper/initproto"
 	"github.com/edgelesssys/constellation/v2/bootstrapper/internal/diskencryption"
-	"github.com/edgelesssys/constellation/v2/bootstrapper/internal/helm"
 	"github.com/edgelesssys/constellation/v2/internal/atls"
 	"github.com/edgelesssys/constellation/v2/internal/attestation"
 	"github.com/edgelesssys/constellation/v2/internal/cloud/vmtype"
@@ -132,10 +131,6 @@ func (s *Server) Init(ctx context.Context, req *initproto.InitRequest) (*initpro
 		req.EnforceIdkeydigest,
 		s.issuerWrapper.IDKeyDigest(),
 		s.issuerWrapper.VMType() == vmtype.AzureCVM,
-		helm.KMSConfig{
-			MasterSecret: req.MasterSecret,
-			Salt:         req.Salt,
-		},
 		sshProtoKeysToMap(req.SshUserKeys),
 		req.HelmDeployments,
 		req.ConformanceMode,
@@ -236,7 +231,6 @@ type ClusterInitializer interface {
 		enforceIDKeyDigest bool,
 		idKeyDigest []byte,
 		azureCVM bool,
-		kmsConfig helm.KMSConfig,
 		sshUserKeys map[string]string,
 		helmDeployments []byte,
 		conformanceMode bool,

bootstrapper/internal/initserver/initserver_test.go

@@ -16,7 +16,6 @@ import (
 	"time"
 
 	"github.com/edgelesssys/constellation/v2/bootstrapper/initproto"
-	"github.com/edgelesssys/constellation/v2/bootstrapper/internal/helm"
 	"github.com/edgelesssys/constellation/v2/internal/crypto/testvector"
 	"github.com/edgelesssys/constellation/v2/internal/file"
 	"github.com/edgelesssys/constellation/v2/internal/logger"
@@ -290,7 +289,7 @@ type stubClusterInitializer struct {
 
 func (i *stubClusterInitializer) InitCluster(
 	context.Context, string, string, []byte, []uint32, bool, []byte, bool,
-	helm.KMSConfig, map[string]string, []byte, bool, *logger.Logger,
+	map[string]string, []byte, bool, *logger.Logger,
 ) ([]byte, error) {
 	return i.initClusterKubeconfig, i.initClusterErr
 }

bootstrapper/internal/kubernetes/k8sutil.go

@@ -10,7 +10,6 @@ import (
 	"context"
 	"net"
 
-	helmClient "github.com/edgelesssys/constellation/v2/bootstrapper/internal/helm"
 	"github.com/edgelesssys/constellation/v2/bootstrapper/internal/kubernetes/k8sapi"
 	"github.com/edgelesssys/constellation/v2/internal/deploy/helm"
 	"github.com/edgelesssys/constellation/v2/internal/kubernetes"
@@ -38,10 +37,10 @@ type clusterUtil interface {
 	StartKubelet() error
 }
 
-// HelmUtil bundles functions related to microservice deployment. Only microservices that can be deployed purely via Helm are deployed with this interface.
+// helmClient bundles functions related to microservice deployment. Only microservices that can be deployed purely via Helm are deployed with this interface.
 // Currently only a subset of microservices is deployed via Helm.
 // Naming is inspired by Helm.
-type HelmUtil interface {
+type helmClient interface {
 	InstallCilium(context.Context, k8sapi.Client, helm.Release, k8sapi.SetupPodNetworkInput) error
-	InstallKMS(context.Context, helm.Release, helmClient.KMSConfig) error
+	InstallConstellationServices(ctx context.Context, release helm.Release) error
 }

bootstrapper/internal/kubernetes/kubernetes.go

@@ -15,7 +15,6 @@ import (
 	"strconv"
 	"strings"
 
-	helmClient "github.com/edgelesssys/constellation/v2/bootstrapper/internal/helm"
 	"github.com/edgelesssys/constellation/v2/bootstrapper/internal/kubernetes/k8sapi"
 	"github.com/edgelesssys/constellation/v2/bootstrapper/internal/kubernetes/k8sapi/resources"
 	"github.com/edgelesssys/constellation/v2/internal/cloud/metadata"
@@ -46,7 +45,7 @@ type configurationProvider interface {
 type KubeWrapper struct {
 	cloudProvider           string
 	clusterUtil             clusterUtil
-	helmUtil                HelmUtil
+	helmClient              helmClient
 	configProvider          configurationProvider
 	client                  k8sapi.Client
 	kubeconfigReader        configReader
@@ -60,12 +59,12 @@ type KubeWrapper struct {
 
 // New creates a new KubeWrapper with real values.
 func New(cloudProvider string, clusterUtil clusterUtil, configProvider configurationProvider, client k8sapi.Client, cloudControllerManager CloudControllerManager,
-	cloudNodeManager CloudNodeManager, clusterAutoscaler ClusterAutoscaler, providerMetadata ProviderMetadata, initialMeasurementsJSON []byte, helmUtil HelmUtil,
+	cloudNodeManager CloudNodeManager, clusterAutoscaler ClusterAutoscaler, providerMetadata ProviderMetadata, initialMeasurementsJSON []byte, helmClient helmClient,
 ) *KubeWrapper {
 	return &KubeWrapper{
 		cloudProvider:           cloudProvider,
 		clusterUtil:             clusterUtil,
-		helmUtil:                helmUtil,
+		helmClient:              helmClient,
 		configProvider:          configProvider,
 		client:                  client,
 		kubeconfigReader:        &KubeconfigReader{fs: afero.Afero{Fs: afero.NewOsFs()}},
@@ -81,7 +80,7 @@ func New(cloudProvider string, clusterUtil clusterUtil, configProvider configura
 // InitCluster initializes a new Kubernetes cluster and applies pod network provider.
 func (k *KubeWrapper) InitCluster(
 	ctx context.Context, cloudServiceAccountURI, versionString string, measurementSalt []byte, enforcedPCRs []uint32,
-	enforceIDKeyDigest bool, idKeyDigest []byte, azureCVM bool, kmsConfig helmClient.KMSConfig, sshUsers map[string]string,
+	enforceIDKeyDigest bool, idKeyDigest []byte, azureCVM bool, sshUsers map[string]string,
 	helmReleasesRaw []byte, conformanceMode bool, log *logger.Logger,
 ) ([]byte, error) {
 	k8sVersion, err := versions.NewValidK8sVersion(versionString)
@@ -184,7 +183,7 @@ func (k *KubeWrapper) InitCluster(
 		return nil, fmt.Errorf("unmarshalling helm releases: %w", err)
 	}
 
-	if err = k.helmUtil.InstallCilium(ctx, k.client, helmReleases.Cilium, setupPodNetworkInput); err != nil {
+	if err = k.helmClient.InstallCilium(ctx, k.client, helmReleases.Cilium, setupPodNetworkInput); err != nil {
 		return nil, fmt.Errorf("installing pod network: %w", err)
 	}
 
@@ -201,7 +200,7 @@ func (k *KubeWrapper) InitCluster(
 		return nil, fmt.Errorf("setting up konnectivity: %w", err)
 	}
 
-	if err = k.helmUtil.InstallKMS(ctx, helmReleases.KMS, kmsConfig); err != nil {
+	if err = k.helmClient.InstallConstellationServices(ctx, helmReleases.ConstellationServices); err != nil {
 		return nil, fmt.Errorf("installing kms: %w", err)
 	}
 

bootstrapper/internal/kubernetes/kubernetes_test.go

@@ -14,7 +14,6 @@ import (
 	"strconv"
 	"testing"
 
-	helmClient "github.com/edgelesssys/constellation/v2/bootstrapper/internal/helm"
 	"github.com/edgelesssys/constellation/v2/bootstrapper/internal/kubernetes/k8sapi"
 	"github.com/edgelesssys/constellation/v2/internal/cloud/metadata"
 	"github.com/edgelesssys/constellation/v2/internal/constants"
@@ -37,7 +36,6 @@ func TestMain(m *testing.M) {
 func TestInitCluster(t *testing.T) {
 	someErr := errors.New("failed")
 	serviceAccountURI := "some-service-account-uri"
-	masterSecret := []byte("some-master-secret")
 
 	nodeName := "node-name"
 	providerID := "provider-id"
@@ -48,7 +46,7 @@ func TestInitCluster(t *testing.T) {
 
 	testCases := map[string]struct {
 		clusterUtil            stubClusterUtil
-		helmUtil               stubHelmClient
+		helmClient             stubHelmClient
 		kubectl                stubKubectl
 		providerMetadata       ProviderMetadata
 		CloudControllerManager CloudControllerManager
@@ -182,7 +180,7 @@ func TestInitCluster(t *testing.T) {
 		},
 		"kubeadm init fails when deploying cilium": {
 			clusterUtil: stubClusterUtil{},
-			helmUtil:    stubHelmClient{ciliumError: someErr},
+			helmClient:  stubHelmClient{ciliumError: someErr},
 			kubeconfigReader: &stubKubeconfigReader{
 				Kubeconfig: []byte("someKubeconfig"),
 			},
@@ -255,7 +253,7 @@ func TestInitCluster(t *testing.T) {
 		},
 		"kubeadm init fails when setting up the kms": {
 			clusterUtil: stubClusterUtil{},
-			helmUtil:    stubHelmClient{kmsError: someErr},
+			helmClient:  stubHelmClient{servicesError: someErr},
 			kubeconfigReader: &stubKubeconfigReader{
 				Kubeconfig: []byte("someKubeconfig"),
 			},
@@ -311,7 +309,7 @@ func TestInitCluster(t *testing.T) {
 
 			kube := KubeWrapper{
 				clusterUtil:            &tc.clusterUtil,
-				helmUtil:               &tc.helmUtil,
+				helmClient:             &tc.helmClient,
 				providerMetadata:       tc.providerMetadata,
 				cloudControllerManager: tc.CloudControllerManager,
 				cloudNodeManager:       tc.CloudNodeManager,
@@ -324,7 +322,7 @@ func TestInitCluster(t *testing.T) {
 
 			_, err := kube.InitCluster(
 				context.Background(), serviceAccountURI, string(tc.k8sVersion),
-				nil, nil, false, nil, true, helmClient.KMSConfig{MasterSecret: masterSecret}, nil, []byte("{}"), false, logger.NewTest(t),
+				nil, nil, false, nil, true, nil, []byte("{}"), false, logger.NewTest(t),
 			)
 
 			if tc.wantErr {
@@ -682,14 +680,14 @@ func (s *stubKubeconfigReader) ReadKubeconfig() ([]byte, error) {
 }
 
 type stubHelmClient struct {
-	ciliumError error
-	kmsError    error
+	ciliumError   error
+	servicesError error
 }
 
 func (s *stubHelmClient) InstallCilium(ctx context.Context, kubectl k8sapi.Client, release helm.Release, in k8sapi.SetupPodNetworkInput) error {
 	return s.ciliumError
 }
 
-func (s *stubHelmClient) InstallKMS(ctx context.Context, release helm.Release, kmsConfig helmClient.KMSConfig) error {
-	return s.kmsError
+func (s *stubHelmClient) InstallConstellationServices(ctx context.Context, release helm.Release) error {
+	return s.servicesError
 }