Microservice upgrades (#729)

Run with: constellation upgrade execute --helm.
This will only upgrade the helm charts. No config is needed.

Upgrades are implemented via helm's upgrade action, i.e. they
automatically roll back if something goes wrong. Releases could 
still be managed via helm, even after an upgrade with constellation
has been done.

Currently not user facing as CRD/CR backups are still in progress.
These backups should be automatically created and saved to the 
user's disk as updates may delete CRs. This happens implicitly 
through CRD upgrades, which are part of microservice upgrades.

bootstrapper/internal/helm/client.go

@@ -61,7 +61,7 @@ func (h *Client) InstallConstellationServices(ctx context.Context, release helm.
 	h.ReleaseName = release.ReleaseName
 	h.Wait = release.Wait
 
-	mergedVals := mergeMaps(release.Values, extraVals)
+	mergedVals := helm.MergeMaps(release.Values, extraVals)
 
 	if err := h.install(ctx, release.Chart, mergedVals); err != nil {
 		return err
@@ -87,7 +87,7 @@ func (h *Client) InstallOperators(ctx context.Context, release helm.Release, ext
 	h.ReleaseName = release.ReleaseName
 	h.Wait = release.Wait
 
-	mergedVals := mergeMaps(release.Values, extraVals)
+	mergedVals := helm.MergeMaps(release.Values, extraVals)
 
 	if err := h.install(ctx, release.Chart, mergedVals); err != nil {
 		return err
@@ -190,24 +190,3 @@ func (h *Client) install(ctx context.Context, chartRaw []byte, values map[string
 	}
 	return nil
 }
-
-// mergeMaps returns a new map that is the merger of it's inputs.
-// Taken from: https://github.com/helm/helm/blob/dbc6d8e20fe1d58d50e6ed30f09a04a77e4c68db/pkg/cli/values/options.go#L91-L108.
-func mergeMaps(a, b map[string]any) map[string]any {
-	out := make(map[string]any, len(a))
-	for k, v := range a {
-		out[k] = v
-	}
-	for k, v := range b {
-		if v, ok := v.(map[string]any); ok {
-			if bv, ok := out[k]; ok {
-				if bv, ok := bv.(map[string]any); ok {
-					out[k] = mergeMaps(bv, v)
-					continue
-				}
-			}
-		}
-		out[k] = v
-	}
-	return out
-}

bootstrapper/internal/helm/client_test.go

@@ -5,88 +5,3 @@ SPDX-License-Identifier: AGPL-3.0-only
 */
 
 package helm
-
-import (
-	"testing"
-
-	"github.com/stretchr/testify/assert"
-)
-
-func TestMe(t *testing.T) {
-	testCases := map[string]struct {
-		vals      map[string]any
-		extraVals map[string]any
-		expected  map[string]any
-	}{
-		"equal": {
-			vals: map[string]any{
-				"join-service": map[string]any{
-					"key1": "foo",
-					"key2": "bar",
-				},
-			},
-			extraVals: map[string]any{
-				"join-service": map[string]any{
-					"extraKey1": "extraFoo",
-					"extraKey2": "extraBar",
-				},
-			},
-			expected: map[string]any{
-				"join-service": map[string]any{
-					"key1":      "foo",
-					"key2":      "bar",
-					"extraKey1": "extraFoo",
-					"extraKey2": "extraBar",
-				},
-			},
-		},
-		"missing join-service extraVals": {
-			vals: map[string]any{
-				"join-service": map[string]any{
-					"key1": "foo",
-					"key2": "bar",
-				},
-			},
-			extraVals: map[string]any{
-				"extraKey1": "extraFoo",
-				"extraKey2": "extraBar",
-			},
-			expected: map[string]any{
-				"join-service": map[string]any{
-					"key1": "foo",
-					"key2": "bar",
-				},
-				"extraKey1": "extraFoo",
-				"extraKey2": "extraBar",
-			},
-		},
-		"missing join-service vals": {
-			vals: map[string]any{
-				"key1": "foo",
-				"key2": "bar",
-			},
-			extraVals: map[string]any{
-				"join-service": map[string]any{
-					"extraKey1": "extraFoo",
-					"extraKey2": "extraBar",
-				},
-			},
-			expected: map[string]any{
-				"key1": "foo",
-				"key2": "bar",
-				"join-service": map[string]any{
-					"extraKey1": "extraFoo",
-					"extraKey2": "extraBar",
-				},
-			},
-		},
-	}
-
-	for name, tc := range testCases {
-		t.Run(name, func(t *testing.T) {
-			assert := assert.New(t)
-			newVals := mergeMaps(tc.vals, tc.extraVals)
-			assert.Equal(tc.expected, newVals)
-		})
-	}
-}

cli/internal/cloudcmd/upgrade.go

@@ -12,11 +12,14 @@ import (
 	"errors"
 	"fmt"
 	"io"
+	"time"
 
 	"github.com/edgelesssys/constellation/v2/cli/internal/helm"
 	"github.com/edgelesssys/constellation/v2/internal/attestation/measurements"
+	"github.com/edgelesssys/constellation/v2/internal/config"
 	"github.com/edgelesssys/constellation/v2/internal/constants"
 	corev1 "k8s.io/api/core/v1"
+	apiextensionsclient "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"k8s.io/apimachinery/pkg/runtime/schema"
@@ -35,7 +38,7 @@ type Upgrader struct {
 }
 
 // NewUpgrader returns a new Upgrader.
-func NewUpgrader(outWriter io.Writer) (*Upgrader, error) {
+func NewUpgrader(outWriter io.Writer, log debugLog) (*Upgrader, error) {
 	kubeConfig, err := clientcmd.BuildConfigFromFlags("", constants.AdminConfFilename)
 	if err != nil {
 		return nil, fmt.Errorf("building kubernetes config: %w", err)
@@ -52,7 +55,12 @@ func NewUpgrader(outWriter io.Writer) (*Upgrader, error) {
 		return nil, fmt.Errorf("setting up custom resource client: %w", err)
 	}
 
-	helmClient, err := helm.NewClient(constants.AdminConfFilename, constants.HelmNamespace)
+	client, err := apiextensionsclient.NewForConfig(kubeConfig)
+	if err != nil {
+		return nil, err
+	}
+
+	helmClient, err := helm.NewClient(constants.AdminConfFilename, constants.HelmNamespace, client, log)
 	if err != nil {
 		return nil, fmt.Errorf("setting up helm client: %w", err)
 	}
@@ -105,6 +113,11 @@ func (u *Upgrader) GetCurrentImage(ctx context.Context) (*unstructured.Unstructu
 	return imageStruct, imageVersion, nil
 }
 
+// UpgradeHelmServices upgrade helm services.
+func (u *Upgrader) UpgradeHelmServices(ctx context.Context, config *config.Config, timeout time.Duration) error {
+	return u.helmClient.Upgrade(ctx, config, timeout)
+}
+
 // CurrentHelmVersion returns the version of the currently installed helm release.
 func (u *Upgrader) CurrentHelmVersion(release string) (string, error) {
 	return u.helmClient.CurrentVersion(release)
@@ -232,4 +245,10 @@ func (u *stableClient) kubernetesVersion() (string, error) {
 
 type helmInterface interface {
 	CurrentVersion(release string) (string, error)
+	Upgrade(ctx context.Context, config *config.Config, timeout time.Duration) error
+}
+
+type debugLog interface {
+	Debugf(format string, args ...any)
+	Sync()
 }

cli/internal/cmd/upgradeexecute.go

@@ -8,6 +8,8 @@ package cmd
 
 import (
 	"context"
+	"fmt"
+	"time"
 
 	"github.com/edgelesssys/constellation/v2/cli/internal/cloudcmd"
 	"github.com/edgelesssys/constellation/v2/internal/attestation/measurements"
@@ -27,13 +29,28 @@ func newUpgradeExecuteCmd() *cobra.Command {
 		RunE:  runUpgradeExecute,
 	}
 
+	cmd.Flags().Bool("helm", false, "Execute helm upgrade. This feature is still in development an may change without anounncement. Upgrades all helm charts deployed during constellation-init.")
+	cmd.Flags().Duration("timeout", 3*time.Minute, "Change helm upgrade timeout. This feature is still in development an may change without anounncement. Might be useful for slow connections or big clusters.")
+	if err := cmd.Flags().MarkHidden("helm"); err != nil {
+		panic(err)
+	}
+	if err := cmd.Flags().MarkHidden("timeout"); err != nil {
+		panic(err)
+	}
+
 	return cmd
 }
 
 func runUpgradeExecute(cmd *cobra.Command, args []string) error {
+	log, err := newCLILogger(cmd)
+	if err != nil {
+		return fmt.Errorf("creating logger: %w", err)
+	}
+	defer log.Sync()
+
 	fileHandler := file.NewHandler(afero.NewOsFs())
 	imageFetcher := image.New()
-	upgrader, err := cloudcmd.NewUpgrader(cmd.OutOrStdout())
+	upgrader, err := cloudcmd.NewUpgrader(cmd.OutOrStdout(), log)
 	if err != nil {
 		return err
 	}
@@ -51,6 +68,21 @@ func upgradeExecute(cmd *cobra.Command, imageFetcher imageFetcher, upgrader clou
 		return displayConfigValidationErrors(cmd.ErrOrStderr(), err)
 	}
 
+	helm, err := cmd.Flags().GetBool("helm")
+	if err != nil {
+		return err
+	}
+	if helm {
+		timeout, err := cmd.Flags().GetDuration("timeout")
+		if err != nil {
+			return err
+		}
+		if err := upgrader.UpgradeHelmServices(cmd.Context(), conf, timeout); err != nil {
+			return fmt.Errorf("upgrading helm: %w", err)
+		}
+		return nil
+	}
+
 	// TODO: validate upgrade config? Should be basic things like checking image is not an empty string
 	// More sophisticated validation, like making sure we don't downgrade the cluster, should be done by `constellation upgrade plan`
 
@@ -66,6 +98,7 @@ func upgradeExecute(cmd *cobra.Command, imageFetcher imageFetcher, upgrader clou
 
 type cloudUpgrader interface {
 	Upgrade(ctx context.Context, imageReference, imageVersion string, measurements measurements.M) error
+	UpgradeHelmServices(ctx context.Context, config *config.Config, timeout time.Duration) error
 }
 
 type imageFetcher interface {

cli/internal/cmd/upgradeexecute_test.go

@@ -10,6 +10,7 @@ import (
 	"context"
 	"errors"
 	"testing"
+	"time"
 
 	"github.com/edgelesssys/constellation/v2/internal/attestation/measurements"
 	"github.com/edgelesssys/constellation/v2/internal/cloud/cloudprovider"
@@ -66,13 +67,18 @@ func TestUpgradeExecute(t *testing.T) {
 }
 
 type stubUpgrader struct {
-	err error
+	err     error
+	helmErr error
 }
 
 func (u stubUpgrader) Upgrade(context.Context, string, string, measurements.M) error {
 	return u.err
 }
 
+func (u stubUpgrader) UpgradeHelmServices(ctx context.Context, config *config.Config, timeout time.Duration) error {
+	return u.helmErr
+}
+
 type stubImageFetcher struct {
 	reference         string
 	fetchReferenceErr error

cli/internal/cmd/upgradeplan.go

@@ -44,12 +44,18 @@ func newUpgradePlanCmd() *cobra.Command {
 }
 
 func runUpgradePlan(cmd *cobra.Command, args []string) error {
+	log, err := newCLILogger(cmd)
+	if err != nil {
+		return fmt.Errorf("creating logger: %w", err)
+	}
+	defer log.Sync()
+
 	fileHandler := file.NewHandler(afero.NewOsFs())
 	flags, err := parseUpgradePlanFlags(cmd)
 	if err != nil {
 		return err
 	}
-	planner, err := cloudcmd.NewUpgrader(cmd.OutOrStdout())
+	planner, err := cloudcmd.NewUpgrader(cmd.OutOrStdout(), log)
 	if err != nil {
 		return err
 	}

cli/internal/helm/README.md

@@ -0,0 +1,23 @@
+# Chart upgrades
+
+All services that are installed via helm-install are upgraded via helm-upgrade.
+Two aspects are not full covered by running helm-upgrade: CRDs and values.
+While helm-install can install CRDs if they are contained in a chart's `crds` folder, upgrade won't change any installed CRDs.
+Furthermore, new values introduced with a new version of a chart will not be installed into the cluster if the `--reuse-values` flag is set.
+Nevertheless, we have to rely on the values already present in the cluster because some of the values are set by the bootstrapper during installation.
+Because upgrades should be a CLI-only operation and we want to avoid the behaviour of `--reuse-values`, we fetch the cluster values and merge them with any new values.
+
+Here is how we manage CRD upgrades for each chart.
+
+## Cilium
+- CRDs are updated by cilium-operator.
+
+## cert-manager
+- installCRDs flag is set during upgrade. This flag is managed by cert-manager. cert-manager is in charge of correctly upgrading the CRDs.
+- WARNING: upgrading cert-manager might break other installations of cert-manager in the cluster, if those other installation are not on the same version as the Constellation-manager installation. This is due to the cluster-wide CRDs.
+
+## Operators
+- Manually update CRDs before upgrading the chart. Update by running applying the CRDs found in the `operators/crds/` folder.
+
+## Constellation-services
+- There currently are no CRDs in this chart.

cli/internal/helm/client.go

@@ -7,28 +7,44 @@ SPDX-License-Identifier: AGPL-3.0-only
 package helm
 
 import (
+	"context"
 	"fmt"
+	"strings"
+	"time"
 
+	"github.com/edgelesssys/constellation/v2/internal/config"
+	"github.com/edgelesssys/constellation/v2/internal/constants"
+	"github.com/edgelesssys/constellation/v2/internal/deploy/helm"
+	"github.com/pkg/errors"
 	"helm.sh/helm/v3/pkg/action"
+	"helm.sh/helm/v3/pkg/chart"
 	"helm.sh/helm/v3/pkg/cli"
+	v1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
+	apiextensionsclient "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/runtime/serializer"
+	"k8s.io/client-go/kubernetes/scheme"
 )
 
 // Client handles interaction with helm.
 type Client struct {
-	config *action.Configuration
+	config    *action.Configuration
+	crdClient *apiextensionsclient.Clientset
+	log       debugLog
 }
 
 // NewClient returns a new initializes client for the namespace Client.
-func NewClient(kubeConfigPath, helmNamespace string) (*Client, error) {
+func NewClient(kubeConfigPath, helmNamespace string, client *apiextensionsclient.Clientset, log debugLog) (*Client, error) {
 	settings := cli.New()
 	settings.KubeConfig = kubeConfigPath // constants.AdminConfFilename
 
 	actionConfig := &action.Configuration{}
-	if err := actionConfig.Init(settings.RESTClientGetter(), helmNamespace, "secret", nil); err != nil {
+	if err := actionConfig.Init(settings.RESTClientGetter(), helmNamespace, "secret", log.Debugf); err != nil {
 		return nil, fmt.Errorf("initializing config: %w", err)
 	}
 
-	return &Client{config: actionConfig}, nil
+	return &Client{config: actionConfig, crdClient: client, log: log}, nil
 }
 
 // CurrentVersion returns the version of the currently installed helm release.
@@ -53,3 +69,153 @@ func (c *Client) CurrentVersion(release string) (string, error) {
 
 	return rel[0].Chart.Metadata.Version, nil
 }
+
+// Upgrade runs a helm-upgrade on all deployments that are managed via Helm.
+// If the CLI receives an interrupt signal it will cancel the context.
+// Canceling the context will prompt helm to abort and roll back the ongoing upgrade.
+func (c *Client) Upgrade(ctx context.Context, config *config.Config, timeout time.Duration) error {
+	action := action.NewUpgrade(c.config)
+	action.Atomic = true
+	action.Namespace = constants.HelmNamespace
+	action.ReuseValues = false
+	action.Timeout = timeout
+
+	ciliumChart, err := loadChartsDir(helmFS, ciliumPath)
+	if err != nil {
+		return fmt.Errorf("loading cilium: %w", err)
+	}
+	certManagerChart, err := loadChartsDir(helmFS, certManagerPath)
+	if err != nil {
+		return fmt.Errorf("loading cert-manager: %w", err)
+	}
+	conOperatorChart, err := loadChartsDir(helmFS, conOperatorsPath)
+	if err != nil {
+		return fmt.Errorf("loading operators: %w", err)
+	}
+	conServicesChart, err := loadChartsDir(helmFS, conServicesPath)
+	if err != nil {
+		return fmt.Errorf("loading constellation-services chart: %w", err)
+	}
+
+	values, err := c.prepareValues(ciliumChart, ciliumReleaseName)
+	if err != nil {
+		return err
+	}
+	if _, err := action.RunWithContext(ctx, ciliumReleaseName, ciliumChart, values); err != nil {
+		return fmt.Errorf("upgrading cilium: %w", err)
+	}
+
+	values, err = c.prepareValues(certManagerChart, certManagerReleaseName)
+	if err != nil {
+		return err
+	}
+	if _, err := action.RunWithContext(ctx, certManagerReleaseName, certManagerChart, values); err != nil {
+		return fmt.Errorf("upgrading cert-manager: %w", err)
+	}
+
+	err = c.updateOperatorCRDs(ctx, conOperatorChart)
+	if err != nil {
+		return fmt.Errorf("updating operator CRDs: %w", err)
+	}
+	values, err = c.prepareValues(conOperatorChart, conOperatorsReleaseName)
+	if err != nil {
+		return err
+	}
+	if _, err := action.RunWithContext(ctx, conOperatorsReleaseName, conOperatorChart, values); err != nil {
+		return fmt.Errorf("upgrading services: %w", err)
+	}
+
+	values, err = c.prepareValues(conServicesChart, conServicesReleaseName)
+	if err != nil {
+		return err
+	}
+	if _, err := action.RunWithContext(ctx, conServicesReleaseName, conServicesChart, values); err != nil {
+		return fmt.Errorf("upgrading operators: %w", err)
+	}
+
+	return nil
+}
+
+// prepareCertManagerValues returns a values map as required for helm-upgrade.
+// It imitates the behaviour of helm's reuse-values flag by fetching the current values from the cluster
+// and merging the fetched values with the locally found values.
+// This is done to ensure that new values (from upgrades of the local files) end up in the cluster.
+// reuse-values does not ensure this.
+func (c *Client) prepareValues(chart *chart.Chart, releaseName string) (map[string]any, error) {
+	// Ensure installCRDs is set for cert-manager chart.
+	if releaseName == certManagerReleaseName {
+		chart.Values["installCRDs"] = true
+	}
+	values, err := c.GetValues(releaseName)
+	if err != nil {
+		return nil, fmt.Errorf("getting values: %w", err)
+	}
+	return helm.MergeMaps(chart.Values, values), nil
+}
+
+// GetValues queries the cluster for the values of the given release.
+func (c *Client) GetValues(release string) (map[string]any, error) {
+	client := action.NewGetValues(c.config)
+	// Version corresponds to the releases revision. Specifying a Version <= 0 yields the latest release.
+	client.Version = 0
+	values, err := client.Run(release)
+	if err != nil {
+		return nil, fmt.Errorf("getting values for %s: %w", release, err)
+	}
+	return values, nil
+}
+
+// ApplyCRD updates the given CRD by parsing it, querying it's version from the cluster and finally updating it.
+func (c *Client) ApplyCRD(ctx context.Context, rawCRD []byte) error {
+	crd, err := parseCRD(rawCRD)
+	if err != nil {
+		return fmt.Errorf("parsing crds: %w", err)
+	}
+
+	clusterCRD, err := c.crdClient.ApiextensionsV1().CustomResourceDefinitions().Get(ctx, crd.Name, metav1.GetOptions{})
+	if err != nil {
+		return fmt.Errorf("getting crd: %w", err)
+	}
+	crd.ResourceVersion = clusterCRD.ResourceVersion
+	_, err = c.crdClient.ApiextensionsV1().CustomResourceDefinitions().Update(ctx, crd, metav1.UpdateOptions{})
+	return err
+}
+
+// parseCRD takes a byte slice of data and tries to create a CustomResourceDefinition object from it.
+func parseCRD(crdString []byte) (*v1.CustomResourceDefinition, error) {
+	sch := runtime.NewScheme()
+	_ = scheme.AddToScheme(sch)
+	_ = v1.AddToScheme(sch)
+	obj, groupVersionKind, err := serializer.NewCodecFactory(sch).UniversalDeserializer().Decode(crdString, nil, nil)
+	if err != nil {
+		return nil, fmt.Errorf("decoding crd: %w", err)
+	}
+	if groupVersionKind.Kind == "CustomResourceDefinition" {
+		return obj.(*v1.CustomResourceDefinition), nil
+	}
+
+	return nil, errors.New("parsed []byte, but did not find a CRD")
+}
+
+// updateOperatorCRDs walks through the dependencies of the given chart and applies
+// the files in the dependencie's 'crds' folder.
+// This function is NOT recursive!
+func (c *Client) updateOperatorCRDs(ctx context.Context, chart *chart.Chart) error {
+	for _, dep := range chart.Dependencies() {
+		for _, crdFile := range dep.Files {
+			if strings.HasPrefix(crdFile.Name, "crds/") {
+				c.log.Debugf("updating crd: %s", crdFile.Name)
+				err := c.ApplyCRD(ctx, crdFile.Data)
+				if err != nil {
+					return err
+				}
+			}
+		}
+	}
+	return nil
+}
+
+type debugLog interface {
+	Debugf(format string, args ...any)
+	Sync()
+}

cli/internal/helm/client_test.go

@@ -0,0 +1,46 @@
+/*
+Copyright (c) Edgeless Systems GmbH
+
+SPDX-License-Identifier: AGPL-3.0-only
+*/
+
+package helm
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestParseCRDs(t *testing.T) {
+	testCases := map[string]struct {
+		data    string
+		wantErr bool
+	}{
+		"success": {
+			data:    "apiVersion: apiextensions.k8s.io/v1\nkind: CustomResourceDefinition\nmetadata:\n  name: nodeimages.update.edgeless.systems\nspec:\n  group: update.edgeless.systems\n  names:\n    kind: NodeImage\n",
+			wantErr: false,
+		},
+		"wrong kind": {
+			data:    "apiVersion: v1\nkind: Secret\ntype: Opaque\nmetadata:\n  name: supersecret\n  namespace: testNamespace\ndata:\n  data: YWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWE=\n",
+			wantErr: true,
+		},
+		"decoding error": {
+			data:    "asdf",
+			wantErr: true,
+		},
+	}
+
+	for name, tc := range testCases {
+		t.Run(name, func(t *testing.T) {
+			assert := assert.New(t)
+
+			_, err := parseCRD([]byte(tc.data))
+			if tc.wantErr {
+				assert.Error(err)
+				return
+			}
+			assert.NoError(err)
+		})
+	}
+}

cli/internal/helm/loader.go

@@ -37,6 +37,18 @@ import (
 //go:embed all:charts/*
 var helmFS embed.FS
 
+const (
+	ciliumReleaseName       = "cilium"
+	conServicesReleaseName  = "constellation-services"
+	conOperatorsReleaseName = "constellation-operators"
+	certManagerReleaseName  = "cert-manager"
+
+	conServicesPath  = "charts/edgeless/constellation-services"
+	conOperatorsPath = "charts/edgeless/operators"
+	certManagerPath  = "charts/cert-manager"
+	ciliumPath       = "charts/cilium"
+)
+
 // ChartLoader loads embedded helm charts.
 type ChartLoader struct {
 	joinServiceImage             string
@@ -109,7 +121,11 @@ func (i *ChartLoader) Load(config *config.Config, conformanceMode bool, masterSe
 }
 
 func (i *ChartLoader) loadCilium(csp cloudprovider.Provider, conformanceMode bool) (helm.Release, error) {
-	chart, values, err := i.loadCiliumHelper(csp, conformanceMode)
+	chart, err := loadChartsDir(helmFS, ciliumPath)
+	if err != nil {
+		return helm.Release{}, fmt.Errorf("loading cilium chart: %w", err)
+	}
+	values, err := i.loadCiliumValues(csp, conformanceMode)
 	if err != nil {
 		return helm.Release{}, err
 	}
@@ -119,17 +135,12 @@ func (i *ChartLoader) loadCilium(csp cloudprovider.Provider, conformanceMode boo
 		return helm.Release{}, fmt.Errorf("packaging cilium chart: %w", err)
 	}
 
-	return helm.Release{Chart: chartRaw, Values: values, ReleaseName: "cilium", Wait: false}, nil
+	return helm.Release{Chart: chartRaw, Values: values, ReleaseName: ciliumReleaseName, Wait: false}, nil
 }
 
 // loadCiliumHelper is used to separate the marshalling step from the loading step.
 // This reduces the time unit tests take to execute.
-func (i *ChartLoader) loadCiliumHelper(csp cloudprovider.Provider, conformanceMode bool) (*chart.Chart, map[string]any, error) {
+func (i *ChartLoader) loadCiliumValues(csp cloudprovider.Provider, conformanceMode bool) (map[string]any, error) {
-	chart, err := loadChartsDir(helmFS, "charts/cilium")
-	if err != nil {
-		return nil, nil, fmt.Errorf("loading cilium chart: %w", err)
-	}
-
 	var values map[string]any
 	switch csp {
 	case cloudprovider.AWS:
@@ -141,7 +152,7 @@ func (i *ChartLoader) loadCiliumHelper(csp cloudprovider.Provider, conformanceMo
 	case cloudprovider.QEMU:
 		values = qemuVals
 	default:
-		return nil, nil, fmt.Errorf("unknown csp: %s", csp)
+		return nil, fmt.Errorf("unknown csp: %s", csp)
 	}
 	if conformanceMode {
 		values["kubeProxyReplacementHealthzBindAddr"] = ""
@@ -152,32 +163,28 @@ func (i *ChartLoader) loadCiliumHelper(csp cloudprovider.Provider, conformanceMo
 		}
 
 	}
-	return chart, values, nil
+	return values, nil
 }
 
 func (i *ChartLoader) loadCertManager() (helm.Release, error) {
-	chart, values, err := i.loadCertManagerHelper()
+	chart, err := loadChartsDir(helmFS, certManagerPath)
 	if err != nil {
-		return helm.Release{}, err
+		return helm.Release{}, fmt.Errorf("loading cert-manager chart: %w", err)
 	}
+	values := i.loadCertManagerValues()
 
 	chartRaw, err := i.marshalChart(chart)
 	if err != nil {
 		return helm.Release{}, fmt.Errorf("packaging cert-manager chart: %w", err)
 	}
 
-	return helm.Release{Chart: chartRaw, Values: values, ReleaseName: "cert-manager", Wait: false}, nil
+	return helm.Release{Chart: chartRaw, Values: values, ReleaseName: certManagerReleaseName, Wait: false}, nil
 }
 
 // loadCertManagerHelper is used to separate the marshalling step from the loading step.
 // This reduces the time unit tests take to execute.
-func (i *ChartLoader) loadCertManagerHelper() (*chart.Chart, map[string]any, error) {
+func (i *ChartLoader) loadCertManagerValues() map[string]any {
-	chart, err := loadChartsDir(helmFS, "charts/cert-manager")
+	return map[string]any{
-	if err != nil {
-		return nil, nil, fmt.Errorf("loading cert-manager chart: %w", err)
-	}
-
-	values := map[string]any{
 		"installCRDs": true,
 		"prometheus": map[string]any{
 			"enabled": false,
@@ -241,11 +248,14 @@ func (i *ChartLoader) loadCertManagerHelper() (*chart.Chart, map[string]any, err
 			},
 		},
 	}
-	return chart, values, nil
 }
 
 func (i *ChartLoader) loadOperators(csp cloudprovider.Provider) (helm.Release, error) {
-	chart, values, err := i.loadOperatorsHelper(csp)
+	chart, err := loadChartsDir(helmFS, conOperatorsPath)
+	if err != nil {
+		return helm.Release{}, fmt.Errorf("loading operators chart: %w", err)
+	}
+	values, err := i.loadOperatorsValues(csp)
 	if err != nil {
 		return helm.Release{}, err
 	}
@@ -255,17 +265,12 @@ func (i *ChartLoader) loadOperators(csp cloudprovider.Provider) (helm.Release, e
 		return helm.Release{}, fmt.Errorf("packaging operators chart: %w", err)
 	}
 
-	return helm.Release{Chart: chartRaw, Values: values, ReleaseName: "con-operators", Wait: false}, nil
+	return helm.Release{Chart: chartRaw, Values: values, ReleaseName: conOperatorsReleaseName, Wait: false}, nil
 }
 
 // loadOperatorsHelper is used to separate the marshalling step from the loading step.
 // This reduces the time unit tests take to execute.
-func (i *ChartLoader) loadOperatorsHelper(csp cloudprovider.Provider) (*chart.Chart, map[string]any, error) {
+func (i *ChartLoader) loadOperatorsValues(csp cloudprovider.Provider) (map[string]any, error) {
-	chart, err := loadChartsDir(helmFS, "charts/edgeless/operators")
-	if err != nil {
-		return nil, nil, fmt.Errorf("loading operators chart: %w", err)
-	}
-
 	values := map[string]any{
 		"constellation-operator": map[string]any{
 			"controllerManager": map[string]any{
@@ -286,7 +291,7 @@ func (i *ChartLoader) loadOperatorsHelper(csp cloudprovider.Provider) (*chart.Ch
 	case cloudprovider.Azure:
 		conOpVals, ok := values["constellation-operator"].(map[string]any)
 		if !ok {
-			return nil, nil, errors.New("invalid constellation-operator values")
+			return nil, errors.New("invalid constellation-operator values")
 		}
 		conOpVals["csp"] = "Azure"
 
@@ -296,7 +301,7 @@ func (i *ChartLoader) loadOperatorsHelper(csp cloudprovider.Provider) (*chart.Ch
 	case cloudprovider.GCP:
 		conOpVals, ok := values["constellation-operator"].(map[string]any)
 		if !ok {
-			return nil, nil, errors.New("invalid constellation-operator values")
+			return nil, errors.New("invalid constellation-operator values")
 		}
 		conOpVals["csp"] = "GCP"
 
@@ -306,7 +311,7 @@ func (i *ChartLoader) loadOperatorsHelper(csp cloudprovider.Provider) (*chart.Ch
 	case cloudprovider.QEMU:
 		conOpVals, ok := values["constellation-operator"].(map[string]any)
 		if !ok {
-			return nil, nil, errors.New("invalid constellation-operator values")
+			return nil, errors.New("invalid constellation-operator values")
 		}
 		conOpVals["csp"] = "QEMU"
 
@@ -316,7 +321,7 @@ func (i *ChartLoader) loadOperatorsHelper(csp cloudprovider.Provider) (*chart.Ch
 	case cloudprovider.AWS:
 		conOpVals, ok := values["constellation-operator"].(map[string]any)
 		if !ok {
-			return nil, nil, errors.New("invalid constellation-operator values")
+			return nil, errors.New("invalid constellation-operator values")
 		}
 		conOpVals["csp"] = "AWS"
 
@@ -325,13 +330,17 @@ func (i *ChartLoader) loadOperatorsHelper(csp cloudprovider.Provider) (*chart.Ch
 		}
 	}
 
-	return chart, values, nil
+	return values, nil
 }
 
 // loadConstellationServices loads the constellation-services chart from the embed.FS,
 // marshals it into a helm-package .tgz and sets the values that can be set in the CLI.
 func (i *ChartLoader) loadConstellationServices(config *config.Config, masterSecret, salt []byte) (helm.Release, error) {
-	chart, values, err := i.loadConstellationServicesHelper(config, masterSecret, salt)
+	chart, err := loadChartsDir(helmFS, conServicesPath)
+	if err != nil {
+		return helm.Release{}, fmt.Errorf("loading constellation-services chart: %w", err)
+	}
+	values, err := i.loadConstellationServicesValues(config, masterSecret, salt)
 	if err != nil {
 		return helm.Release{}, err
 	}
@@ -341,17 +350,12 @@ func (i *ChartLoader) loadConstellationServices(config *config.Config, masterSec
 		return helm.Release{}, fmt.Errorf("packaging constellation-services chart: %w", err)
 	}
 
-	return helm.Release{Chart: chartRaw, Values: values, ReleaseName: "constellation-services", Wait: false}, nil
+	return helm.Release{Chart: chartRaw, Values: values, ReleaseName: conServicesReleaseName, Wait: false}, nil
 }
 
 // loadConstellationServicesHelper is used to separate the marshalling step from the loading step.
 // This reduces the time unit tests take to execute.
-func (i *ChartLoader) loadConstellationServicesHelper(config *config.Config, masterSecret, salt []byte) (*chart.Chart, map[string]any, error) {
+func (i *ChartLoader) loadConstellationServicesValues(config *config.Config, masterSecret, salt []byte) (map[string]any, error) {
-	chart, err := loadChartsDir(helmFS, "charts/edgeless/constellation-services")
-	if err != nil {
-		return nil, nil, fmt.Errorf("loading constellation-services chart: %w", err)
-	}
-
 	csp := config.GetProvider()
 	values := map[string]any{
 		"global": map[string]any{
@@ -397,13 +401,13 @@ func (i *ChartLoader) loadConstellationServicesHelper(config *config.Config, mas
 	case cloudprovider.Azure:
 		joinServiceVals, ok := values["join-service"].(map[string]any)
 		if !ok {
-			return nil, nil, errors.New("invalid join-service values")
+			return nil, errors.New("invalid join-service values")
 		}
 		joinServiceVals["enforceIdKeyDigest"] = config.EnforcesIDKeyDigest()
 
 		ccmVals, ok := values["ccm"].(map[string]any)
 		if !ok {
-			return nil, nil, errors.New("invalid ccm values")
+			return nil, errors.New("invalid ccm values")
 		}
 		ccmVals["Azure"] = map[string]any{
 			"image": i.ccmImage,
@@ -431,7 +435,7 @@ func (i *ChartLoader) loadConstellationServicesHelper(config *config.Config, mas
 	case cloudprovider.GCP:
 		ccmVals, ok := values["ccm"].(map[string]any)
 		if !ok {
-			return nil, nil, errors.New("invalid ccm values")
+			return nil, errors.New("invalid ccm values")
 		}
 		ccmVals["GCP"] = map[string]any{
 			"image": i.ccmImage,
@@ -460,7 +464,7 @@ func (i *ChartLoader) loadConstellationServicesHelper(config *config.Config, mas
 	case cloudprovider.AWS:
 		ccmVals, ok := values["ccm"].(map[string]any)
 		if !ok {
-			return nil, nil, errors.New("invalid ccm values")
+			return nil, errors.New("invalid ccm values")
 		}
 		ccmVals["AWS"] = map[string]any{
 			"image": i.ccmImage,
@@ -470,7 +474,7 @@ func (i *ChartLoader) loadConstellationServicesHelper(config *config.Config, mas
 			"AWS": true,
 		}
 	}
-	return chart, values, nil
+	return values, nil
 }
 
 // marshalChart takes a Chart object, packages it to a temporary file and returns the content of that file.

cli/internal/helm/loader_test.go

@@ -98,7 +98,9 @@ func TestConstellationServices(t *testing.T) {
 				konnectivityImage:        "konnectivityImage",
 				gcpGuestAgentImage:       "gcpGuestAgentImage",
 			}
-			chart, values, err := chartLoader.loadConstellationServicesHelper(tc.config, []byte("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"), []byte("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"))
+			chart, err := loadChartsDir(helmFS, conServicesPath)
+			require.NoError(err)
+			values, err := chartLoader.loadConstellationServicesValues(tc.config, []byte("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"), []byte("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"))
 			require.NoError(err)
 
 			options := chartutil.ReleaseOptions{
@@ -164,7 +166,9 @@ func TestOperators(t *testing.T) {
 				constellationOperatorImage:   "constellationOperatorImage",
 				nodeMaintenanceOperatorImage: "nodeMaintenanceOperatorImage",
 			}
-			chart, vals, err := chartLoader.loadOperatorsHelper(tc.csp)
+			chart, err := loadChartsDir(helmFS, conOperatorsPath)
+			require.NoError(err)
+			vals, err := chartLoader.loadOperatorsValues(tc.csp)
 			require.NoError(err)
 
 			options := chartutil.ReleaseOptions{

internal/deploy/helm/helm.go

@@ -21,3 +21,25 @@ type Releases struct {
 	Operators             Release
 	ConstellationServices Release
 }
+
+// MergeMaps returns a new map that is the merger of it's inputs.
+// Key colissions are resolved by taking the value of the second argument (map b).
+// Taken from: https://github.com/helm/helm/blob/dbc6d8e20fe1d58d50e6ed30f09a04a77e4c68db/pkg/cli/values/options.go#L91-L108.
+func MergeMaps(a, b map[string]any) map[string]any {
+	out := make(map[string]any, len(a))
+	for k, v := range a {
+		out[k] = v
+	}
+	for k, v := range b {
+		if v, ok := v.(map[string]any); ok {
+			if bv, ok := out[k]; ok {
+				if bv, ok := bv.(map[string]any); ok {
+					out[k] = MergeMaps(bv, v)
+					continue
+				}
+			}
+		}
+		out[k] = v
+	}
+	return out
+}

internal/deploy/helm/helm_test.go

@@ -0,0 +1,109 @@
+/*
+Copyright (c) Edgeless Systems GmbH
+
+SPDX-License-Identifier: AGPL-3.0-only
+*/
+
+package helm
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestMergeMaps(t *testing.T) {
+	testCases := map[string]struct {
+		vals      map[string]any
+		extraVals map[string]any
+		expected  map[string]any
+	}{
+		"equal": {
+			vals: map[string]any{
+				"join-service": map[string]any{
+					"key1": "foo",
+					"key2": "bar",
+				},
+			},
+			extraVals: map[string]any{
+				"join-service": map[string]any{
+					"extraKey1": "extraFoo",
+					"extraKey2": "extraBar",
+				},
+			},
+			expected: map[string]any{
+				"join-service": map[string]any{
+					"key1":      "foo",
+					"key2":      "bar",
+					"extraKey1": "extraFoo",
+					"extraKey2": "extraBar",
+				},
+			},
+		},
+		"missing join-service extraVals": {
+			vals: map[string]any{
+				"join-service": map[string]any{
+					"key1": "foo",
+					"key2": "bar",
+				},
+			},
+			extraVals: map[string]any{
+				"extraKey1": "extraFoo",
+				"extraKey2": "extraBar",
+			},
+			expected: map[string]any{
+				"join-service": map[string]any{
+					"key1": "foo",
+					"key2": "bar",
+				},
+				"extraKey1": "extraFoo",
+				"extraKey2": "extraBar",
+			},
+		},
+		"missing join-service vals": {
+			vals: map[string]any{
+				"key1": "foo",
+				"key2": "bar",
+			},
+			extraVals: map[string]any{
+				"join-service": map[string]any{
+					"extraKey1": "extraFoo",
+					"extraKey2": "extraBar",
+				},
+			},
+			expected: map[string]any{
+				"key1": "foo",
+				"key2": "bar",
+				"join-service": map[string]any{
+					"extraKey1": "extraFoo",
+					"extraKey2": "extraBar",
+				},
+			},
+		},
+		"key collision": {
+			vals: map[string]any{
+				"join-service": map[string]any{
+					"key1": "foo",
+				},
+			},
+			extraVals: map[string]any{
+				"join-service": map[string]any{
+					"key1": "bar",
+				},
+			},
+			expected: map[string]any{
+				"join-service": map[string]any{
+					"key1": "bar",
+				},
+			},
+		},
+	}
+
+	for name, tc := range testCases {
+		t.Run(name, func(t *testing.T) {
+			assert := assert.New(t)
+			newVals := MergeMaps(tc.vals, tc.extraVals)
+			assert.Equal(tc.expected, newVals)
+		})
+	}
+}