Microservice upgrades (#729)

Run with: constellation upgrade execute --helm.
This will only upgrade the helm charts. No config is needed.

Upgrades are implemented via helm's upgrade action, i.e. they
automatically roll back if something goes wrong. Releases could 
still be managed via helm, even after an upgrade with constellation
has been done.

Currently not user facing as CRD/CR backups are still in progress.
These backups should be automatically created and saved to the 
user's disk as updates may delete CRs. This happens implicitly 
through CRD upgrades, which are part of microservice upgrades.

cli/internal/helm/README.md

@@ -0,0 +1,23 @@
+# Chart upgrades
+
+All services that are installed via helm-install are upgraded via helm-upgrade.
+Two aspects are not full covered by running helm-upgrade: CRDs and values.
+While helm-install can install CRDs if they are contained in a chart's `crds` folder, upgrade won't change any installed CRDs.
+Furthermore, new values introduced with a new version of a chart will not be installed into the cluster if the `--reuse-values` flag is set.
+Nevertheless, we have to rely on the values already present in the cluster because some of the values are set by the bootstrapper during installation.
+Because upgrades should be a CLI-only operation and we want to avoid the behaviour of `--reuse-values`, we fetch the cluster values and merge them with any new values.
+
+Here is how we manage CRD upgrades for each chart.
+
+## Cilium
+- CRDs are updated by cilium-operator.
+
+## cert-manager
+- installCRDs flag is set during upgrade. This flag is managed by cert-manager. cert-manager is in charge of correctly upgrading the CRDs.
+- WARNING: upgrading cert-manager might break other installations of cert-manager in the cluster, if those other installation are not on the same version as the Constellation-manager installation. This is due to the cluster-wide CRDs.
+
+## Operators
+- Manually update CRDs before upgrading the chart. Update by running applying the CRDs found in the `operators/crds/` folder.
+
+## Constellation-services
+- There currently are no CRDs in this chart.

cli/internal/helm/client.go

@@ -7,28 +7,44 @@ SPDX-License-Identifier: AGPL-3.0-only
 package helm
 
 import (
+	"context"
 	"fmt"
+	"strings"
+	"time"
 
+	"github.com/edgelesssys/constellation/v2/internal/config"
+	"github.com/edgelesssys/constellation/v2/internal/constants"
+	"github.com/edgelesssys/constellation/v2/internal/deploy/helm"
+	"github.com/pkg/errors"
 	"helm.sh/helm/v3/pkg/action"
+	"helm.sh/helm/v3/pkg/chart"
 	"helm.sh/helm/v3/pkg/cli"
+	v1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
+	apiextensionsclient "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/runtime/serializer"
+	"k8s.io/client-go/kubernetes/scheme"
 )
 
 // Client handles interaction with helm.
 type Client struct {
-	config *action.Configuration
+	config    *action.Configuration
+	crdClient *apiextensionsclient.Clientset
+	log       debugLog
 }
 
 // NewClient returns a new initializes client for the namespace Client.
-func NewClient(kubeConfigPath, helmNamespace string) (*Client, error) {
+func NewClient(kubeConfigPath, helmNamespace string, client *apiextensionsclient.Clientset, log debugLog) (*Client, error) {
 	settings := cli.New()
 	settings.KubeConfig = kubeConfigPath // constants.AdminConfFilename
 
 	actionConfig := &action.Configuration{}
-	if err := actionConfig.Init(settings.RESTClientGetter(), helmNamespace, "secret", nil); err != nil {
+	if err := actionConfig.Init(settings.RESTClientGetter(), helmNamespace, "secret", log.Debugf); err != nil {
 		return nil, fmt.Errorf("initializing config: %w", err)
 	}
 
-	return &Client{config: actionConfig}, nil
+	return &Client{config: actionConfig, crdClient: client, log: log}, nil
 }
 
 // CurrentVersion returns the version of the currently installed helm release.
@@ -53,3 +69,153 @@ func (c *Client) CurrentVersion(release string) (string, error) {
 
 	return rel[0].Chart.Metadata.Version, nil
 }
+
+// Upgrade runs a helm-upgrade on all deployments that are managed via Helm.
+// If the CLI receives an interrupt signal it will cancel the context.
+// Canceling the context will prompt helm to abort and roll back the ongoing upgrade.
+func (c *Client) Upgrade(ctx context.Context, config *config.Config, timeout time.Duration) error {
+	action := action.NewUpgrade(c.config)
+	action.Atomic = true
+	action.Namespace = constants.HelmNamespace
+	action.ReuseValues = false
+	action.Timeout = timeout
+
+	ciliumChart, err := loadChartsDir(helmFS, ciliumPath)
+	if err != nil {
+		return fmt.Errorf("loading cilium: %w", err)
+	}
+	certManagerChart, err := loadChartsDir(helmFS, certManagerPath)
+	if err != nil {
+		return fmt.Errorf("loading cert-manager: %w", err)
+	}
+	conOperatorChart, err := loadChartsDir(helmFS, conOperatorsPath)
+	if err != nil {
+		return fmt.Errorf("loading operators: %w", err)
+	}
+	conServicesChart, err := loadChartsDir(helmFS, conServicesPath)
+	if err != nil {
+		return fmt.Errorf("loading constellation-services chart: %w", err)
+	}
+
+	values, err := c.prepareValues(ciliumChart, ciliumReleaseName)
+	if err != nil {
+		return err
+	}
+	if _, err := action.RunWithContext(ctx, ciliumReleaseName, ciliumChart, values); err != nil {
+		return fmt.Errorf("upgrading cilium: %w", err)
+	}
+
+	values, err = c.prepareValues(certManagerChart, certManagerReleaseName)
+	if err != nil {
+		return err
+	}
+	if _, err := action.RunWithContext(ctx, certManagerReleaseName, certManagerChart, values); err != nil {
+		return fmt.Errorf("upgrading cert-manager: %w", err)
+	}
+
+	err = c.updateOperatorCRDs(ctx, conOperatorChart)
+	if err != nil {
+		return fmt.Errorf("updating operator CRDs: %w", err)
+	}
+	values, err = c.prepareValues(conOperatorChart, conOperatorsReleaseName)
+	if err != nil {
+		return err
+	}
+	if _, err := action.RunWithContext(ctx, conOperatorsReleaseName, conOperatorChart, values); err != nil {
+		return fmt.Errorf("upgrading services: %w", err)
+	}
+
+	values, err = c.prepareValues(conServicesChart, conServicesReleaseName)
+	if err != nil {
+		return err
+	}
+	if _, err := action.RunWithContext(ctx, conServicesReleaseName, conServicesChart, values); err != nil {
+		return fmt.Errorf("upgrading operators: %w", err)
+	}
+
+	return nil
+}
+
+// prepareCertManagerValues returns a values map as required for helm-upgrade.
+// It imitates the behaviour of helm's reuse-values flag by fetching the current values from the cluster
+// and merging the fetched values with the locally found values.
+// This is done to ensure that new values (from upgrades of the local files) end up in the cluster.
+// reuse-values does not ensure this.
+func (c *Client) prepareValues(chart *chart.Chart, releaseName string) (map[string]any, error) {
+	// Ensure installCRDs is set for cert-manager chart.
+	if releaseName == certManagerReleaseName {
+		chart.Values["installCRDs"] = true
+	}
+	values, err := c.GetValues(releaseName)
+	if err != nil {
+		return nil, fmt.Errorf("getting values: %w", err)
+	}
+	return helm.MergeMaps(chart.Values, values), nil
+}
+
+// GetValues queries the cluster for the values of the given release.
+func (c *Client) GetValues(release string) (map[string]any, error) {
+	client := action.NewGetValues(c.config)
+	// Version corresponds to the releases revision. Specifying a Version <= 0 yields the latest release.
+	client.Version = 0
+	values, err := client.Run(release)
+	if err != nil {
+		return nil, fmt.Errorf("getting values for %s: %w", release, err)
+	}
+	return values, nil
+}
+
+// ApplyCRD updates the given CRD by parsing it, querying it's version from the cluster and finally updating it.
+func (c *Client) ApplyCRD(ctx context.Context, rawCRD []byte) error {
+	crd, err := parseCRD(rawCRD)
+	if err != nil {
+		return fmt.Errorf("parsing crds: %w", err)
+	}
+
+	clusterCRD, err := c.crdClient.ApiextensionsV1().CustomResourceDefinitions().Get(ctx, crd.Name, metav1.GetOptions{})
+	if err != nil {
+		return fmt.Errorf("getting crd: %w", err)
+	}
+	crd.ResourceVersion = clusterCRD.ResourceVersion
+	_, err = c.crdClient.ApiextensionsV1().CustomResourceDefinitions().Update(ctx, crd, metav1.UpdateOptions{})
+	return err
+}
+
+// parseCRD takes a byte slice of data and tries to create a CustomResourceDefinition object from it.
+func parseCRD(crdString []byte) (*v1.CustomResourceDefinition, error) {
+	sch := runtime.NewScheme()
+	_ = scheme.AddToScheme(sch)
+	_ = v1.AddToScheme(sch)
+	obj, groupVersionKind, err := serializer.NewCodecFactory(sch).UniversalDeserializer().Decode(crdString, nil, nil)
+	if err != nil {
+		return nil, fmt.Errorf("decoding crd: %w", err)
+	}
+	if groupVersionKind.Kind == "CustomResourceDefinition" {
+		return obj.(*v1.CustomResourceDefinition), nil
+	}
+
+	return nil, errors.New("parsed []byte, but did not find a CRD")
+}
+
+// updateOperatorCRDs walks through the dependencies of the given chart and applies
+// the files in the dependencie's 'crds' folder.
+// This function is NOT recursive!
+func (c *Client) updateOperatorCRDs(ctx context.Context, chart *chart.Chart) error {
+	for _, dep := range chart.Dependencies() {
+		for _, crdFile := range dep.Files {
+			if strings.HasPrefix(crdFile.Name, "crds/") {
+				c.log.Debugf("updating crd: %s", crdFile.Name)
+				err := c.ApplyCRD(ctx, crdFile.Data)
+				if err != nil {
+					return err
+				}
+			}
+		}
+	}
+	return nil
+}
+
+type debugLog interface {
+	Debugf(format string, args ...any)
+	Sync()
+}

cli/internal/helm/client_test.go

@@ -0,0 +1,46 @@
+/*
+Copyright (c) Edgeless Systems GmbH
+
+SPDX-License-Identifier: AGPL-3.0-only
+*/
+
+package helm
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestParseCRDs(t *testing.T) {
+	testCases := map[string]struct {
+		data    string
+		wantErr bool
+	}{
+		"success": {
+			data:    "apiVersion: apiextensions.k8s.io/v1\nkind: CustomResourceDefinition\nmetadata:\n  name: nodeimages.update.edgeless.systems\nspec:\n  group: update.edgeless.systems\n  names:\n    kind: NodeImage\n",
+			wantErr: false,
+		},
+		"wrong kind": {
+			data:    "apiVersion: v1\nkind: Secret\ntype: Opaque\nmetadata:\n  name: supersecret\n  namespace: testNamespace\ndata:\n  data: YWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWE=\n",
+			wantErr: true,
+		},
+		"decoding error": {
+			data:    "asdf",
+			wantErr: true,
+		},
+	}
+
+	for name, tc := range testCases {
+		t.Run(name, func(t *testing.T) {
+			assert := assert.New(t)
+
+			_, err := parseCRD([]byte(tc.data))
+			if tc.wantErr {
+				assert.Error(err)
+				return
+			}
+			assert.NoError(err)
+		})
+	}
+}

cli/internal/helm/loader.go

@@ -37,6 +37,18 @@ import (
 //go:embed all:charts/*
 var helmFS embed.FS
 
+const (
+	ciliumReleaseName       = "cilium"
+	conServicesReleaseName  = "constellation-services"
+	conOperatorsReleaseName = "constellation-operators"
+	certManagerReleaseName  = "cert-manager"
+
+	conServicesPath  = "charts/edgeless/constellation-services"
+	conOperatorsPath = "charts/edgeless/operators"
+	certManagerPath  = "charts/cert-manager"
+	ciliumPath       = "charts/cilium"
+)
+
 // ChartLoader loads embedded helm charts.
 type ChartLoader struct {
 	joinServiceImage             string
@@ -109,7 +121,11 @@ func (i *ChartLoader) Load(config *config.Config, conformanceMode bool, masterSe
 }
 
 func (i *ChartLoader) loadCilium(csp cloudprovider.Provider, conformanceMode bool) (helm.Release, error) {
-	chart, values, err := i.loadCiliumHelper(csp, conformanceMode)
+	chart, err := loadChartsDir(helmFS, ciliumPath)
+	if err != nil {
+		return helm.Release{}, fmt.Errorf("loading cilium chart: %w", err)
+	}
+	values, err := i.loadCiliumValues(csp, conformanceMode)
 	if err != nil {
 		return helm.Release{}, err
 	}
@@ -119,17 +135,12 @@ func (i *ChartLoader) loadCilium(csp cloudprovider.Provider, conformanceMode boo
 		return helm.Release{}, fmt.Errorf("packaging cilium chart: %w", err)
 	}
 
-	return helm.Release{Chart: chartRaw, Values: values, ReleaseName: "cilium", Wait: false}, nil
+	return helm.Release{Chart: chartRaw, Values: values, ReleaseName: ciliumReleaseName, Wait: false}, nil
 }
 
 // loadCiliumHelper is used to separate the marshalling step from the loading step.
 // This reduces the time unit tests take to execute.
-func (i *ChartLoader) loadCiliumHelper(csp cloudprovider.Provider, conformanceMode bool) (*chart.Chart, map[string]any, error) {
-	chart, err := loadChartsDir(helmFS, "charts/cilium")
-	if err != nil {
-		return nil, nil, fmt.Errorf("loading cilium chart: %w", err)
-	}
-
+func (i *ChartLoader) loadCiliumValues(csp cloudprovider.Provider, conformanceMode bool) (map[string]any, error) {
 	var values map[string]any
 	switch csp {
 	case cloudprovider.AWS:
@@ -141,7 +152,7 @@ func (i *ChartLoader) loadCiliumHelper(csp cloudprovider.Provider, conformanceMo
 	case cloudprovider.QEMU:
 		values = qemuVals
 	default:
-		return nil, nil, fmt.Errorf("unknown csp: %s", csp)
+		return nil, fmt.Errorf("unknown csp: %s", csp)
 	}
 	if conformanceMode {
 		values["kubeProxyReplacementHealthzBindAddr"] = ""
@@ -152,32 +163,28 @@ func (i *ChartLoader) loadCiliumHelper(csp cloudprovider.Provider, conformanceMo
 		}
 
 	}
-	return chart, values, nil
+	return values, nil
 }
 
 func (i *ChartLoader) loadCertManager() (helm.Release, error) {
-	chart, values, err := i.loadCertManagerHelper()
+	chart, err := loadChartsDir(helmFS, certManagerPath)
 	if err != nil {
-		return helm.Release{}, err
+		return helm.Release{}, fmt.Errorf("loading cert-manager chart: %w", err)
 	}
+	values := i.loadCertManagerValues()
 
 	chartRaw, err := i.marshalChart(chart)
 	if err != nil {
 		return helm.Release{}, fmt.Errorf("packaging cert-manager chart: %w", err)
 	}
 
-	return helm.Release{Chart: chartRaw, Values: values, ReleaseName: "cert-manager", Wait: false}, nil
+	return helm.Release{Chart: chartRaw, Values: values, ReleaseName: certManagerReleaseName, Wait: false}, nil
 }
 
 // loadCertManagerHelper is used to separate the marshalling step from the loading step.
 // This reduces the time unit tests take to execute.
-func (i *ChartLoader) loadCertManagerHelper() (*chart.Chart, map[string]any, error) {
-	chart, err := loadChartsDir(helmFS, "charts/cert-manager")
-	if err != nil {
-		return nil, nil, fmt.Errorf("loading cert-manager chart: %w", err)
-	}
-
-	values := map[string]any{
+func (i *ChartLoader) loadCertManagerValues() map[string]any {
+	return map[string]any{
 		"installCRDs": true,
 		"prometheus": map[string]any{
 			"enabled": false,
@@ -241,11 +248,14 @@ func (i *ChartLoader) loadCertManagerHelper() (*chart.Chart, map[string]any, err
 			},
 		},
 	}
-	return chart, values, nil
 }
 
 func (i *ChartLoader) loadOperators(csp cloudprovider.Provider) (helm.Release, error) {
-	chart, values, err := i.loadOperatorsHelper(csp)
+	chart, err := loadChartsDir(helmFS, conOperatorsPath)
+	if err != nil {
+		return helm.Release{}, fmt.Errorf("loading operators chart: %w", err)
+	}
+	values, err := i.loadOperatorsValues(csp)
 	if err != nil {
 		return helm.Release{}, err
 	}
@@ -255,17 +265,12 @@ func (i *ChartLoader) loadOperators(csp cloudprovider.Provider) (helm.Release, e
 		return helm.Release{}, fmt.Errorf("packaging operators chart: %w", err)
 	}
 
-	return helm.Release{Chart: chartRaw, Values: values, ReleaseName: "con-operators", Wait: false}, nil
+	return helm.Release{Chart: chartRaw, Values: values, ReleaseName: conOperatorsReleaseName, Wait: false}, nil
 }
 
 // loadOperatorsHelper is used to separate the marshalling step from the loading step.
 // This reduces the time unit tests take to execute.
-func (i *ChartLoader) loadOperatorsHelper(csp cloudprovider.Provider) (*chart.Chart, map[string]any, error) {
-	chart, err := loadChartsDir(helmFS, "charts/edgeless/operators")
-	if err != nil {
-		return nil, nil, fmt.Errorf("loading operators chart: %w", err)
-	}
-
+func (i *ChartLoader) loadOperatorsValues(csp cloudprovider.Provider) (map[string]any, error) {
 	values := map[string]any{
 		"constellation-operator": map[string]any{
 			"controllerManager": map[string]any{
@@ -286,7 +291,7 @@ func (i *ChartLoader) loadOperatorsHelper(csp cloudprovider.Provider) (*chart.Ch
 	case cloudprovider.Azure:
 		conOpVals, ok := values["constellation-operator"].(map[string]any)
 		if !ok {
-			return nil, nil, errors.New("invalid constellation-operator values")
+			return nil, errors.New("invalid constellation-operator values")
 		}
 		conOpVals["csp"] = "Azure"
 
@@ -296,7 +301,7 @@ func (i *ChartLoader) loadOperatorsHelper(csp cloudprovider.Provider) (*chart.Ch
 	case cloudprovider.GCP:
 		conOpVals, ok := values["constellation-operator"].(map[string]any)
 		if !ok {
-			return nil, nil, errors.New("invalid constellation-operator values")
+			return nil, errors.New("invalid constellation-operator values")
 		}
 		conOpVals["csp"] = "GCP"
 
@@ -306,7 +311,7 @@ func (i *ChartLoader) loadOperatorsHelper(csp cloudprovider.Provider) (*chart.Ch
 	case cloudprovider.QEMU:
 		conOpVals, ok := values["constellation-operator"].(map[string]any)
 		if !ok {
-			return nil, nil, errors.New("invalid constellation-operator values")
+			return nil, errors.New("invalid constellation-operator values")
 		}
 		conOpVals["csp"] = "QEMU"
 
@@ -316,7 +321,7 @@ func (i *ChartLoader) loadOperatorsHelper(csp cloudprovider.Provider) (*chart.Ch
 	case cloudprovider.AWS:
 		conOpVals, ok := values["constellation-operator"].(map[string]any)
 		if !ok {
-			return nil, nil, errors.New("invalid constellation-operator values")
+			return nil, errors.New("invalid constellation-operator values")
 		}
 		conOpVals["csp"] = "AWS"
 
@@ -325,13 +330,17 @@ func (i *ChartLoader) loadOperatorsHelper(csp cloudprovider.Provider) (*chart.Ch
 		}
 	}
 
-	return chart, values, nil
+	return values, nil
 }
 
 // loadConstellationServices loads the constellation-services chart from the embed.FS,
 // marshals it into a helm-package .tgz and sets the values that can be set in the CLI.
 func (i *ChartLoader) loadConstellationServices(config *config.Config, masterSecret, salt []byte) (helm.Release, error) {
-	chart, values, err := i.loadConstellationServicesHelper(config, masterSecret, salt)
+	chart, err := loadChartsDir(helmFS, conServicesPath)
+	if err != nil {
+		return helm.Release{}, fmt.Errorf("loading constellation-services chart: %w", err)
+	}
+	values, err := i.loadConstellationServicesValues(config, masterSecret, salt)
 	if err != nil {
 		return helm.Release{}, err
 	}
@@ -341,17 +350,12 @@ func (i *ChartLoader) loadConstellationServices(config *config.Config, masterSec
 		return helm.Release{}, fmt.Errorf("packaging constellation-services chart: %w", err)
 	}
 
-	return helm.Release{Chart: chartRaw, Values: values, ReleaseName: "constellation-services", Wait: false}, nil
+	return helm.Release{Chart: chartRaw, Values: values, ReleaseName: conServicesReleaseName, Wait: false}, nil
 }
 
 // loadConstellationServicesHelper is used to separate the marshalling step from the loading step.
 // This reduces the time unit tests take to execute.
-func (i *ChartLoader) loadConstellationServicesHelper(config *config.Config, masterSecret, salt []byte) (*chart.Chart, map[string]any, error) {
-	chart, err := loadChartsDir(helmFS, "charts/edgeless/constellation-services")
-	if err != nil {
-		return nil, nil, fmt.Errorf("loading constellation-services chart: %w", err)
-	}
-
+func (i *ChartLoader) loadConstellationServicesValues(config *config.Config, masterSecret, salt []byte) (map[string]any, error) {
 	csp := config.GetProvider()
 	values := map[string]any{
 		"global": map[string]any{
@@ -397,13 +401,13 @@ func (i *ChartLoader) loadConstellationServicesHelper(config *config.Config, mas
 	case cloudprovider.Azure:
 		joinServiceVals, ok := values["join-service"].(map[string]any)
 		if !ok {
-			return nil, nil, errors.New("invalid join-service values")
+			return nil, errors.New("invalid join-service values")
 		}
 		joinServiceVals["enforceIdKeyDigest"] = config.EnforcesIDKeyDigest()
 
 		ccmVals, ok := values["ccm"].(map[string]any)
 		if !ok {
-			return nil, nil, errors.New("invalid ccm values")
+			return nil, errors.New("invalid ccm values")
 		}
 		ccmVals["Azure"] = map[string]any{
 			"image": i.ccmImage,
@@ -431,7 +435,7 @@ func (i *ChartLoader) loadConstellationServicesHelper(config *config.Config, mas
 	case cloudprovider.GCP:
 		ccmVals, ok := values["ccm"].(map[string]any)
 		if !ok {
-			return nil, nil, errors.New("invalid ccm values")
+			return nil, errors.New("invalid ccm values")
 		}
 		ccmVals["GCP"] = map[string]any{
 			"image": i.ccmImage,
@@ -460,7 +464,7 @@ func (i *ChartLoader) loadConstellationServicesHelper(config *config.Config, mas
 	case cloudprovider.AWS:
 		ccmVals, ok := values["ccm"].(map[string]any)
 		if !ok {
-			return nil, nil, errors.New("invalid ccm values")
+			return nil, errors.New("invalid ccm values")
 		}
 		ccmVals["AWS"] = map[string]any{
 			"image": i.ccmImage,
@@ -470,7 +474,7 @@ func (i *ChartLoader) loadConstellationServicesHelper(config *config.Config, mas
 			"AWS": true,
 		}
 	}
-	return chart, values, nil
+	return values, nil
 }
 
 // marshalChart takes a Chart object, packages it to a temporary file and returns the content of that file.

cli/internal/helm/loader_test.go

@@ -98,7 +98,9 @@ func TestConstellationServices(t *testing.T) {
 				konnectivityImage:        "konnectivityImage",
 				gcpGuestAgentImage:       "gcpGuestAgentImage",
 			}
-			chart, values, err := chartLoader.loadConstellationServicesHelper(tc.config, []byte("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"), []byte("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"))
+			chart, err := loadChartsDir(helmFS, conServicesPath)
+			require.NoError(err)
+			values, err := chartLoader.loadConstellationServicesValues(tc.config, []byte("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"), []byte("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"))
 			require.NoError(err)
 
 			options := chartutil.ReleaseOptions{
@@ -164,7 +166,9 @@ func TestOperators(t *testing.T) {
 				constellationOperatorImage:   "constellationOperatorImage",
 				nodeMaintenanceOperatorImage: "nodeMaintenanceOperatorImage",
 			}
-			chart, vals, err := chartLoader.loadOperatorsHelper(tc.csp)
+			chart, err := loadChartsDir(helmFS, conOperatorsPath)
+			require.NoError(err)
+			vals, err := chartLoader.loadOperatorsValues(tc.csp)
 			require.NoError(err)
 
 			options := chartutil.ReleaseOptions{