From efcd0337b4ad22f89018b5c408b4a83f5ed5e43e Mon Sep 17 00:00:00 2001 From: Otto Bittner Date: Mon, 19 Dec 2022 16:52:15 +0100 Subject: [PATCH] Microservice upgrades (#729) Run with: constellation upgrade execute --helm. This will only upgrade the helm charts. No config is needed. Upgrades are implemented via helm's upgrade action, i.e. they automatically roll back if something goes wrong. Releases could still be managed via helm, even after an upgrade with constellation has been done. Currently not user facing as CRD/CR backups are still in progress. These backups should be automatically created and saved to the user's disk as updates may delete CRs. This happens implicitly through CRD upgrades, which are part of microservice upgrades. --- bootstrapper/internal/helm/client.go | 25 +--- bootstrapper/internal/helm/client_test.go | 85 ----------- cli/internal/cloudcmd/upgrade.go | 23 ++- cli/internal/cmd/upgradeexecute.go | 35 ++++- cli/internal/cmd/upgradeexecute_test.go | 8 +- cli/internal/cmd/upgradeplan.go | 8 +- cli/internal/helm/README.md | 23 +++ cli/internal/helm/client.go | 174 +++++++++++++++++++++- cli/internal/helm/client_test.go | 46 ++++++ cli/internal/helm/loader.go | 98 ++++++------ cli/internal/helm/loader_test.go | 8 +- internal/deploy/helm/helm.go | 22 +++ internal/deploy/helm/helm_test.go | 109 ++++++++++++++ 13 files changed, 498 insertions(+), 166 deletions(-) create mode 100644 cli/internal/helm/README.md create mode 100644 cli/internal/helm/client_test.go create mode 100644 internal/deploy/helm/helm_test.go diff --git a/bootstrapper/internal/helm/client.go b/bootstrapper/internal/helm/client.go index a88ed7e46..c5dc0652b 100644 --- a/bootstrapper/internal/helm/client.go +++ b/bootstrapper/internal/helm/client.go @@ -61,7 +61,7 @@ func (h *Client) InstallConstellationServices(ctx context.Context, release helm. h.ReleaseName = release.ReleaseName h.Wait = release.Wait - mergedVals := mergeMaps(release.Values, extraVals) + mergedVals := helm.MergeMaps(release.Values, extraVals) if err := h.install(ctx, release.Chart, mergedVals); err != nil { return err @@ -87,7 +87,7 @@ func (h *Client) InstallOperators(ctx context.Context, release helm.Release, ext h.ReleaseName = release.ReleaseName h.Wait = release.Wait - mergedVals := mergeMaps(release.Values, extraVals) + mergedVals := helm.MergeMaps(release.Values, extraVals) if err := h.install(ctx, release.Chart, mergedVals); err != nil { return err @@ -190,24 +190,3 @@ func (h *Client) install(ctx context.Context, chartRaw []byte, values map[string } return nil } - -// mergeMaps returns a new map that is the merger of it's inputs. -// Taken from: https://github.com/helm/helm/blob/dbc6d8e20fe1d58d50e6ed30f09a04a77e4c68db/pkg/cli/values/options.go#L91-L108. -func mergeMaps(a, b map[string]any) map[string]any { - out := make(map[string]any, len(a)) - for k, v := range a { - out[k] = v - } - for k, v := range b { - if v, ok := v.(map[string]any); ok { - if bv, ok := out[k]; ok { - if bv, ok := bv.(map[string]any); ok { - out[k] = mergeMaps(bv, v) - continue - } - } - } - out[k] = v - } - return out -} diff --git a/bootstrapper/internal/helm/client_test.go b/bootstrapper/internal/helm/client_test.go index f223af2ae..cbeff5be0 100644 --- a/bootstrapper/internal/helm/client_test.go +++ b/bootstrapper/internal/helm/client_test.go @@ -5,88 +5,3 @@ SPDX-License-Identifier: AGPL-3.0-only */ package helm - -import ( - "testing" - - "github.com/stretchr/testify/assert" -) - -func TestMe(t *testing.T) { - testCases := map[string]struct { - vals map[string]any - extraVals map[string]any - expected map[string]any - }{ - "equal": { - vals: map[string]any{ - "join-service": map[string]any{ - "key1": "foo", - "key2": "bar", - }, - }, - extraVals: map[string]any{ - "join-service": map[string]any{ - "extraKey1": "extraFoo", - "extraKey2": "extraBar", - }, - }, - expected: map[string]any{ - "join-service": map[string]any{ - "key1": "foo", - "key2": "bar", - "extraKey1": "extraFoo", - "extraKey2": "extraBar", - }, - }, - }, - "missing join-service extraVals": { - vals: map[string]any{ - "join-service": map[string]any{ - "key1": "foo", - "key2": "bar", - }, - }, - extraVals: map[string]any{ - "extraKey1": "extraFoo", - "extraKey2": "extraBar", - }, - expected: map[string]any{ - "join-service": map[string]any{ - "key1": "foo", - "key2": "bar", - }, - "extraKey1": "extraFoo", - "extraKey2": "extraBar", - }, - }, - "missing join-service vals": { - vals: map[string]any{ - "key1": "foo", - "key2": "bar", - }, - extraVals: map[string]any{ - "join-service": map[string]any{ - "extraKey1": "extraFoo", - "extraKey2": "extraBar", - }, - }, - expected: map[string]any{ - "key1": "foo", - "key2": "bar", - "join-service": map[string]any{ - "extraKey1": "extraFoo", - "extraKey2": "extraBar", - }, - }, - }, - } - - for name, tc := range testCases { - t.Run(name, func(t *testing.T) { - assert := assert.New(t) - newVals := mergeMaps(tc.vals, tc.extraVals) - assert.Equal(tc.expected, newVals) - }) - } -} diff --git a/cli/internal/cloudcmd/upgrade.go b/cli/internal/cloudcmd/upgrade.go index c86558d6a..02b168442 100644 --- a/cli/internal/cloudcmd/upgrade.go +++ b/cli/internal/cloudcmd/upgrade.go @@ -12,11 +12,14 @@ import ( "errors" "fmt" "io" + "time" "github.com/edgelesssys/constellation/v2/cli/internal/helm" "github.com/edgelesssys/constellation/v2/internal/attestation/measurements" + "github.com/edgelesssys/constellation/v2/internal/config" "github.com/edgelesssys/constellation/v2/internal/constants" corev1 "k8s.io/api/core/v1" + apiextensionsclient "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime/schema" @@ -35,7 +38,7 @@ type Upgrader struct { } // NewUpgrader returns a new Upgrader. -func NewUpgrader(outWriter io.Writer) (*Upgrader, error) { +func NewUpgrader(outWriter io.Writer, log debugLog) (*Upgrader, error) { kubeConfig, err := clientcmd.BuildConfigFromFlags("", constants.AdminConfFilename) if err != nil { return nil, fmt.Errorf("building kubernetes config: %w", err) @@ -52,7 +55,12 @@ func NewUpgrader(outWriter io.Writer) (*Upgrader, error) { return nil, fmt.Errorf("setting up custom resource client: %w", err) } - helmClient, err := helm.NewClient(constants.AdminConfFilename, constants.HelmNamespace) + client, err := apiextensionsclient.NewForConfig(kubeConfig) + if err != nil { + return nil, err + } + + helmClient, err := helm.NewClient(constants.AdminConfFilename, constants.HelmNamespace, client, log) if err != nil { return nil, fmt.Errorf("setting up helm client: %w", err) } @@ -105,6 +113,11 @@ func (u *Upgrader) GetCurrentImage(ctx context.Context) (*unstructured.Unstructu return imageStruct, imageVersion, nil } +// UpgradeHelmServices upgrade helm services. +func (u *Upgrader) UpgradeHelmServices(ctx context.Context, config *config.Config, timeout time.Duration) error { + return u.helmClient.Upgrade(ctx, config, timeout) +} + // CurrentHelmVersion returns the version of the currently installed helm release. func (u *Upgrader) CurrentHelmVersion(release string) (string, error) { return u.helmClient.CurrentVersion(release) @@ -232,4 +245,10 @@ func (u *stableClient) kubernetesVersion() (string, error) { type helmInterface interface { CurrentVersion(release string) (string, error) + Upgrade(ctx context.Context, config *config.Config, timeout time.Duration) error +} + +type debugLog interface { + Debugf(format string, args ...any) + Sync() } diff --git a/cli/internal/cmd/upgradeexecute.go b/cli/internal/cmd/upgradeexecute.go index 837cd088d..7f8c51b09 100644 --- a/cli/internal/cmd/upgradeexecute.go +++ b/cli/internal/cmd/upgradeexecute.go @@ -8,6 +8,8 @@ package cmd import ( "context" + "fmt" + "time" "github.com/edgelesssys/constellation/v2/cli/internal/cloudcmd" "github.com/edgelesssys/constellation/v2/internal/attestation/measurements" @@ -27,13 +29,28 @@ func newUpgradeExecuteCmd() *cobra.Command { RunE: runUpgradeExecute, } + cmd.Flags().Bool("helm", false, "Execute helm upgrade. This feature is still in development an may change without anounncement. Upgrades all helm charts deployed during constellation-init.") + cmd.Flags().Duration("timeout", 3*time.Minute, "Change helm upgrade timeout. This feature is still in development an may change without anounncement. Might be useful for slow connections or big clusters.") + if err := cmd.Flags().MarkHidden("helm"); err != nil { + panic(err) + } + if err := cmd.Flags().MarkHidden("timeout"); err != nil { + panic(err) + } + return cmd } func runUpgradeExecute(cmd *cobra.Command, args []string) error { + log, err := newCLILogger(cmd) + if err != nil { + return fmt.Errorf("creating logger: %w", err) + } + defer log.Sync() + fileHandler := file.NewHandler(afero.NewOsFs()) imageFetcher := image.New() - upgrader, err := cloudcmd.NewUpgrader(cmd.OutOrStdout()) + upgrader, err := cloudcmd.NewUpgrader(cmd.OutOrStdout(), log) if err != nil { return err } @@ -51,6 +68,21 @@ func upgradeExecute(cmd *cobra.Command, imageFetcher imageFetcher, upgrader clou return displayConfigValidationErrors(cmd.ErrOrStderr(), err) } + helm, err := cmd.Flags().GetBool("helm") + if err != nil { + return err + } + if helm { + timeout, err := cmd.Flags().GetDuration("timeout") + if err != nil { + return err + } + if err := upgrader.UpgradeHelmServices(cmd.Context(), conf, timeout); err != nil { + return fmt.Errorf("upgrading helm: %w", err) + } + return nil + } + // TODO: validate upgrade config? Should be basic things like checking image is not an empty string // More sophisticated validation, like making sure we don't downgrade the cluster, should be done by `constellation upgrade plan` @@ -66,6 +98,7 @@ func upgradeExecute(cmd *cobra.Command, imageFetcher imageFetcher, upgrader clou type cloudUpgrader interface { Upgrade(ctx context.Context, imageReference, imageVersion string, measurements measurements.M) error + UpgradeHelmServices(ctx context.Context, config *config.Config, timeout time.Duration) error } type imageFetcher interface { diff --git a/cli/internal/cmd/upgradeexecute_test.go b/cli/internal/cmd/upgradeexecute_test.go index f86e80f76..02fef4aa6 100644 --- a/cli/internal/cmd/upgradeexecute_test.go +++ b/cli/internal/cmd/upgradeexecute_test.go @@ -10,6 +10,7 @@ import ( "context" "errors" "testing" + "time" "github.com/edgelesssys/constellation/v2/internal/attestation/measurements" "github.com/edgelesssys/constellation/v2/internal/cloud/cloudprovider" @@ -66,13 +67,18 @@ func TestUpgradeExecute(t *testing.T) { } type stubUpgrader struct { - err error + err error + helmErr error } func (u stubUpgrader) Upgrade(context.Context, string, string, measurements.M) error { return u.err } +func (u stubUpgrader) UpgradeHelmServices(ctx context.Context, config *config.Config, timeout time.Duration) error { + return u.helmErr +} + type stubImageFetcher struct { reference string fetchReferenceErr error diff --git a/cli/internal/cmd/upgradeplan.go b/cli/internal/cmd/upgradeplan.go index 070efad12..a4dba4bff 100644 --- a/cli/internal/cmd/upgradeplan.go +++ b/cli/internal/cmd/upgradeplan.go @@ -44,12 +44,18 @@ func newUpgradePlanCmd() *cobra.Command { } func runUpgradePlan(cmd *cobra.Command, args []string) error { + log, err := newCLILogger(cmd) + if err != nil { + return fmt.Errorf("creating logger: %w", err) + } + defer log.Sync() + fileHandler := file.NewHandler(afero.NewOsFs()) flags, err := parseUpgradePlanFlags(cmd) if err != nil { return err } - planner, err := cloudcmd.NewUpgrader(cmd.OutOrStdout()) + planner, err := cloudcmd.NewUpgrader(cmd.OutOrStdout(), log) if err != nil { return err } diff --git a/cli/internal/helm/README.md b/cli/internal/helm/README.md new file mode 100644 index 000000000..b34d13d04 --- /dev/null +++ b/cli/internal/helm/README.md @@ -0,0 +1,23 @@ +# Chart upgrades + +All services that are installed via helm-install are upgraded via helm-upgrade. +Two aspects are not full covered by running helm-upgrade: CRDs and values. +While helm-install can install CRDs if they are contained in a chart's `crds` folder, upgrade won't change any installed CRDs. +Furthermore, new values introduced with a new version of a chart will not be installed into the cluster if the `--reuse-values` flag is set. +Nevertheless, we have to rely on the values already present in the cluster because some of the values are set by the bootstrapper during installation. +Because upgrades should be a CLI-only operation and we want to avoid the behaviour of `--reuse-values`, we fetch the cluster values and merge them with any new values. + +Here is how we manage CRD upgrades for each chart. + +## Cilium +- CRDs are updated by cilium-operator. + +## cert-manager +- installCRDs flag is set during upgrade. This flag is managed by cert-manager. cert-manager is in charge of correctly upgrading the CRDs. +- WARNING: upgrading cert-manager might break other installations of cert-manager in the cluster, if those other installation are not on the same version as the Constellation-manager installation. This is due to the cluster-wide CRDs. + +## Operators +- Manually update CRDs before upgrading the chart. Update by running applying the CRDs found in the `operators/crds/` folder. + +## Constellation-services +- There currently are no CRDs in this chart. diff --git a/cli/internal/helm/client.go b/cli/internal/helm/client.go index 49ca3edae..d67fc2f84 100644 --- a/cli/internal/helm/client.go +++ b/cli/internal/helm/client.go @@ -7,28 +7,44 @@ SPDX-License-Identifier: AGPL-3.0-only package helm import ( + "context" "fmt" + "strings" + "time" + "github.com/edgelesssys/constellation/v2/internal/config" + "github.com/edgelesssys/constellation/v2/internal/constants" + "github.com/edgelesssys/constellation/v2/internal/deploy/helm" + "github.com/pkg/errors" "helm.sh/helm/v3/pkg/action" + "helm.sh/helm/v3/pkg/chart" "helm.sh/helm/v3/pkg/cli" + v1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + apiextensionsclient "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/serializer" + "k8s.io/client-go/kubernetes/scheme" ) // Client handles interaction with helm. type Client struct { - config *action.Configuration + config *action.Configuration + crdClient *apiextensionsclient.Clientset + log debugLog } // NewClient returns a new initializes client for the namespace Client. -func NewClient(kubeConfigPath, helmNamespace string) (*Client, error) { +func NewClient(kubeConfigPath, helmNamespace string, client *apiextensionsclient.Clientset, log debugLog) (*Client, error) { settings := cli.New() settings.KubeConfig = kubeConfigPath // constants.AdminConfFilename actionConfig := &action.Configuration{} - if err := actionConfig.Init(settings.RESTClientGetter(), helmNamespace, "secret", nil); err != nil { + if err := actionConfig.Init(settings.RESTClientGetter(), helmNamespace, "secret", log.Debugf); err != nil { return nil, fmt.Errorf("initializing config: %w", err) } - return &Client{config: actionConfig}, nil + return &Client{config: actionConfig, crdClient: client, log: log}, nil } // CurrentVersion returns the version of the currently installed helm release. @@ -53,3 +69,153 @@ func (c *Client) CurrentVersion(release string) (string, error) { return rel[0].Chart.Metadata.Version, nil } + +// Upgrade runs a helm-upgrade on all deployments that are managed via Helm. +// If the CLI receives an interrupt signal it will cancel the context. +// Canceling the context will prompt helm to abort and roll back the ongoing upgrade. +func (c *Client) Upgrade(ctx context.Context, config *config.Config, timeout time.Duration) error { + action := action.NewUpgrade(c.config) + action.Atomic = true + action.Namespace = constants.HelmNamespace + action.ReuseValues = false + action.Timeout = timeout + + ciliumChart, err := loadChartsDir(helmFS, ciliumPath) + if err != nil { + return fmt.Errorf("loading cilium: %w", err) + } + certManagerChart, err := loadChartsDir(helmFS, certManagerPath) + if err != nil { + return fmt.Errorf("loading cert-manager: %w", err) + } + conOperatorChart, err := loadChartsDir(helmFS, conOperatorsPath) + if err != nil { + return fmt.Errorf("loading operators: %w", err) + } + conServicesChart, err := loadChartsDir(helmFS, conServicesPath) + if err != nil { + return fmt.Errorf("loading constellation-services chart: %w", err) + } + + values, err := c.prepareValues(ciliumChart, ciliumReleaseName) + if err != nil { + return err + } + if _, err := action.RunWithContext(ctx, ciliumReleaseName, ciliumChart, values); err != nil { + return fmt.Errorf("upgrading cilium: %w", err) + } + + values, err = c.prepareValues(certManagerChart, certManagerReleaseName) + if err != nil { + return err + } + if _, err := action.RunWithContext(ctx, certManagerReleaseName, certManagerChart, values); err != nil { + return fmt.Errorf("upgrading cert-manager: %w", err) + } + + err = c.updateOperatorCRDs(ctx, conOperatorChart) + if err != nil { + return fmt.Errorf("updating operator CRDs: %w", err) + } + values, err = c.prepareValues(conOperatorChart, conOperatorsReleaseName) + if err != nil { + return err + } + if _, err := action.RunWithContext(ctx, conOperatorsReleaseName, conOperatorChart, values); err != nil { + return fmt.Errorf("upgrading services: %w", err) + } + + values, err = c.prepareValues(conServicesChart, conServicesReleaseName) + if err != nil { + return err + } + if _, err := action.RunWithContext(ctx, conServicesReleaseName, conServicesChart, values); err != nil { + return fmt.Errorf("upgrading operators: %w", err) + } + + return nil +} + +// prepareCertManagerValues returns a values map as required for helm-upgrade. +// It imitates the behaviour of helm's reuse-values flag by fetching the current values from the cluster +// and merging the fetched values with the locally found values. +// This is done to ensure that new values (from upgrades of the local files) end up in the cluster. +// reuse-values does not ensure this. +func (c *Client) prepareValues(chart *chart.Chart, releaseName string) (map[string]any, error) { + // Ensure installCRDs is set for cert-manager chart. + if releaseName == certManagerReleaseName { + chart.Values["installCRDs"] = true + } + values, err := c.GetValues(releaseName) + if err != nil { + return nil, fmt.Errorf("getting values: %w", err) + } + return helm.MergeMaps(chart.Values, values), nil +} + +// GetValues queries the cluster for the values of the given release. +func (c *Client) GetValues(release string) (map[string]any, error) { + client := action.NewGetValues(c.config) + // Version corresponds to the releases revision. Specifying a Version <= 0 yields the latest release. + client.Version = 0 + values, err := client.Run(release) + if err != nil { + return nil, fmt.Errorf("getting values for %s: %w", release, err) + } + return values, nil +} + +// ApplyCRD updates the given CRD by parsing it, querying it's version from the cluster and finally updating it. +func (c *Client) ApplyCRD(ctx context.Context, rawCRD []byte) error { + crd, err := parseCRD(rawCRD) + if err != nil { + return fmt.Errorf("parsing crds: %w", err) + } + + clusterCRD, err := c.crdClient.ApiextensionsV1().CustomResourceDefinitions().Get(ctx, crd.Name, metav1.GetOptions{}) + if err != nil { + return fmt.Errorf("getting crd: %w", err) + } + crd.ResourceVersion = clusterCRD.ResourceVersion + _, err = c.crdClient.ApiextensionsV1().CustomResourceDefinitions().Update(ctx, crd, metav1.UpdateOptions{}) + return err +} + +// parseCRD takes a byte slice of data and tries to create a CustomResourceDefinition object from it. +func parseCRD(crdString []byte) (*v1.CustomResourceDefinition, error) { + sch := runtime.NewScheme() + _ = scheme.AddToScheme(sch) + _ = v1.AddToScheme(sch) + obj, groupVersionKind, err := serializer.NewCodecFactory(sch).UniversalDeserializer().Decode(crdString, nil, nil) + if err != nil { + return nil, fmt.Errorf("decoding crd: %w", err) + } + if groupVersionKind.Kind == "CustomResourceDefinition" { + return obj.(*v1.CustomResourceDefinition), nil + } + + return nil, errors.New("parsed []byte, but did not find a CRD") +} + +// updateOperatorCRDs walks through the dependencies of the given chart and applies +// the files in the dependencie's 'crds' folder. +// This function is NOT recursive! +func (c *Client) updateOperatorCRDs(ctx context.Context, chart *chart.Chart) error { + for _, dep := range chart.Dependencies() { + for _, crdFile := range dep.Files { + if strings.HasPrefix(crdFile.Name, "crds/") { + c.log.Debugf("updating crd: %s", crdFile.Name) + err := c.ApplyCRD(ctx, crdFile.Data) + if err != nil { + return err + } + } + } + } + return nil +} + +type debugLog interface { + Debugf(format string, args ...any) + Sync() +} diff --git a/cli/internal/helm/client_test.go b/cli/internal/helm/client_test.go new file mode 100644 index 000000000..cec58c7b1 --- /dev/null +++ b/cli/internal/helm/client_test.go @@ -0,0 +1,46 @@ +/* +Copyright (c) Edgeless Systems GmbH + +SPDX-License-Identifier: AGPL-3.0-only +*/ + +package helm + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestParseCRDs(t *testing.T) { + testCases := map[string]struct { + data string + wantErr bool + }{ + "success": { + data: "apiVersion: apiextensions.k8s.io/v1\nkind: CustomResourceDefinition\nmetadata:\n name: nodeimages.update.edgeless.systems\nspec:\n group: update.edgeless.systems\n names:\n kind: NodeImage\n", + wantErr: false, + }, + "wrong kind": { + data: "apiVersion: v1\nkind: Secret\ntype: Opaque\nmetadata:\n name: supersecret\n namespace: testNamespace\ndata:\n data: YWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWE=\n", + wantErr: true, + }, + "decoding error": { + data: "asdf", + wantErr: true, + }, + } + + for name, tc := range testCases { + t.Run(name, func(t *testing.T) { + assert := assert.New(t) + + _, err := parseCRD([]byte(tc.data)) + if tc.wantErr { + assert.Error(err) + return + } + assert.NoError(err) + }) + } +} diff --git a/cli/internal/helm/loader.go b/cli/internal/helm/loader.go index 3b359b5a4..4250dd2b8 100644 --- a/cli/internal/helm/loader.go +++ b/cli/internal/helm/loader.go @@ -37,6 +37,18 @@ import ( //go:embed all:charts/* var helmFS embed.FS +const ( + ciliumReleaseName = "cilium" + conServicesReleaseName = "constellation-services" + conOperatorsReleaseName = "constellation-operators" + certManagerReleaseName = "cert-manager" + + conServicesPath = "charts/edgeless/constellation-services" + conOperatorsPath = "charts/edgeless/operators" + certManagerPath = "charts/cert-manager" + ciliumPath = "charts/cilium" +) + // ChartLoader loads embedded helm charts. type ChartLoader struct { joinServiceImage string @@ -109,7 +121,11 @@ func (i *ChartLoader) Load(config *config.Config, conformanceMode bool, masterSe } func (i *ChartLoader) loadCilium(csp cloudprovider.Provider, conformanceMode bool) (helm.Release, error) { - chart, values, err := i.loadCiliumHelper(csp, conformanceMode) + chart, err := loadChartsDir(helmFS, ciliumPath) + if err != nil { + return helm.Release{}, fmt.Errorf("loading cilium chart: %w", err) + } + values, err := i.loadCiliumValues(csp, conformanceMode) if err != nil { return helm.Release{}, err } @@ -119,17 +135,12 @@ func (i *ChartLoader) loadCilium(csp cloudprovider.Provider, conformanceMode boo return helm.Release{}, fmt.Errorf("packaging cilium chart: %w", err) } - return helm.Release{Chart: chartRaw, Values: values, ReleaseName: "cilium", Wait: false}, nil + return helm.Release{Chart: chartRaw, Values: values, ReleaseName: ciliumReleaseName, Wait: false}, nil } // loadCiliumHelper is used to separate the marshalling step from the loading step. // This reduces the time unit tests take to execute. -func (i *ChartLoader) loadCiliumHelper(csp cloudprovider.Provider, conformanceMode bool) (*chart.Chart, map[string]any, error) { - chart, err := loadChartsDir(helmFS, "charts/cilium") - if err != nil { - return nil, nil, fmt.Errorf("loading cilium chart: %w", err) - } - +func (i *ChartLoader) loadCiliumValues(csp cloudprovider.Provider, conformanceMode bool) (map[string]any, error) { var values map[string]any switch csp { case cloudprovider.AWS: @@ -141,7 +152,7 @@ func (i *ChartLoader) loadCiliumHelper(csp cloudprovider.Provider, conformanceMo case cloudprovider.QEMU: values = qemuVals default: - return nil, nil, fmt.Errorf("unknown csp: %s", csp) + return nil, fmt.Errorf("unknown csp: %s", csp) } if conformanceMode { values["kubeProxyReplacementHealthzBindAddr"] = "" @@ -152,32 +163,28 @@ func (i *ChartLoader) loadCiliumHelper(csp cloudprovider.Provider, conformanceMo } } - return chart, values, nil + return values, nil } func (i *ChartLoader) loadCertManager() (helm.Release, error) { - chart, values, err := i.loadCertManagerHelper() + chart, err := loadChartsDir(helmFS, certManagerPath) if err != nil { - return helm.Release{}, err + return helm.Release{}, fmt.Errorf("loading cert-manager chart: %w", err) } + values := i.loadCertManagerValues() chartRaw, err := i.marshalChart(chart) if err != nil { return helm.Release{}, fmt.Errorf("packaging cert-manager chart: %w", err) } - return helm.Release{Chart: chartRaw, Values: values, ReleaseName: "cert-manager", Wait: false}, nil + return helm.Release{Chart: chartRaw, Values: values, ReleaseName: certManagerReleaseName, Wait: false}, nil } // loadCertManagerHelper is used to separate the marshalling step from the loading step. // This reduces the time unit tests take to execute. -func (i *ChartLoader) loadCertManagerHelper() (*chart.Chart, map[string]any, error) { - chart, err := loadChartsDir(helmFS, "charts/cert-manager") - if err != nil { - return nil, nil, fmt.Errorf("loading cert-manager chart: %w", err) - } - - values := map[string]any{ +func (i *ChartLoader) loadCertManagerValues() map[string]any { + return map[string]any{ "installCRDs": true, "prometheus": map[string]any{ "enabled": false, @@ -241,11 +248,14 @@ func (i *ChartLoader) loadCertManagerHelper() (*chart.Chart, map[string]any, err }, }, } - return chart, values, nil } func (i *ChartLoader) loadOperators(csp cloudprovider.Provider) (helm.Release, error) { - chart, values, err := i.loadOperatorsHelper(csp) + chart, err := loadChartsDir(helmFS, conOperatorsPath) + if err != nil { + return helm.Release{}, fmt.Errorf("loading operators chart: %w", err) + } + values, err := i.loadOperatorsValues(csp) if err != nil { return helm.Release{}, err } @@ -255,17 +265,12 @@ func (i *ChartLoader) loadOperators(csp cloudprovider.Provider) (helm.Release, e return helm.Release{}, fmt.Errorf("packaging operators chart: %w", err) } - return helm.Release{Chart: chartRaw, Values: values, ReleaseName: "con-operators", Wait: false}, nil + return helm.Release{Chart: chartRaw, Values: values, ReleaseName: conOperatorsReleaseName, Wait: false}, nil } // loadOperatorsHelper is used to separate the marshalling step from the loading step. // This reduces the time unit tests take to execute. -func (i *ChartLoader) loadOperatorsHelper(csp cloudprovider.Provider) (*chart.Chart, map[string]any, error) { - chart, err := loadChartsDir(helmFS, "charts/edgeless/operators") - if err != nil { - return nil, nil, fmt.Errorf("loading operators chart: %w", err) - } - +func (i *ChartLoader) loadOperatorsValues(csp cloudprovider.Provider) (map[string]any, error) { values := map[string]any{ "constellation-operator": map[string]any{ "controllerManager": map[string]any{ @@ -286,7 +291,7 @@ func (i *ChartLoader) loadOperatorsHelper(csp cloudprovider.Provider) (*chart.Ch case cloudprovider.Azure: conOpVals, ok := values["constellation-operator"].(map[string]any) if !ok { - return nil, nil, errors.New("invalid constellation-operator values") + return nil, errors.New("invalid constellation-operator values") } conOpVals["csp"] = "Azure" @@ -296,7 +301,7 @@ func (i *ChartLoader) loadOperatorsHelper(csp cloudprovider.Provider) (*chart.Ch case cloudprovider.GCP: conOpVals, ok := values["constellation-operator"].(map[string]any) if !ok { - return nil, nil, errors.New("invalid constellation-operator values") + return nil, errors.New("invalid constellation-operator values") } conOpVals["csp"] = "GCP" @@ -306,7 +311,7 @@ func (i *ChartLoader) loadOperatorsHelper(csp cloudprovider.Provider) (*chart.Ch case cloudprovider.QEMU: conOpVals, ok := values["constellation-operator"].(map[string]any) if !ok { - return nil, nil, errors.New("invalid constellation-operator values") + return nil, errors.New("invalid constellation-operator values") } conOpVals["csp"] = "QEMU" @@ -316,7 +321,7 @@ func (i *ChartLoader) loadOperatorsHelper(csp cloudprovider.Provider) (*chart.Ch case cloudprovider.AWS: conOpVals, ok := values["constellation-operator"].(map[string]any) if !ok { - return nil, nil, errors.New("invalid constellation-operator values") + return nil, errors.New("invalid constellation-operator values") } conOpVals["csp"] = "AWS" @@ -325,13 +330,17 @@ func (i *ChartLoader) loadOperatorsHelper(csp cloudprovider.Provider) (*chart.Ch } } - return chart, values, nil + return values, nil } // loadConstellationServices loads the constellation-services chart from the embed.FS, // marshals it into a helm-package .tgz and sets the values that can be set in the CLI. func (i *ChartLoader) loadConstellationServices(config *config.Config, masterSecret, salt []byte) (helm.Release, error) { - chart, values, err := i.loadConstellationServicesHelper(config, masterSecret, salt) + chart, err := loadChartsDir(helmFS, conServicesPath) + if err != nil { + return helm.Release{}, fmt.Errorf("loading constellation-services chart: %w", err) + } + values, err := i.loadConstellationServicesValues(config, masterSecret, salt) if err != nil { return helm.Release{}, err } @@ -341,17 +350,12 @@ func (i *ChartLoader) loadConstellationServices(config *config.Config, masterSec return helm.Release{}, fmt.Errorf("packaging constellation-services chart: %w", err) } - return helm.Release{Chart: chartRaw, Values: values, ReleaseName: "constellation-services", Wait: false}, nil + return helm.Release{Chart: chartRaw, Values: values, ReleaseName: conServicesReleaseName, Wait: false}, nil } // loadConstellationServicesHelper is used to separate the marshalling step from the loading step. // This reduces the time unit tests take to execute. -func (i *ChartLoader) loadConstellationServicesHelper(config *config.Config, masterSecret, salt []byte) (*chart.Chart, map[string]any, error) { - chart, err := loadChartsDir(helmFS, "charts/edgeless/constellation-services") - if err != nil { - return nil, nil, fmt.Errorf("loading constellation-services chart: %w", err) - } - +func (i *ChartLoader) loadConstellationServicesValues(config *config.Config, masterSecret, salt []byte) (map[string]any, error) { csp := config.GetProvider() values := map[string]any{ "global": map[string]any{ @@ -397,13 +401,13 @@ func (i *ChartLoader) loadConstellationServicesHelper(config *config.Config, mas case cloudprovider.Azure: joinServiceVals, ok := values["join-service"].(map[string]any) if !ok { - return nil, nil, errors.New("invalid join-service values") + return nil, errors.New("invalid join-service values") } joinServiceVals["enforceIdKeyDigest"] = config.EnforcesIDKeyDigest() ccmVals, ok := values["ccm"].(map[string]any) if !ok { - return nil, nil, errors.New("invalid ccm values") + return nil, errors.New("invalid ccm values") } ccmVals["Azure"] = map[string]any{ "image": i.ccmImage, @@ -431,7 +435,7 @@ func (i *ChartLoader) loadConstellationServicesHelper(config *config.Config, mas case cloudprovider.GCP: ccmVals, ok := values["ccm"].(map[string]any) if !ok { - return nil, nil, errors.New("invalid ccm values") + return nil, errors.New("invalid ccm values") } ccmVals["GCP"] = map[string]any{ "image": i.ccmImage, @@ -460,7 +464,7 @@ func (i *ChartLoader) loadConstellationServicesHelper(config *config.Config, mas case cloudprovider.AWS: ccmVals, ok := values["ccm"].(map[string]any) if !ok { - return nil, nil, errors.New("invalid ccm values") + return nil, errors.New("invalid ccm values") } ccmVals["AWS"] = map[string]any{ "image": i.ccmImage, @@ -470,7 +474,7 @@ func (i *ChartLoader) loadConstellationServicesHelper(config *config.Config, mas "AWS": true, } } - return chart, values, nil + return values, nil } // marshalChart takes a Chart object, packages it to a temporary file and returns the content of that file. diff --git a/cli/internal/helm/loader_test.go b/cli/internal/helm/loader_test.go index 5904a1de5..b23566948 100644 --- a/cli/internal/helm/loader_test.go +++ b/cli/internal/helm/loader_test.go @@ -98,7 +98,9 @@ func TestConstellationServices(t *testing.T) { konnectivityImage: "konnectivityImage", gcpGuestAgentImage: "gcpGuestAgentImage", } - chart, values, err := chartLoader.loadConstellationServicesHelper(tc.config, []byte("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"), []byte("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa")) + chart, err := loadChartsDir(helmFS, conServicesPath) + require.NoError(err) + values, err := chartLoader.loadConstellationServicesValues(tc.config, []byte("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"), []byte("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa")) require.NoError(err) options := chartutil.ReleaseOptions{ @@ -164,7 +166,9 @@ func TestOperators(t *testing.T) { constellationOperatorImage: "constellationOperatorImage", nodeMaintenanceOperatorImage: "nodeMaintenanceOperatorImage", } - chart, vals, err := chartLoader.loadOperatorsHelper(tc.csp) + chart, err := loadChartsDir(helmFS, conOperatorsPath) + require.NoError(err) + vals, err := chartLoader.loadOperatorsValues(tc.csp) require.NoError(err) options := chartutil.ReleaseOptions{ diff --git a/internal/deploy/helm/helm.go b/internal/deploy/helm/helm.go index 332ce60a8..8355bd658 100644 --- a/internal/deploy/helm/helm.go +++ b/internal/deploy/helm/helm.go @@ -21,3 +21,25 @@ type Releases struct { Operators Release ConstellationServices Release } + +// MergeMaps returns a new map that is the merger of it's inputs. +// Key colissions are resolved by taking the value of the second argument (map b). +// Taken from: https://github.com/helm/helm/blob/dbc6d8e20fe1d58d50e6ed30f09a04a77e4c68db/pkg/cli/values/options.go#L91-L108. +func MergeMaps(a, b map[string]any) map[string]any { + out := make(map[string]any, len(a)) + for k, v := range a { + out[k] = v + } + for k, v := range b { + if v, ok := v.(map[string]any); ok { + if bv, ok := out[k]; ok { + if bv, ok := bv.(map[string]any); ok { + out[k] = MergeMaps(bv, v) + continue + } + } + } + out[k] = v + } + return out +} diff --git a/internal/deploy/helm/helm_test.go b/internal/deploy/helm/helm_test.go new file mode 100644 index 000000000..78576d5a5 --- /dev/null +++ b/internal/deploy/helm/helm_test.go @@ -0,0 +1,109 @@ +/* +Copyright (c) Edgeless Systems GmbH + +SPDX-License-Identifier: AGPL-3.0-only +*/ + +package helm + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestMergeMaps(t *testing.T) { + testCases := map[string]struct { + vals map[string]any + extraVals map[string]any + expected map[string]any + }{ + "equal": { + vals: map[string]any{ + "join-service": map[string]any{ + "key1": "foo", + "key2": "bar", + }, + }, + extraVals: map[string]any{ + "join-service": map[string]any{ + "extraKey1": "extraFoo", + "extraKey2": "extraBar", + }, + }, + expected: map[string]any{ + "join-service": map[string]any{ + "key1": "foo", + "key2": "bar", + "extraKey1": "extraFoo", + "extraKey2": "extraBar", + }, + }, + }, + "missing join-service extraVals": { + vals: map[string]any{ + "join-service": map[string]any{ + "key1": "foo", + "key2": "bar", + }, + }, + extraVals: map[string]any{ + "extraKey1": "extraFoo", + "extraKey2": "extraBar", + }, + expected: map[string]any{ + "join-service": map[string]any{ + "key1": "foo", + "key2": "bar", + }, + "extraKey1": "extraFoo", + "extraKey2": "extraBar", + }, + }, + "missing join-service vals": { + vals: map[string]any{ + "key1": "foo", + "key2": "bar", + }, + extraVals: map[string]any{ + "join-service": map[string]any{ + "extraKey1": "extraFoo", + "extraKey2": "extraBar", + }, + }, + expected: map[string]any{ + "key1": "foo", + "key2": "bar", + "join-service": map[string]any{ + "extraKey1": "extraFoo", + "extraKey2": "extraBar", + }, + }, + }, + "key collision": { + vals: map[string]any{ + "join-service": map[string]any{ + "key1": "foo", + }, + }, + extraVals: map[string]any{ + "join-service": map[string]any{ + "key1": "bar", + }, + }, + expected: map[string]any{ + "join-service": map[string]any{ + "key1": "bar", + }, + }, + }, + } + + for name, tc := range testCases { + t.Run(name, func(t *testing.T) { + assert := assert.New(t) + newVals := MergeMaps(tc.vals, tc.extraVals) + assert.Equal(tc.expected, newVals) + }) + } +}