Git-Mirrors/constellation
1
0
Fork 0
mirror of https://github.com/edgelesssys/constellation.git synced 2025-02-04 09:05:31 -05:00
Code Issues Packages Projects Releases Wiki Activity

Microservice upgrades (#729)

Browse Source 
Run with: constellation upgrade execute --helm.
This will only upgrade the helm charts. No config is needed.

Upgrades are implemented via helm's upgrade action, i.e. they
automatically roll back if something goes wrong. Releases could 
still be managed via helm, even after an upgrade with constellation
has been done.

Currently not user facing as CRD/CR backups are still in progress.
These backups should be automatically created and saved to the 
user's disk as updates may delete CRs. This happens implicitly 
through CRD upgrades, which are part of microservice upgrades.
This commit is contained in:
Otto Bittner 2022-12-19 16:52:15 +01:00 committed by GitHub
parent 990cae58a5
commit efcd0337b4
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
13 changed files with 498 additions and 166 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

25
bootstrapper/internal/helm/client.go
View File

@ -61,7 +61,7 @@ func (h *Client) InstallConstellationServices(ctx context.Context, release helm.
h.ReleaseName = release.ReleaseName
h.Wait = release.Wait
mergedVals := mergeMaps(release.Values, extraVals)
mergedVals := helm.MergeMaps(release.Values, extraVals)
if err := h.install(ctx, release.Chart, mergedVals); err != nil {
return err
@ -87,7 +87,7 @@ func (h *Client) InstallOperators(ctx context.Context, release helm.Release, ext
h.ReleaseName = release.ReleaseName
h.Wait = release.Wait
mergedVals := mergeMaps(release.Values, extraVals)
mergedVals := helm.MergeMaps(release.Values, extraVals)
if err := h.install(ctx, release.Chart, mergedVals); err != nil {
return err
@ -190,24 +190,3 @@ func (h *Client) install(ctx context.Context, chartRaw []byte, values map[string
}
return nil
}
// mergeMaps returns a new map that is the merger of it's inputs.
// Taken from: https://github.com/helm/helm/blob/dbc6d8e20fe1d58d50e6ed30f09a04a77e4c68db/pkg/cli/values/options.go#L91-L108.
func mergeMaps(a, b map[string]any) map[string]any {
out := make(map[string]any, len(a))
for k, v := range a {
out[k] = v
}
for k, v := range b {
if v, ok := v.(map[string]any); ok {
if bv, ok := out[k]; ok {
if bv, ok := bv.(map[string]any); ok {
out[k] = mergeMaps(bv, v)
continue
}
}
}
out[k] = v
}
return out
}

85
bootstrapper/internal/helm/client_test.go
View File

@ -5,88 +5,3 @@ SPDX-License-Identifier: AGPL-3.0-only
*/
package helm
import (
"testing"
"github.com/stretchr/testify/assert"
)
func TestMe(t *testing.T) {
testCases := map[string]struct {
vals map[string]any
extraVals map[string]any
expected map[string]any
}{
"equal": {
vals: map[string]any{
"join-service": map[string]any{
"key1": "foo",
"key2": "bar",
},
},
extraVals: map[string]any{
"join-service": map[string]any{
"extraKey1": "extraFoo",
"extraKey2": "extraBar",
},
},
expected: map[string]any{
"join-service": map[string]any{
"key1": "foo",
"key2": "bar",
"extraKey1": "extraFoo",
"extraKey2": "extraBar",
},
},
},
"missing join-service extraVals": {
vals: map[string]any{
"join-service": map[string]any{
"key1": "foo",
"key2": "bar",
},
},
extraVals: map[string]any{
"extraKey1": "extraFoo",
"extraKey2": "extraBar",
},
expected: map[string]any{
"join-service": map[string]any{
"key1": "foo",
"key2": "bar",
},
"extraKey1": "extraFoo",
"extraKey2": "extraBar",
},
},
"missing join-service vals": {
vals: map[string]any{
"key1": "foo",
"key2": "bar",
},
extraVals: map[string]any{
"join-service": map[string]any{
"extraKey1": "extraFoo",
"extraKey2": "extraBar",
},
},
expected: map[string]any{
"key1": "foo",
"key2": "bar",
"join-service": map[string]any{
"extraKey1": "extraFoo",
"extraKey2": "extraBar",
},
},
},
}
for name, tc := range testCases {
t.Run(name, func(t *testing.T) {
assert := assert.New(t)
newVals := mergeMaps(tc.vals, tc.extraVals)
assert.Equal(tc.expected, newVals)
})
}
}

23
cli/internal/cloudcmd/upgrade.go
View File

@ -12,11 +12,14 @@ import (
"errors"
"fmt"
"io"
"time"
"github.com/edgelesssys/constellation/v2/cli/internal/helm"
"github.com/edgelesssys/constellation/v2/internal/attestation/measurements"
"github.com/edgelesssys/constellation/v2/internal/config"
"github.com/edgelesssys/constellation/v2/internal/constants"
corev1 "k8s.io/api/core/v1"
apiextensionsclient "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/runtime/schema"
@ -35,7 +38,7 @@ type Upgrader struct {
}
// NewUpgrader returns a new Upgrader.
func NewUpgrader(outWriter io.Writer) (*Upgrader, error) {
func NewUpgrader(outWriter io.Writer, log debugLog) (*Upgrader, error) {
kubeConfig, err := clientcmd.BuildConfigFromFlags("", constants.AdminConfFilename)
if err != nil {
return nil, fmt.Errorf("building kubernetes config: %w", err)
@ -52,7 +55,12 @@ func NewUpgrader(outWriter io.Writer) (*Upgrader, error) {
return nil, fmt.Errorf("setting up custom resource client: %w", err)
}
helmClient, err := helm.NewClient(constants.AdminConfFilename, constants.HelmNamespace)
client, err := apiextensionsclient.NewForConfig(kubeConfig)
if err != nil {
return nil, err
}
helmClient, err := helm.NewClient(constants.AdminConfFilename, constants.HelmNamespace, client, log)
if err != nil {
return nil, fmt.Errorf("setting up helm client: %w", err)
}
@ -105,6 +113,11 @@ func (u *Upgrader) GetCurrentImage(ctx context.Context) (*unstructured.Unstructu
return imageStruct, imageVersion, nil
}
// UpgradeHelmServices upgrade helm services.
func (u *Upgrader) UpgradeHelmServices(ctx context.Context, config *config.Config, timeout time.Duration) error {
return u.helmClient.Upgrade(ctx, config, timeout)
}
// CurrentHelmVersion returns the version of the currently installed helm release.
func (u *Upgrader) CurrentHelmVersion(release string) (string, error) {
return u.helmClient.CurrentVersion(release)
@ -232,4 +245,10 @@ func (u *stableClient) kubernetesVersion() (string, error) {
type helmInterface interface {
CurrentVersion(release string) (string, error)
Upgrade(ctx context.Context, config *config.Config, timeout time.Duration) error
}
type debugLog interface {
Debugf(format string, args ...any)
Sync()
}

35
cli/internal/cmd/upgradeexecute.go
View File

@ -8,6 +8,8 @@ package cmd
import (
"context"
"fmt"
"time"
"github.com/edgelesssys/constellation/v2/cli/internal/cloudcmd"
"github.com/edgelesssys/constellation/v2/internal/attestation/measurements"
@ -27,13 +29,28 @@ func newUpgradeExecuteCmd() *cobra.Command {
RunE: runUpgradeExecute,
}
cmd.Flags().Bool("helm", false, "Execute helm upgrade. This feature is still in development an may change without anounncement. Upgrades all helm charts deployed during constellation-init.")
cmd.Flags().Duration("timeout", 3*time.Minute, "Change helm upgrade timeout. This feature is still in development an may change without anounncement. Might be useful for slow connections or big clusters.")
if err := cmd.Flags().MarkHidden("helm"); err != nil {
panic(err)
}
if err := cmd.Flags().MarkHidden("timeout"); err != nil {
panic(err)
}
return cmd
}
func runUpgradeExecute(cmd *cobra.Command, args []string) error {
log, err := newCLILogger(cmd)
if err != nil {
return fmt.Errorf("creating logger: %w", err)
}
defer log.Sync()
fileHandler := file.NewHandler(afero.NewOsFs())
imageFetcher := image.New()
upgrader, err := cloudcmd.NewUpgrader(cmd.OutOrStdout())
upgrader, err := cloudcmd.NewUpgrader(cmd.OutOrStdout(), log)
if err != nil {
return err
}
@ -51,6 +68,21 @@ func upgradeExecute(cmd *cobra.Command, imageFetcher imageFetcher, upgrader clou
return displayConfigValidationErrors(cmd.ErrOrStderr(), err)
}
helm, err := cmd.Flags().GetBool("helm")
if err != nil {
return err
}
if helm {
timeout, err := cmd.Flags().GetDuration("timeout")
if err != nil {
return err
}
if err := upgrader.UpgradeHelmServices(cmd.Context(), conf, timeout); err != nil {
return fmt.Errorf("upgrading helm: %w", err)
}
return nil
}
// TODO: validate upgrade config? Should be basic things like checking image is not an empty string
// More sophisticated validation, like making sure we don't downgrade the cluster, should be done by `constellation upgrade plan`
@ -66,6 +98,7 @@ func upgradeExecute(cmd *cobra.Command, imageFetcher imageFetcher, upgrader clou
type cloudUpgrader interface {
Upgrade(ctx context.Context, imageReference, imageVersion string, measurements measurements.M) error
UpgradeHelmServices(ctx context.Context, config *config.Config, timeout time.Duration) error
}
type imageFetcher interface {

8
cli/internal/cmd/upgradeexecute_test.go
View File

@ -10,6 +10,7 @@ import (
"context"
"errors"
"testing"
"time"
"github.com/edgelesssys/constellation/v2/internal/attestation/measurements"
"github.com/edgelesssys/constellation/v2/internal/cloud/cloudprovider"
@ -66,13 +67,18 @@ func TestUpgradeExecute(t *testing.T) {
}
type stubUpgrader struct {
err error
err error
helmErr error
}
func (u stubUpgrader) Upgrade(context.Context, string, string, measurements.M) error {
return u.err
}
func (u stubUpgrader) UpgradeHelmServices(ctx context.Context, config *config.Config, timeout time.Duration) error {
return u.helmErr
}
type stubImageFetcher struct {
reference string
fetchReferenceErr error

8
cli/internal/cmd/upgradeplan.go
View File

@ -44,12 +44,18 @@ func newUpgradePlanCmd() *cobra.Command {
}
func runUpgradePlan(cmd *cobra.Command, args []string) error {
log, err := newCLILogger(cmd)
if err != nil {
return fmt.Errorf("creating logger: %w", err)
}
defer log.Sync()
fileHandler := file.NewHandler(afero.NewOsFs())
flags, err := parseUpgradePlanFlags(cmd)
if err != nil {
return err
}
planner, err := cloudcmd.NewUpgrader(cmd.OutOrStdout())
planner, err := cloudcmd.NewUpgrader(cmd.OutOrStdout(), log)
if err != nil {
return err
}

23
cli/internal/helm/README.md Normal file
View File

@ -0,0 +1,23 @@
# Chart upgrades
All services that are installed via helm-install are upgraded via helm-upgrade.
Two aspects are not full covered by running helm-upgrade: CRDs and values.
While helm-install can install CRDs if they are contained in a chart's `crds` folder, upgrade won't change any installed CRDs.
Furthermore, new values introduced with a new version of a chart will not be installed into the cluster if the `--reuse-values` flag is set.
Nevertheless, we have to rely on the values already present in the cluster because some of the values are set by the bootstrapper during installation.
Because upgrades should be a CLI-only operation and we want to avoid the behaviour of `--reuse-values`, we fetch the cluster values and merge them with any new values.
Here is how we manage CRD upgrades for each chart.
## Cilium
- CRDs are updated by cilium-operator.
## cert-manager
- installCRDs flag is set during upgrade. This flag is managed by cert-manager. cert-manager is in charge of correctly upgrading the CRDs.
- WARNING: upgrading cert-manager might break other installations of cert-manager in the cluster, if those other installation are not on the same version as the Constellation-manager installation. This is due to the cluster-wide CRDs.
## Operators
- Manually update CRDs before upgrading the chart. Update by running applying the CRDs found in the `operators/crds/` folder.
## Constellation-services
- There currently are no CRDs in this chart.

174
cli/internal/helm/client.go
View File

@ -7,28 +7,44 @@ SPDX-License-Identifier: AGPL-3.0-only
package helm
import (
"context"
"fmt"
"strings"
"time"
"github.com/edgelesssys/constellation/v2/internal/config"
"github.com/edgelesssys/constellation/v2/internal/constants"
"github.com/edgelesssys/constellation/v2/internal/deploy/helm"
"github.com/pkg/errors"
"helm.sh/helm/v3/pkg/action"
"helm.sh/helm/v3/pkg/chart"
"helm.sh/helm/v3/pkg/cli"
v1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
apiextensionsclient "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/serializer"
"k8s.io/client-go/kubernetes/scheme"
)
// Client handles interaction with helm.
type Client struct {
config *action.Configuration
config *action.Configuration
crdClient *apiextensionsclient.Clientset
log debugLog
}
// NewClient returns a new initializes client for the namespace Client.
func NewClient(kubeConfigPath, helmNamespace string) (*Client, error) {
func NewClient(kubeConfigPath, helmNamespace string, client *apiextensionsclient.Clientset, log debugLog) (*Client, error) {
settings := cli.New()
settings.KubeConfig = kubeConfigPath // constants.AdminConfFilename
actionConfig := &action.Configuration{}
if err := actionConfig.Init(settings.RESTClientGetter(), helmNamespace, "secret", nil); err != nil {
if err := actionConfig.Init(settings.RESTClientGetter(), helmNamespace, "secret", log.Debugf); err != nil {
return nil, fmt.Errorf("initializing config: %w", err)
}
return &Client{config: actionConfig}, nil
return &Client{config: actionConfig, crdClient: client, log: log}, nil
}
// CurrentVersion returns the version of the currently installed helm release.
@ -53,3 +69,153 @@ func (c *Client) CurrentVersion(release string) (string, error) {
return rel[0].Chart.Metadata.Version, nil
}
// Upgrade runs a helm-upgrade on all deployments that are managed via Helm.
// If the CLI receives an interrupt signal it will cancel the context.
// Canceling the context will prompt helm to abort and roll back the ongoing upgrade.
func (c *Client) Upgrade(ctx context.Context, config *config.Config, timeout time.Duration) error {
action := action.NewUpgrade(c.config)
action.Atomic = true
action.Namespace = constants.HelmNamespace
action.ReuseValues = false
action.Timeout = timeout
ciliumChart, err := loadChartsDir(helmFS, ciliumPath)
if err != nil {
return fmt.Errorf("loading cilium: %w", err)
}
certManagerChart, err := loadChartsDir(helmFS, certManagerPath)
if err != nil {
return fmt.Errorf("loading cert-manager: %w", err)
}
conOperatorChart, err := loadChartsDir(helmFS, conOperatorsPath)
if err != nil {
return fmt.Errorf("loading operators: %w", err)
}
conServicesChart, err := loadChartsDir(helmFS, conServicesPath)
if err != nil {
return fmt.Errorf("loading constellation-services chart: %w", err)
}
values, err := c.prepareValues(ciliumChart, ciliumReleaseName)
if err != nil {
return err
}
if _, err := action.RunWithContext(ctx, ciliumReleaseName, ciliumChart, values); err != nil {
return fmt.Errorf("upgrading cilium: %w", err)
}
values, err = c.prepareValues(certManagerChart, certManagerReleaseName)
if err != nil {
return err
}
if _, err := action.RunWithContext(ctx, certManagerReleaseName, certManagerChart, values); err != nil {
return fmt.Errorf("upgrading cert-manager: %w", err)
}
err = c.updateOperatorCRDs(ctx, conOperatorChart)
if err != nil {
return fmt.Errorf("updating operator CRDs: %w", err)
}
values, err = c.prepareValues(conOperatorChart, conOperatorsReleaseName)
if err != nil {
return err
}
if _, err := action.RunWithContext(ctx, conOperatorsReleaseName, conOperatorChart, values); err != nil {
return fmt.Errorf("upgrading services: %w", err)
}
values, err = c.prepareValues(conServicesChart, conServicesReleaseName)
if err != nil {
return err
}
if _, err := action.RunWithContext(ctx, conServicesReleaseName, conServicesChart, values); err != nil {
return fmt.Errorf("upgrading operators: %w", err)
}
return nil
}
// prepareCertManagerValues returns a values map as required for helm-upgrade.
// It imitates the behaviour of helm's reuse-values flag by fetching the current values from the cluster
// and merging the fetched values with the locally found values.
// This is done to ensure that new values (from upgrades of the local files) end up in the cluster.
// reuse-values does not ensure this.
func (c *Client) prepareValues(chart *chart.Chart, releaseName string) (map[string]any, error) {
// Ensure installCRDs is set for cert-manager chart.
if releaseName == certManagerReleaseName {
chart.Values["installCRDs"] = true
}
values, err := c.GetValues(releaseName)
if err != nil {
return nil, fmt.Errorf("getting values: %w", err)
}
return helm.MergeMaps(chart.Values, values), nil
}
// GetValues queries the cluster for the values of the given release.
func (c *Client) GetValues(release string) (map[string]any, error) {
client := action.NewGetValues(c.config)
// Version corresponds to the releases revision. Specifying a Version <= 0 yields the latest release.
client.Version = 0
values, err := client.Run(release)
if err != nil {
return nil, fmt.Errorf("getting values for %s: %w", release, err)
}
return values, nil
}
// ApplyCRD updates the given CRD by parsing it, querying it's version from the cluster and finally updating it.
func (c *Client) ApplyCRD(ctx context.Context, rawCRD []byte) error {
crd, err := parseCRD(rawCRD)
if err != nil {
return fmt.Errorf("parsing crds: %w", err)
}
clusterCRD, err := c.crdClient.ApiextensionsV1().CustomResourceDefinitions().Get(ctx, crd.Name, metav1.GetOptions{})
if err != nil {
return fmt.Errorf("getting crd: %w", err)
}
crd.ResourceVersion = clusterCRD.ResourceVersion
_, err = c.crdClient.ApiextensionsV1().CustomResourceDefinitions().Update(ctx, crd, metav1.UpdateOptions{})
return err
}
// parseCRD takes a byte slice of data and tries to create a CustomResourceDefinition object from it.
func parseCRD(crdString []byte) (*v1.CustomResourceDefinition, error) {
sch := runtime.NewScheme()
_ = scheme.AddToScheme(sch)
_ = v1.AddToScheme(sch)
obj, groupVersionKind, err := serializer.NewCodecFactory(sch).UniversalDeserializer().Decode(crdString, nil, nil)
if err != nil {
return nil, fmt.Errorf("decoding crd: %w", err)
}
if groupVersionKind.Kind == "CustomResourceDefinition" {
return obj.(*v1.CustomResourceDefinition), nil
}
return nil, errors.New("parsed []byte, but did not find a CRD")
}
// updateOperatorCRDs walks through the dependencies of the given chart and applies
// the files in the dependencie's 'crds' folder.
// This function is NOT recursive!
func (c *Client) updateOperatorCRDs(ctx context.Context, chart *chart.Chart) error {
for _, dep := range chart.Dependencies() {
for _, crdFile := range dep.Files {
if strings.HasPrefix(crdFile.Name, "crds/") {
c.log.Debugf("updating crd: %s", crdFile.Name)
err := c.ApplyCRD(ctx, crdFile.Data)
if err != nil {
return err
}
}
}
}
return nil
}
type debugLog interface {
Debugf(format string, args ...any)
Sync()
}

46
cli/internal/helm/client_test.go Normal file
View File

@ -0,0 +1,46 @@
/*
Copyright (c) Edgeless Systems GmbH
SPDX-License-Identifier: AGPL-3.0-only
*/
package helm
import (
"testing"
"github.com/stretchr/testify/assert"
)
func TestParseCRDs(t *testing.T) {
testCases := map[string]struct {
data string
wantErr bool
}{
"success": {
data: "apiVersion: apiextensions.k8s.io/v1\nkind: CustomResourceDefinition\nmetadata:\n name: nodeimages.update.edgeless.systems\nspec:\n group: update.edgeless.systems\n names:\n kind: NodeImage\n",
wantErr: false,
},
"wrong kind": {
data: "apiVersion: v1\nkind: Secret\ntype: Opaque\nmetadata:\n name: supersecret\n namespace: testNamespace\ndata:\n data: YWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWE=\n",
wantErr: true,
},
"decoding error": {
data: "asdf",
wantErr: true,
},
}
for name, tc := range testCases {
t.Run(name, func(t *testing.T) {
assert := assert.New(t)
_, err := parseCRD([]byte(tc.data))
if tc.wantErr {
assert.Error(err)
return
}
assert.NoError(err)
})
}
}

98
cli/internal/helm/loader.go
View File

@ -37,6 +37,18 @@ import (
//go:embed all:charts/*
var helmFS embed.FS
const (
ciliumReleaseName = "cilium"
conServicesReleaseName = "constellation-services"
conOperatorsReleaseName = "constellation-operators"
certManagerReleaseName = "cert-manager"
conServicesPath = "charts/edgeless/constellation-services"
conOperatorsPath = "charts/edgeless/operators"
certManagerPath = "charts/cert-manager"
ciliumPath = "charts/cilium"
)
// ChartLoader loads embedded helm charts.
type ChartLoader struct {
joinServiceImage string
@ -109,7 +121,11 @@ func (i *ChartLoader) Load(config *config.Config, conformanceMode bool, masterSe
}
func (i *ChartLoader) loadCilium(csp cloudprovider.Provider, conformanceMode bool) (helm.Release, error) {
chart, values, err := i.loadCiliumHelper(csp, conformanceMode)
chart, err := loadChartsDir(helmFS, ciliumPath)
if err != nil {
return helm.Release{}, fmt.Errorf("loading cilium chart: %w", err)
}
values, err := i.loadCiliumValues(csp, conformanceMode)
if err != nil {
return helm.Release{}, err
}
@ -119,17 +135,12 @@ func (i *ChartLoader) loadCilium(csp cloudprovider.Provider, conformanceMode boo
return helm.Release{}, fmt.Errorf("packaging cilium chart: %w", err)
}
return helm.Release{Chart: chartRaw, Values: values, ReleaseName: "cilium", Wait: false}, nil
return helm.Release{Chart: chartRaw, Values: values, ReleaseName: ciliumReleaseName, Wait: false}, nil
}
// loadCiliumHelper is used to separate the marshalling step from the loading step.
// This reduces the time unit tests take to execute.
func (i *ChartLoader) loadCiliumHelper(csp cloudprovider.Provider, conformanceMode bool) (*chart.Chart, map[string]any, error) {
chart, err := loadChartsDir(helmFS, "charts/cilium")
if err != nil {
return nil, nil, fmt.Errorf("loading cilium chart: %w", err)
}
func (i *ChartLoader) loadCiliumValues(csp cloudprovider.Provider, conformanceMode bool) (map[string]any, error) {
var values map[string]any
switch csp {
case cloudprovider.AWS:
@ -141,7 +152,7 @@ func (i *ChartLoader) loadCiliumHelper(csp cloudprovider.Provider, conformanceMo
case cloudprovider.QEMU:
values = qemuVals
default:
return nil, nil, fmt.Errorf("unknown csp: %s", csp)
return nil, fmt.Errorf("unknown csp: %s", csp)
}
if conformanceMode {
values["kubeProxyReplacementHealthzBindAddr"] = ""
@ -152,32 +163,28 @@ func (i *ChartLoader) loadCiliumHelper(csp cloudprovider.Provider, conformanceMo
}
}
return chart, values, nil
return values, nil
}
func (i *ChartLoader) loadCertManager() (helm.Release, error) {
chart, values, err := i.loadCertManagerHelper()
chart, err := loadChartsDir(helmFS, certManagerPath)
if err != nil {
return helm.Release{}, err
return helm.Release{}, fmt.Errorf("loading cert-manager chart: %w", err)
}
values := i.loadCertManagerValues()
chartRaw, err := i.marshalChart(chart)
if err != nil {
return helm.Release{}, fmt.Errorf("packaging cert-manager chart: %w", err)
}
return helm.Release{Chart: chartRaw, Values: values, ReleaseName: "cert-manager", Wait: false}, nil
return helm.Release{Chart: chartRaw, Values: values, ReleaseName: certManagerReleaseName, Wait: false}, nil
}
// loadCertManagerHelper is used to separate the marshalling step from the loading step.
// This reduces the time unit tests take to execute.
func (i *ChartLoader) loadCertManagerHelper() (*chart.Chart, map[string]any, error) {
chart, err := loadChartsDir(helmFS, "charts/cert-manager")
if err != nil {
return nil, nil, fmt.Errorf("loading cert-manager chart: %w", err)
}
values := map[string]any{
func (i *ChartLoader) loadCertManagerValues() map[string]any {
return map[string]any{
"installCRDs": true,
"prometheus": map[string]any{
"enabled": false,
@ -241,11 +248,14 @@ func (i *ChartLoader) loadCertManagerHelper() (*chart.Chart, map[string]any, err
},
},
}
return chart, values, nil
}
func (i *ChartLoader) loadOperators(csp cloudprovider.Provider) (helm.Release, error) {
chart, values, err := i.loadOperatorsHelper(csp)
chart, err := loadChartsDir(helmFS, conOperatorsPath)
if err != nil {
return helm.Release{}, fmt.Errorf("loading operators chart: %w", err)
}
values, err := i.loadOperatorsValues(csp)
if err != nil {
return helm.Release{}, err
}
@ -255,17 +265,12 @@ func (i *ChartLoader) loadOperators(csp cloudprovider.Provider) (helm.Release, e
return helm.Release{}, fmt.Errorf("packaging operators chart: %w", err)
}
return helm.Release{Chart: chartRaw, Values: values, ReleaseName: "con-operators", Wait: false}, nil
return helm.Release{Chart: chartRaw, Values: values, ReleaseName: conOperatorsReleaseName, Wait: false}, nil
}
// loadOperatorsHelper is used to separate the marshalling step from the loading step.
// This reduces the time unit tests take to execute.
func (i *ChartLoader) loadOperatorsHelper(csp cloudprovider.Provider) (*chart.Chart, map[string]any, error) {
chart, err := loadChartsDir(helmFS, "charts/edgeless/operators")
if err != nil {
return nil, nil, fmt.Errorf("loading operators chart: %w", err)
}
func (i *ChartLoader) loadOperatorsValues(csp cloudprovider.Provider) (map[string]any, error) {
values := map[string]any{
"constellation-operator": map[string]any{
"controllerManager": map[string]any{
@ -286,7 +291,7 @@ func (i *ChartLoader) loadOperatorsHelper(csp cloudprovider.Provider) (*chart.Ch
case cloudprovider.Azure:
conOpVals, ok := values["constellation-operator"].(map[string]any)
if !ok {
return nil, nil, errors.New("invalid constellation-operator values")
return nil, errors.New("invalid constellation-operator values")
}
conOpVals["csp"] = "Azure"
@ -296,7 +301,7 @@ func (i *ChartLoader) loadOperatorsHelper(csp cloudprovider.Provider) (*chart.Ch
case cloudprovider.GCP:
conOpVals, ok := values["constellation-operator"].(map[string]any)
if !ok {
return nil, nil, errors.New("invalid constellation-operator values")
return nil, errors.New("invalid constellation-operator values")
}
conOpVals["csp"] = "GCP"
@ -306,7 +311,7 @@ func (i *ChartLoader) loadOperatorsHelper(csp cloudprovider.Provider) (*chart.Ch
case cloudprovider.QEMU:
conOpVals, ok := values["constellation-operator"].(map[string]any)
if !ok {
return nil, nil, errors.New("invalid constellation-operator values")
return nil, errors.New("invalid constellation-operator values")
}
conOpVals["csp"] = "QEMU"
@ -316,7 +321,7 @@ func (i *ChartLoader) loadOperatorsHelper(csp cloudprovider.Provider) (*chart.Ch
case cloudprovider.AWS:
conOpVals, ok := values["constellation-operator"].(map[string]any)
if !ok {
return nil, nil, errors.New("invalid constellation-operator values")
return nil, errors.New("invalid constellation-operator values")
}
conOpVals["csp"] = "AWS"
@ -325,13 +330,17 @@ func (i *ChartLoader) loadOperatorsHelper(csp cloudprovider.Provider) (*chart.Ch
}
}
return chart, values, nil
return values, nil
}
// loadConstellationServices loads the constellation-services chart from the embed.FS,
// marshals it into a helm-package .tgz and sets the values that can be set in the CLI.
func (i *ChartLoader) loadConstellationServices(config *config.Config, masterSecret, salt []byte) (helm.Release, error) {
chart, values, err := i.loadConstellationServicesHelper(config, masterSecret, salt)
chart, err := loadChartsDir(helmFS, conServicesPath)
if err != nil {
return helm.Release{}, fmt.Errorf("loading constellation-services chart: %w", err)
}
values, err := i.loadConstellationServicesValues(config, masterSecret, salt)
if err != nil {
return helm.Release{}, err
}
@ -341,17 +350,12 @@ func (i *ChartLoader) loadConstellationServices(config *config.Config, masterSec
return helm.Release{}, fmt.Errorf("packaging constellation-services chart: %w", err)
}
return helm.Release{Chart: chartRaw, Values: values, ReleaseName: "constellation-services", Wait: false}, nil
return helm.Release{Chart: chartRaw, Values: values, ReleaseName: conServicesReleaseName, Wait: false}, nil
}
// loadConstellationServicesHelper is used to separate the marshalling step from the loading step.
// This reduces the time unit tests take to execute.
func (i *ChartLoader) loadConstellationServicesHelper(config *config.Config, masterSecret, salt []byte) (*chart.Chart, map[string]any, error) {
chart, err := loadChartsDir(helmFS, "charts/edgeless/constellation-services")
if err != nil {
return nil, nil, fmt.Errorf("loading constellation-services chart: %w", err)
}
func (i *ChartLoader) loadConstellationServicesValues(config *config.Config, masterSecret, salt []byte) (map[string]any, error) {
csp := config.GetProvider()
values := map[string]any{
"global": map[string]any{
@ -397,13 +401,13 @@ func (i *ChartLoader) loadConstellationServicesHelper(config *config.Config, mas
case cloudprovider.Azure:
joinServiceVals, ok := values["join-service"].(map[string]any)
if !ok {
return nil, nil, errors.New("invalid join-service values")
return nil, errors.New("invalid join-service values")
}
joinServiceVals["enforceIdKeyDigest"] = config.EnforcesIDKeyDigest()
ccmVals, ok := values["ccm"].(map[string]any)
if !ok {
return nil, nil, errors.New("invalid ccm values")
return nil, errors.New("invalid ccm values")
}
ccmVals["Azure"] = map[string]any{
"image": i.ccmImage,
@ -431,7 +435,7 @@ func (i *ChartLoader) loadConstellationServicesHelper(config *config.Config, mas
case cloudprovider.GCP:
ccmVals, ok := values["ccm"].(map[string]any)
if !ok {
return nil, nil, errors.New("invalid ccm values")
return nil, errors.New("invalid ccm values")
}
ccmVals["GCP"] = map[string]any{
"image": i.ccmImage,
@ -460,7 +464,7 @@ func (i *ChartLoader) loadConstellationServicesHelper(config *config.Config, mas
case cloudprovider.AWS:
ccmVals, ok := values["ccm"].(map[string]any)
if !ok {
return nil, nil, errors.New("invalid ccm values")
return nil, errors.New("invalid ccm values")
}
ccmVals["AWS"] = map[string]any{
"image": i.ccmImage,
@ -470,7 +474,7 @@ func (i *ChartLoader) loadConstellationServicesHelper(config *config.Config, mas
"AWS": true,
}
}
return chart, values, nil
return values, nil
}
// marshalChart takes a Chart object, packages it to a temporary file and returns the content of that file.

8
cli/internal/helm/loader_test.go
View File

@ -98,7 +98,9 @@ func TestConstellationServices(t *testing.T) {
konnectivityImage: "konnectivityImage",
gcpGuestAgentImage: "gcpGuestAgentImage",
}
chart, values, err := chartLoader.loadConstellationServicesHelper(tc.config, []byte("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"), []byte("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"))
chart, err := loadChartsDir(helmFS, conServicesPath)
require.NoError(err)
values, err := chartLoader.loadConstellationServicesValues(tc.config, []byte("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"), []byte("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"))
require.NoError(err)
options := chartutil.ReleaseOptions{
@ -164,7 +166,9 @@ func TestOperators(t *testing.T) {
constellationOperatorImage: "constellationOperatorImage",
nodeMaintenanceOperatorImage: "nodeMaintenanceOperatorImage",
}
chart, vals, err := chartLoader.loadOperatorsHelper(tc.csp)
chart, err := loadChartsDir(helmFS, conOperatorsPath)
require.NoError(err)
vals, err := chartLoader.loadOperatorsValues(tc.csp)
require.NoError(err)
options := chartutil.ReleaseOptions{

22
internal/deploy/helm/helm.go
View File

@ -21,3 +21,25 @@ type Releases struct {
Operators Release
ConstellationServices Release
}
// MergeMaps returns a new map that is the merger of it's inputs.
// Key colissions are resolved by taking the value of the second argument (map b).
// Taken from: https://github.com/helm/helm/blob/dbc6d8e20fe1d58d50e6ed30f09a04a77e4c68db/pkg/cli/values/options.go#L91-L108.
func MergeMaps(a, b map[string]any) map[string]any {
out := make(map[string]any, len(a))
for k, v := range a {
out[k] = v
}
for k, v := range b {
if v, ok := v.(map[string]any); ok {
if bv, ok := out[k]; ok {
if bv, ok := bv.(map[string]any); ok {
out[k] = MergeMaps(bv, v)
continue
}
}
}
out[k] = v
}
return out
}

109
internal/deploy/helm/helm_test.go Normal file
View File

@ -0,0 +1,109 @@
/*
Copyright (c) Edgeless Systems GmbH
SPDX-License-Identifier: AGPL-3.0-only
*/
package helm
import (
"testing"
"github.com/stretchr/testify/assert"
)
func TestMergeMaps(t *testing.T) {
testCases := map[string]struct {
vals map[string]any
extraVals map[string]any
expected map[string]any
}{
"equal": {
vals: map[string]any{
"join-service": map[string]any{
"key1": "foo",
"key2": "bar",
},
},
extraVals: map[string]any{
"join-service": map[string]any{
"extraKey1": "extraFoo",
"extraKey2": "extraBar",
},
},
expected: map[string]any{
"join-service": map[string]any{
"key1": "foo",
"key2": "bar",
"extraKey1": "extraFoo",
"extraKey2": "extraBar",
},
},
},
"missing join-service extraVals": {
vals: map[string]any{
"join-service": map[string]any{
"key1": "foo",
"key2": "bar",
},
},
extraVals: map[string]any{
"extraKey1": "extraFoo",
"extraKey2": "extraBar",
},
expected: map[string]any{
"join-service": map[string]any{
"key1": "foo",
"key2": "bar",
},
"extraKey1": "extraFoo",
"extraKey2": "extraBar",
},
},
"missing join-service vals": {
vals: map[string]any{
"key1": "foo",
"key2": "bar",
},
extraVals: map[string]any{
"join-service": map[string]any{
"extraKey1": "extraFoo",
"extraKey2": "extraBar",
},
},
expected: map[string]any{
"key1": "foo",
"key2": "bar",
"join-service": map[string]any{
"extraKey1": "extraFoo",
"extraKey2": "extraBar",
},
},
},
"key collision": {
vals: map[string]any{
"join-service": map[string]any{
"key1": "foo",
},
},
extraVals: map[string]any{
"join-service": map[string]any{
"key1": "bar",
},
},
expected: map[string]any{
"join-service": map[string]any{
"key1": "bar",
},
},
},
}
for name, tc := range testCases {
t.Run(name, func(t *testing.T) {
assert := assert.New(t)
newVals := MergeMaps(tc.vals, tc.extraVals)
assert.Equal(tc.expected, newVals)
})
}
}