Git-Mirrors/constellation
1
0
Fork 0
mirror of https://github.com/edgelesssys/constellation.git synced 2025-08-02 12:06:09 -04:00
Code Issues Projects Releases Packages Wiki Activity

cli: helm install and upgrade unification (#2244)

Browse source
This commit is contained in:
Adrian Stobbe 2023-08-24 16:40:47 +02:00 • committed by GitHub
parent 9e79e2e0a1
commit a03325466c
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
29 changed files with 1140 additions and 1054 deletions
Download patch file Download diff file Expand all files Collapse all files

31
cli/internal/helm/BUILD.bazel
View file

@ -4,17 +4,17 @@ load("//bazel/go:go_test.bzl", "go_test")
go_library(
name = "helm",
srcs = [
"backup.go",
"action.go",
"actionfactory.go",
"ciliumhelper.go",
"helm.go",
"init.go",
"install.go",
"loader.go",
"overrides.go",
"release.go",
"retryaction.go",
"serviceversion.go",
"upgrade.go",
"values.go",
"versionlister.go",
],
embedsrcs = [
"charts/cert-manager/Chart.yaml",
@ -429,22 +429,17 @@ go_library(
"//internal/compatibility",
"//internal/config",
"//internal/constants",
"//internal/file",
"//internal/kms/uri",
"//internal/kubernetes/kubectl",
"//internal/retry",
"//internal/semver",
"//internal/versions",
"@com_github_pkg_errors//:errors",
"@com_github_spf13_afero//:afero",
"@io_k8s_apiextensions_apiserver//pkg/apis/apiextensions/v1:apiextensions",
"@io_k8s_apimachinery//pkg/api/errors",
"@io_k8s_apimachinery//pkg/apis/meta/v1:meta",
"@io_k8s_apimachinery//pkg/apis/meta/v1/unstructured",
"@io_k8s_apimachinery//pkg/runtime/schema",
"@io_k8s_apimachinery//pkg/util/wait",
"@io_k8s_client_go//kubernetes",
"@io_k8s_client_go//tools/clientcmd",
"@io_k8s_sigs_yaml//:yaml",
"@io_k8s_client_go//util/retry",
"@sh_helm_helm//pkg/ignore",
"@sh_helm_helm_v3//pkg/action",
"@sh_helm_helm_v3//pkg/chart",
@ -457,10 +452,8 @@ go_library(
go_test(
name = "helm_test",
srcs = [
"backup_test.go",
"helm_test.go",
"loader_test.go",
"upgrade_test.go",
],
data = glob(["testdata/**"]),
embed = [":helm"],
@ -473,22 +466,16 @@ go_test(
"//internal/cloud/gcpshared",
"//internal/compatibility",
"//internal/config",
"//internal/file",
"//internal/kms/uri",
"//internal/logger",
"//internal/semver",
"//internal/versions",
"@com_github_pkg_errors//:errors",
"@com_github_spf13_afero//:afero",
"@com_github_stretchr_testify//assert",
"@com_github_stretchr_testify//mock",
"@com_github_stretchr_testify//require",
"@io_k8s_apiextensions_apiserver//pkg/apis/apiextensions/v1:apiextensions",
"@io_k8s_apimachinery//pkg/api/errors",
"@io_k8s_apimachinery//pkg/apis/meta/v1/unstructured",
"@io_k8s_apimachinery//pkg/runtime/schema",
"@io_k8s_sigs_yaml//:yaml",
"@sh_helm_helm_v3//pkg/chart",
"@sh_helm_helm_v3//pkg/action",
"@sh_helm_helm_v3//pkg/chartutil",
"@sh_helm_helm_v3//pkg/engine",
"@sh_helm_helm_v3//pkg/release",
],
)

161
cli/internal/helm/action.go Normal file
View file

@ -0,0 +1,161 @@
/*
Copyright (c) Edgeless Systems GmbH
SPDX-License-Identifier: AGPL-3.0-only
*/
package helm
import (
"context"
"fmt"
"time"
"github.com/edgelesssys/constellation/v2/internal/constants"
"helm.sh/helm/v3/pkg/action"
"helm.sh/helm/v3/pkg/cli"
)
const (
// timeout is the maximum time given per helm action.
timeout = 10 * time.Minute
)
type applyAction interface {
Apply(context.Context) error
ReleaseName() string
IsAtomic() bool
}
// newActionConfig creates a new action configuration for helm actions.
func newActionConfig(kubeconfig string, logger debugLog) (*action.Configuration, error) {
settings := cli.New()
settings.KubeConfig = kubeconfig
actionConfig := &action.Configuration{}
if err := actionConfig.Init(settings.RESTClientGetter(), constants.HelmNamespace,
"secret", logger.Debugf); err != nil {
return nil, err
}
return actionConfig, nil
}
func newHelmInstallAction(config *action.Configuration, release Release) *action.Install {
action := action.NewInstall(config)
action.Namespace = constants.HelmNamespace
action.Timeout = timeout
action.ReleaseName = release.ReleaseName
setWaitMode(action, release.WaitMode)
return action
}
func setWaitMode(a *action.Install, waitMode WaitMode) {
switch waitMode {
case WaitModeNone:
a.Wait = false
a.Atomic = false
case WaitModeWait:
a.Wait = true
a.Atomic = false
case WaitModeAtomic:
a.Wait = true
a.Atomic = true
default:
panic(fmt.Errorf("unknown wait mode %q", waitMode))
}
}
// installAction is an action that installs a helm chart.
type installAction struct {
preInstall func(context.Context) error
release Release
helmAction *action.Install
postInstall func(context.Context) error
log debugLog
}
// Apply installs the chart.
func (a *installAction) Apply(ctx context.Context) error {
if a.preInstall != nil {
if err := a.preInstall(ctx); err != nil {
return err
}
}
if err := retryApply(ctx, a, a.log); err != nil {
return err
}
if a.postInstall != nil {
if err := a.postInstall(ctx); err != nil {
return err
}
}
return nil
}
func (a *installAction) apply(ctx context.Context) error {
_, err := a.helmAction.RunWithContext(ctx, a.release.Chart, a.release.Values)
return err
}
// ReleaseName returns the release name.
func (a *installAction) ReleaseName() string {
return a.release.ReleaseName
}
// IsAtomic returns true if the action is atomic.
func (a *installAction) IsAtomic() bool {
return a.helmAction.Atomic
}
func newHelmUpgradeAction(config *action.Configuration) *action.Upgrade {
action := action.NewUpgrade(config)
action.Namespace = constants.HelmNamespace
action.Timeout = timeout
action.ReuseValues = false
action.Atomic = true
return action
}
// upgradeAction is an action that upgrades a helm chart.
type upgradeAction struct {
preUpgrade func(context.Context) error
postUpgrade func(context.Context) error
release Release
helmAction *action.Upgrade
log debugLog
}
// Apply installs the chart.
func (a *upgradeAction) Apply(ctx context.Context) error {
if a.preUpgrade != nil {
if err := a.preUpgrade(ctx); err != nil {
return err
}
}
if err := retryApply(ctx, a, a.log); err != nil {
return err
}
if a.postUpgrade != nil {
if err := a.postUpgrade(ctx); err != nil {
return err
}
}
return nil
}
func (a *upgradeAction) apply(ctx context.Context) error {
_, err := a.helmAction.RunWithContext(ctx, a.release.ReleaseName, a.release.Chart, a.release.Values)
return err
}
// ReleaseName returns the release name.
func (a *upgradeAction) ReleaseName() string {
return a.release.ReleaseName
}
// IsAtomic returns true if the action is atomic.
func (a *upgradeAction) IsAtomic() bool {
return a.helmAction.Atomic
}

186
cli/internal/helm/actionfactory.go Normal file
View file

@ -0,0 +1,186 @@
/*
Copyright (c) Edgeless Systems GmbH
SPDX-License-Identifier: AGPL-3.0-only
*/
package helm
import (
"context"
"errors"
"fmt"
"strings"
"time"
"github.com/edgelesssys/constellation/v2/internal/compatibility"
"github.com/edgelesssys/constellation/v2/internal/constants"
"github.com/edgelesssys/constellation/v2/internal/semver"
"helm.sh/helm/v3/pkg/action"
"helm.sh/helm/v3/pkg/chart"
)
// ErrConfirmationMissing signals that an action requires user confirmation.
var ErrConfirmationMissing = errors.New("action requires user confirmation")
var errReleaseNotFound = errors.New("release not found")
type actionFactory struct {
versionLister releaseVersionLister
cfg *action.Configuration
kubeClient crdClient
cliVersion semver.Semver
log debugLog
}
type crdClient interface {
ApplyCRD(ctx context.Context, rawCRD []byte) error
}
// newActionFactory creates a new action factory for managing helm releases.
func newActionFactory(kubeClient crdClient, lister releaseVersionLister, actionConfig *action.Configuration, cliVersion semver.Semver, log debugLog) *actionFactory {
return &actionFactory{
cliVersion: cliVersion,
versionLister: lister,
cfg: actionConfig,
kubeClient: kubeClient,
log: log,
}
}
// GetActions returns a list of actions to apply the given releases.
func (a actionFactory) GetActions(releases []Release, force, allowDestructive bool) (actions []applyAction, includesUpgrade bool, err error) {
upgradeErrs := []error{}
for _, release := range releases {
err := a.appendNewAction(release, force, allowDestructive, &actions)
var invalidUpgrade *compatibility.InvalidUpgradeError
if errors.As(err, &invalidUpgrade) {
upgradeErrs = append(upgradeErrs, err)
continue
}
if err != nil {
return actions, includesUpgrade, fmt.Errorf("creating action for %s: %w", release.ReleaseName, err)
}
}
for _, action := range actions {
if _, ok := action.(*upgradeAction); ok {
includesUpgrade = true
break
}
}
return actions, includesUpgrade, errors.Join(upgradeErrs...)
}
func (a actionFactory) appendNewAction(release Release, force, allowDestructive bool, actions *[]applyAction) error {
newVersion, err := semver.New(release.Chart.Metadata.Version)
if err != nil {
return fmt.Errorf("parsing chart version: %w", err)
}
currentVersion, err := a.versionLister.currentVersion(release.ReleaseName)
if errors.Is(err, errReleaseNotFound) {
a.log.Debugf("Release %s not found, adding to new releases...", release.ReleaseName)
*actions = append(*actions, a.newInstall(release))
return nil
}
if err != nil {
return fmt.Errorf("getting version for %s: %w", release.ReleaseName, err)
}
a.log.Debugf("Current %s version: %s", release.ReleaseName, currentVersion)
a.log.Debugf("New %s version: %s", release.ReleaseName, newVersion)
// This may break for cert-manager or cilium if we decide to upgrade more than one minor version at a time.
// Leaving it as is since it is not clear to me what kind of sanity check we could do.
if !force {
if err := newVersion.IsUpgradeTo(currentVersion); err != nil {
return fmt.Errorf("invalid upgrade for %s: %w", release.ReleaseName, err)
}
}
// at this point we conclude that the release should be upgraded. check that this CLI supports the upgrade.
if isCLIVersionedRelease(release.ReleaseName) && a.cliVersion.Compare(newVersion) != 0 {
return fmt.Errorf("this CLI only supports microservice version %s for upgrading", a.cliVersion.String())
}
if !allowDestructive &&
release.ReleaseName == certManagerInfo.releaseName {
return ErrConfirmationMissing
}
a.log.Debugf("Upgrading %s from %s to %s", release.ReleaseName, currentVersion, newVersion)
*actions = append(*actions, a.newUpgrade(release))
return nil
}
func (a actionFactory) newInstall(release Release) *installAction {
action := &installAction{helmAction: newHelmInstallAction(a.cfg, release), release: release, log: a.log}
if action.ReleaseName() == ciliumInfo.releaseName {
action.postInstall = func(ctx context.Context) error {
return ciliumPostInstall(ctx, a.log)
}
}
return action
}
func ciliumPostInstall(ctx context.Context, log debugLog) error {
log.Debugf("Waiting for Cilium to become ready")
helper, err := newK8sCiliumHelper(constants.AdminConfFilename)
if err != nil {
return fmt.Errorf("creating Kubernetes client: %w", err)
}
timeToStartWaiting := time.Now()
// TODO(3u13r): Reduce the timeout when we switched the package repository - this is only this high because we once
// saw polling times of ~16 minutes when hitting a slow PoP from Fastly (GitHub's / ghcr.io CDN).
if err := helper.WaitForDS(ctx, "kube-system", "cilium", log); err != nil {
return fmt.Errorf("waiting for Cilium to become healthy: %w", err)
}
timeUntilFinishedWaiting := time.Since(timeToStartWaiting)
log.Debugf("Cilium became healthy after %s", timeUntilFinishedWaiting.String())
log.Debugf("Fix Cilium through restart")
if err := helper.RestartDS("kube-system", "cilium"); err != nil {
return fmt.Errorf("restarting Cilium: %w", err)
}
return nil
}
func (a actionFactory) newUpgrade(release Release) *upgradeAction {
action := &upgradeAction{helmAction: newHelmUpgradeAction(a.cfg), release: release, log: a.log}
if release.ReleaseName == constellationOperatorsInfo.releaseName {
action.preUpgrade = func(ctx context.Context) error {
if err := a.updateCRDs(ctx, release.Chart); err != nil {
return fmt.Errorf("updating operator CRDs: %w", err)
}
return nil
}
}
return action
}
// updateCRDs walks through the dependencies of the given chart and applies
// the files in the dependencie's 'crds' folder.
// This function is NOT recursive!
func (a actionFactory) updateCRDs(ctx context.Context, chart *chart.Chart) error {
for _, dep := range chart.Dependencies() {
for _, crdFile := range dep.Files {
if strings.HasPrefix(crdFile.Name, "crds/") {
a.log.Debugf("Updating crd: %s", crdFile.Name)
err := a.kubeClient.ApplyCRD(ctx, crdFile.Data)
if err != nil {
return err
}
}
}
}
return nil
}
// isCLIVersionedRelease checks if the given release is versioned by the CLI,
// meaning that the version of the Helm release is equal to the version of the CLI that installed it.
func isCLIVersionedRelease(releaseName string) bool {
return releaseName == constellationOperatorsInfo.releaseName ||
releaseName == constellationServicesInfo.releaseName ||
releaseName == csiInfo.releaseName
}
// releaseVersionLister can list the versions of a helm release.
type releaseVersionLister interface {
currentVersion(release string) (semver.Semver, error)
}

107
cli/internal/helm/backup.go
View file

@ -1,107 +0,0 @@
/*
Copyright (c) Edgeless Systems GmbH
SPDX-License-Identifier: AGPL-3.0-only
*/
package helm
import (
"context"
"fmt"
"path/filepath"
apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
k8serrors "k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/runtime/schema"
"sigs.k8s.io/yaml"
)
func (c *UpgradeClient) backupCRDs(ctx context.Context, upgradeDir string) ([]apiextensionsv1.CustomResourceDefinition, error) {
c.log.Debugf("Starting CRD backup")
crds, err := c.kubectl.ListCRDs(ctx)
if err != nil {
return nil, fmt.Errorf("getting CRDs: %w", err)
}
crdBackupFolder := c.crdBackupFolder(upgradeDir)
if err := c.fs.MkdirAll(crdBackupFolder); err != nil {
return nil, fmt.Errorf("creating backup dir: %w", err)
}
for i := range crds {
path := filepath.Join(crdBackupFolder, crds[i].Name+".yaml")
c.log.Debugf("Creating CRD backup: %s", path)
// We have to manually set kind/apiversion because of a long-standing limitation of the API:
// https://github.com/kubernetes/kubernetes/issues/3030#issuecomment-67543738
// The comment states that kind/version are encoded in the type.
// The package holding the CRD type encodes the version.
crds[i].Kind = "CustomResourceDefinition"
crds[i].APIVersion = "apiextensions.k8s.io/v1"
yamlBytes, err := yaml.Marshal(crds[i])
if err != nil {
return nil, err
}
if err := c.fs.Write(path, yamlBytes); err != nil {
return nil, err
}
}
c.log.Debugf("CRD backup complete")
return crds, nil
}
func (c *UpgradeClient) backupCRs(ctx context.Context, crds []apiextensionsv1.CustomResourceDefinition, upgradeID string) error {
c.log.Debugf("Starting CR backup")
for _, crd := range crds {
c.log.Debugf("Creating backup for resource type: %s", crd.Name)
// Iterate over all versions of the CRD
// TODO: Consider iterating over crd.Status.StoredVersions instead
// Currently, we have to ignore not-found errors, because a CRD might define
// a version that is not installed in the cluster.
// With the StoredVersions field, we could only iterate over the installed versions.
for _, version := range crd.Spec.Versions {
c.log.Debugf("Creating backup of CRs for %q at version %q", crd.Name, version.Name)
gvr := schema.GroupVersionResource{Group: crd.Spec.Group, Version: version.Name, Resource: crd.Spec.Names.Plural}
crs, err := c.kubectl.ListCRs(ctx, gvr)
if err != nil {
if !k8serrors.IsNotFound(err) {
return fmt.Errorf("retrieving CR %s: %w", crd.Name, err)
}
c.log.Debugf("No CRs found for %q at version %q, skipping...", crd.Name, version.Name)
continue
}
backupFolder := c.backupFolder(upgradeID)
for _, cr := range crs {
targetFolder := filepath.Join(backupFolder, gvr.Group, gvr.Version, cr.GetNamespace(), cr.GetKind())
if err := c.fs.MkdirAll(targetFolder); err != nil {
return fmt.Errorf("creating resource dir: %w", err)
}
path := filepath.Join(targetFolder, cr.GetName()+".yaml")
yamlBytes, err := yaml.Marshal(cr.Object)
if err != nil {
return err
}
if err := c.fs.Write(path, yamlBytes); err != nil {
return err
}
}
}
c.log.Debugf("Backup for resource type %q complete", crd.Name)
}
c.log.Debugf("CR backup complete")
return nil
}
func (c *UpgradeClient) backupFolder(upgradeDir string) string {
return filepath.Join(upgradeDir, "backups")
}
func (c *UpgradeClient) crdBackupFolder(upgradeDir string) string {
return filepath.Join(c.backupFolder(upgradeDir), "crds")
}

196
cli/internal/helm/backup_test.go
View file

@ -1,196 +0,0 @@
/*
Copyright (c) Edgeless Systems GmbH
SPDX-License-Identifier: AGPL-3.0-only
*/
package helm
import (
"context"
"path/filepath"
"testing"
"github.com/edgelesssys/constellation/v2/internal/file"
"github.com/pkg/errors"
"github.com/spf13/afero"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
k8serrors "k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/runtime/schema"
"sigs.k8s.io/yaml"
)
func TestBackupCRDs(t *testing.T) {
testCases := map[string]struct {
upgradeID string
crd string
expectedFile string
getCRDsError error
wantError bool
}{
"success": {
upgradeID: "1234",
crd: "apiVersion: \nkind: \nmetadata:\n name: foobar\n creationTimestamp: null\nspec:\n group: \"\"\n names:\n kind: \"somename\"\n plural: \"somenames\"\n scope: \"\"\n versions: null\nstatus:\n acceptedNames:\n kind: \"\"\n plural: \"\"\n conditions: null\n storedVersions: null\n",
expectedFile: "apiVersion: apiextensions.k8s.io/v1\nkind: CustomResourceDefinition\nmetadata:\n name: foobar\n creationTimestamp: null\nspec:\n group: \"\"\n names:\n kind: \"somename\"\n plural: \"somenames\"\n scope: \"\"\n versions: null\nstatus:\n acceptedNames:\n kind: \"\"\n plural: \"\"\n conditions: null\n storedVersions: null\n",
},
"api request fails": {
upgradeID: "1234",
getCRDsError: errors.New("api error"),
wantError: true,
},
}
for name, tc := range testCases {
t.Run(name, func(t *testing.T) {
assert := assert.New(t)
require := require.New(t)
memFs := afero.NewMemMapFs()
crd := apiextensionsv1.CustomResourceDefinition{}
err := yaml.Unmarshal([]byte(tc.crd), &crd)
require.NoError(err)
client := UpgradeClient{
config: nil,
kubectl: stubCrdClient{crds: []apiextensionsv1.CustomResourceDefinition{crd}, getCRDsError: tc.getCRDsError},
fs: file.NewHandler(memFs),
log: stubLog{},
}
_, err = client.backupCRDs(context.Background(), tc.upgradeID)
if tc.wantError {
assert.Error(err)
return
}
assert.NoError(err)
data, err := afero.ReadFile(memFs, filepath.Join(client.crdBackupFolder(tc.upgradeID), crd.Name+".yaml"))
require.NoError(err)
assert.YAMLEq(tc.expectedFile, string(data))
})
}
}
func TestBackupCRs(t *testing.T) {
testCases := map[string]struct {
upgradeID string
crd apiextensionsv1.CustomResourceDefinition
resource unstructured.Unstructured
expectedFile string
getCRsError error
wantError bool
}{
"success": {
upgradeID: "1234",
crd: apiextensionsv1.CustomResourceDefinition{
Spec: apiextensionsv1.CustomResourceDefinitionSpec{
Names: apiextensionsv1.CustomResourceDefinitionNames{
Plural: "foobars",
},
Group: "some.group",
Versions: []apiextensionsv1.CustomResourceDefinitionVersion{
{
Name: "versionZero",
},
},
},
},
resource: unstructured.Unstructured{Object: map[string]any{"metadata": map[string]any{"name": "foobar"}}},
expectedFile: "metadata:\n name: foobar\n",
},
"api request fails": {
upgradeID: "1234",
crd: apiextensionsv1.CustomResourceDefinition{
Spec: apiextensionsv1.CustomResourceDefinitionSpec{
Names: apiextensionsv1.CustomResourceDefinitionNames{
Plural: "foobars",
},
Group: "some.group",
Versions: []apiextensionsv1.CustomResourceDefinitionVersion{
{
Name: "versionZero",
},
},
},
},
getCRsError: errors.New("api error"),
wantError: true,
},
"custom resource not found": {
upgradeID: "1234",
crd: apiextensionsv1.CustomResourceDefinition{
Spec: apiextensionsv1.CustomResourceDefinitionSpec{
Names: apiextensionsv1.CustomResourceDefinitionNames{
Plural: "foobars",
},
Group: "some.group",
Versions: []apiextensionsv1.CustomResourceDefinitionVersion{
{
Name: "versionZero",
},
},
},
},
getCRsError: k8serrors.NewNotFound(schema.GroupResource{Group: "some.group"}, "foobars"),
},
}
for name, tc := range testCases {
t.Run(name, func(t *testing.T) {
assert := assert.New(t)
require := require.New(t)
memFs := afero.NewMemMapFs()
client := UpgradeClient{
config: nil,
kubectl: stubCrdClient{crs: []unstructured.Unstructured{tc.resource}, getCRsError: tc.getCRsError},
fs: file.NewHandler(memFs),
log: stubLog{},
}
err := client.backupCRs(context.Background(), []apiextensionsv1.CustomResourceDefinition{tc.crd}, tc.upgradeID)
if tc.wantError {
assert.Error(err)
return
}
assert.NoError(err)
data, err := afero.ReadFile(memFs, filepath.Join(client.backupFolder(tc.upgradeID), tc.crd.Spec.Group, tc.crd.Spec.Versions[0].Name, tc.resource.GetNamespace(), tc.resource.GetKind(), tc.resource.GetName()+".yaml"))
if tc.expectedFile == "" {
assert.Error(err)
return
}
require.NoError(err)
assert.YAMLEq(tc.expectedFile, string(data))
})
}
}
type stubLog struct{}
func (s stubLog) Debugf(_ string, _ ...any) {}
func (s stubLog) Sync() {}
type stubCrdClient struct {
crds []apiextensionsv1.CustomResourceDefinition
getCRDsError error
crs []unstructured.Unstructured
getCRsError error
crdClient
}
func (c stubCrdClient) ListCRDs(_ context.Context) ([]apiextensionsv1.CustomResourceDefinition, error) {
if c.getCRDsError != nil {
return nil, c.getCRDsError
}
return c.crds, nil
}
func (c stubCrdClient) ListCRs(_ context.Context, _ schema.GroupVersionResource) ([]unstructured.Unstructured, error) {
if c.getCRsError != nil {
return nil, c.getCRsError
}
return c.crs, nil
}

98
cli/internal/helm/helm.go
View file

@ -28,6 +28,104 @@ As such, the number of exported functions should be kept minimal.
*/
package helm
import (
"context"
"fmt"
"github.com/edgelesssys/constellation/v2/cli/internal/clusterid"
"github.com/edgelesssys/constellation/v2/cli/internal/terraform"
"github.com/edgelesssys/constellation/v2/internal/config"
"github.com/edgelesssys/constellation/v2/internal/constants"
"github.com/edgelesssys/constellation/v2/internal/kms/uri"
"github.com/edgelesssys/constellation/v2/internal/kubernetes/kubectl"
"github.com/edgelesssys/constellation/v2/internal/semver"
"github.com/edgelesssys/constellation/v2/internal/versions"
)
const (
// AllowDestructive is a named bool to signal that destructive actions have been confirmed by the user.
AllowDestructive = true
// DenyDestructive is a named bool to signal that destructive actions have not been confirmed by the user yet.
DenyDestructive = false
)
type debugLog interface {
Debugf(format string, args ...any)
Sync()
}
// Client is a Helm client to apply charts.
type Client struct {
factory *actionFactory
cliVersion semver.Semver
log debugLog
}
// NewClient returns a new Helm client.
func NewClient(kubeConfigPath string, log debugLog) (*Client, error) {
kubeClient, err := kubectl.NewFromConfig(kubeConfigPath)
if err != nil {
return nil, fmt.Errorf("initializing kubectl: %w", err)
}
actionConfig, err := newActionConfig(kubeConfigPath, log)
if err != nil {
return nil, fmt.Errorf("creating action config: %w", err)
}
lister := ReleaseVersionClient{actionConfig}
cliVersion := constants.BinaryVersion()
factory := newActionFactory(kubeClient, lister, actionConfig, cliVersion, log)
return &Client{factory, cliVersion, log}, nil
}
// Options are options for loading charts.
type Options struct {
Conformance bool
HelmWaitMode WaitMode
AllowDestructive bool
Force bool
}
// PrepareApply loads the charts and returns the executor to apply them.
// TODO(elchead): remove validK8sVersion by putting ValidK8sVersion into config.Config, see AB#3374.
func (h Client) PrepareApply(conf *config.Config, validK8sversion versions.ValidK8sVersion, idFile clusterid.File, flags Options, tfOutput terraform.ApplyOutput, serviceAccURI string, masterSecret uri.MasterSecret) (Applier, bool, error) {
releases, err := h.loadReleases(conf, masterSecret, validK8sversion, idFile, flags, tfOutput, serviceAccURI)
if err != nil {
return nil, false, fmt.Errorf("loading Helm releases: %w", err)
}
h.log.Debugf("Loaded Helm releases")
actions, includesUpgrades, err := h.factory.GetActions(releases, flags.Force, flags.AllowDestructive)
return &ChartApplyExecutor{actions: actions, log: h.log}, includesUpgrades, err
}
func (h Client) loadReleases(conf *config.Config, secret uri.MasterSecret, validK8sVersion versions.ValidK8sVersion, idFile clusterid.File, flags Options, tfOutput terraform.ApplyOutput, serviceAccURI string) ([]Release, error) {
helmLoader := newLoader(conf, idFile, validK8sVersion, h.cliVersion)
h.log.Debugf("Created new Helm loader")
return helmLoader.loadReleases(flags.Conformance, flags.HelmWaitMode, secret,
serviceAccURI, tfOutput)
}
// Applier runs the Helm actions.
type Applier interface {
Apply(ctx context.Context) error
}
// ChartApplyExecutor is a Helm action executor that applies all actions.
type ChartApplyExecutor struct {
actions []applyAction
log debugLog
}
// Apply applies the charts in order.
func (c ChartApplyExecutor) Apply(ctx context.Context) error {
for _, action := range c.actions {
c.log.Debugf("Applying %q", action.ReleaseName())
if err := action.Apply(ctx); err != nil {
return fmt.Errorf("applying %s: %w", action.ReleaseName(), err)
}
}
return nil
}
// mergeMaps returns a new map that is the merger of it's inputs.
// Key collisions are resolved by taking the value of the second argument (map b).
// Taken from: https://github.com/helm/helm/blob/dbc6d8e20fe1d58d50e6ed30f09a04a77e4c68db/pkg/cli/values/options.go#L91-L108.

154
cli/internal/helm/helm_test.go
View file

@ -7,9 +7,21 @@ SPDX-License-Identifier: AGPL-3.0-only
package helm
import (
"errors"
"testing"
"github.com/edgelesssys/constellation/v2/cli/internal/clusterid"
"github.com/edgelesssys/constellation/v2/cli/internal/terraform"
"github.com/edgelesssys/constellation/v2/internal/cloud/cloudprovider"
"github.com/edgelesssys/constellation/v2/internal/compatibility"
"github.com/edgelesssys/constellation/v2/internal/config"
"github.com/edgelesssys/constellation/v2/internal/kms/uri"
"github.com/edgelesssys/constellation/v2/internal/logger"
"github.com/edgelesssys/constellation/v2/internal/semver"
"github.com/edgelesssys/constellation/v2/internal/versions"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/mock"
"helm.sh/helm/v3/pkg/action"
)
func TestMergeMaps(t *testing.T) {
@ -107,3 +119,145 @@ func TestMergeMaps(t *testing.T) {
})
}
}
func TestHelmApply(t *testing.T) {
cliVersion := semver.NewFromInt(1, 99, 0, "")
csp := cloudprovider.AWS // using AWS since it has an additional chart: aws-load-balancer-controller
microserviceCharts := []string{
"constellation-services",
"constellation-operators",
"constellation-csi",
}
testCases := map[string]struct {
clusterMicroServiceVersion string
expectedActions []string
expectUpgrade bool
clusterCertManagerVersion *string
clusterAWSLBVersion *string
allowDestructive bool
expectError bool
}{
"CLI microservices are 1 minor version newer than cluster ones": {
clusterMicroServiceVersion: "v1.98.1",
expectedActions: microserviceCharts,
expectUpgrade: true,
},
"CLI microservices are 2 minor versions newer than cluster ones": {
clusterMicroServiceVersion: "v1.97.0",
expectedActions: []string{},
},
"cluster microservices are newer than CLI": {
clusterMicroServiceVersion: "v1.100.0",
},
"cluster and CLI microservices have the same version": {
clusterMicroServiceVersion: "v1.99.0",
expectedActions: []string{},
},
"cert-manager upgrade is ignored when denying destructive upgrades": {
clusterMicroServiceVersion: "v1.99.0",
clusterCertManagerVersion: toPtr("v1.9.0"),
allowDestructive: false,
expectError: true,
},
"both microservices and cert-manager are upgraded in destructive mode": {
clusterMicroServiceVersion: "v1.98.1",
clusterCertManagerVersion: toPtr("v1.9.0"),
expectedActions: append(microserviceCharts, "cert-manager"),
expectUpgrade: true,
allowDestructive: true,
},
"only missing aws-load-balancer-controller is installed": {
clusterMicroServiceVersion: "v1.99.0",
clusterAWSLBVersion: toPtr(""),
expectedActions: []string{"aws-load-balancer-controller"},
},
}
cfg := config.Default()
cfg.RemoveProviderAndAttestationExcept(csp)
log := logger.NewTest(t)
options := Options{
Conformance: false,
HelmWaitMode: WaitModeWait,
AllowDestructive: true,
Force: false,
}
for name, tc := range testCases {
t.Run(name, func(t *testing.T) {
lister := &ReleaseVersionStub{}
sut := Client{
factory: newActionFactory(nil, lister, &action.Configuration{}, cliVersion, log),
log: log,
cliVersion: cliVersion,
}
awsLbVersion := "v1.5.4" // current version
if tc.clusterAWSLBVersion != nil {
awsLbVersion = *tc.clusterAWSLBVersion
}
certManagerVersion := "v1.10.0" // current version
if tc.clusterCertManagerVersion != nil {
certManagerVersion = *tc.clusterCertManagerVersion
}
helmListVersion(lister, "cilium", "v1.12.1")
helmListVersion(lister, "cert-manager", certManagerVersion)
helmListVersion(lister, "constellation-services", tc.clusterMicroServiceVersion)
helmListVersion(lister, "constellation-operators", tc.clusterMicroServiceVersion)
helmListVersion(lister, "constellation-csi", tc.clusterMicroServiceVersion)
helmListVersion(lister, "aws-load-balancer-controller", awsLbVersion)
options.AllowDestructive = tc.allowDestructive
ex, includesUpgrade, err := sut.PrepareApply(cfg, versions.ValidK8sVersion("v1.27.4"),
clusterid.File{UID: "testuid", MeasurementSalt: []byte("measurementSalt")}, options,
fakeTerraformOutput(csp), fakeServiceAccURI(csp),
uri.MasterSecret{Key: []byte("secret"), Salt: []byte("masterSalt")})
var upgradeErr *compatibility.InvalidUpgradeError
if tc.expectError {
assert.Error(t, err)
} else {
assert.True(t, err == nil || errors.As(err, &upgradeErr))
}
assert.Equal(t, tc.expectUpgrade, includesUpgrade)
chartExecutor, ok := ex.(*ChartApplyExecutor)
assert.True(t, ok)
assert.ElementsMatch(t, tc.expectedActions, getActionReleaseNames(chartExecutor.actions))
})
}
}
func fakeTerraformOutput(csp cloudprovider.Provider) terraform.ApplyOutput {
switch csp {
case cloudprovider.AWS:
return terraform.ApplyOutput{}
case cloudprovider.GCP:
return terraform.ApplyOutput{GCP: &terraform.GCPApplyOutput{}}
default:
panic("invalid csp")
}
}
func getActionReleaseNames(actions []applyAction) []string {
releaseActionNames := []string{}
for _, action := range actions {
releaseActionNames = append(releaseActionNames, action.ReleaseName())
}
return releaseActionNames
}
func helmListVersion(l *ReleaseVersionStub, releaseName string, installedVersion string) {
if installedVersion == "" {
l.On("currentVersion", releaseName).Return(semver.Semver{}, errReleaseNotFound)
return
}
v, _ := semver.New(installedVersion)
l.On("currentVersion", releaseName).Return(v, nil)
}
type ReleaseVersionStub struct {
mock.Mock
}
func (s *ReleaseVersionStub) currentVersion(release string) (semver.Semver, error) {
args := s.Called(release)
return args.Get(0).(semver.Semver), args.Error(1)
}

110
cli/internal/helm/init.go
View file

@ -1,110 +0,0 @@
/*
Copyright (c) Edgeless Systems GmbH
SPDX-License-Identifier: AGPL-3.0-only
*/
package helm
import (
"context"
"fmt"
"time"
"github.com/edgelesssys/constellation/v2/internal/constants"
)
// InitializationClient installs all Helm charts required for a Constellation cluster.
type InitializationClient struct {
log debugLog
installer installer
}
// NewInitializer creates a new client to install all Helm charts required for a constellation cluster.
func NewInitializer(log debugLog, adminConfPath string) (*InitializationClient, error) {
installer, err := NewInstaller(adminConfPath, log)
if err != nil {
return nil, fmt.Errorf("creating Helm installer: %w", err)
}
return &InitializationClient{log: log, installer: installer}, nil
}
// Install installs all Helm charts required for a constellation cluster.
func (i InitializationClient) Install(ctx context.Context, releases *Releases) error {
if err := i.installer.InstallChart(ctx, releases.Cilium); err != nil {
return fmt.Errorf("installing Cilium: %w", err)
}
i.log.Debugf("Waiting for Cilium to become ready")
helper, err := newK8sCiliumHelper(constants.AdminConfFilename)
if err != nil {
return fmt.Errorf("creating Kubernetes client: %w", err)
}
timeToStartWaiting := time.Now()
// TODO(3u13r): Reduce the timeout when we switched the package repository - this is only this high because we once
// saw polling times of ~16 minutes when hitting a slow PoP from Fastly (GitHub's / ghcr.io CDN).
if err := helper.WaitForDS(ctx, "kube-system", "cilium", i.log); err != nil {
return fmt.Errorf("waiting for Cilium to become healthy: %w", err)
}
timeUntilFinishedWaiting := time.Since(timeToStartWaiting)
i.log.Debugf("Cilium became healthy after %s", timeUntilFinishedWaiting.String())
i.log.Debugf("Fix Cilium through restart")
if err := helper.RestartDS("kube-system", "cilium"); err != nil {
return fmt.Errorf("restarting Cilium: %w", err)
}
i.log.Debugf("Installing microservices")
if err := i.installer.InstallChart(ctx, releases.ConstellationServices); err != nil {
return fmt.Errorf("installing microservices: %w", err)
}
i.log.Debugf("Installing cert-manager")
if err := i.installer.InstallChart(ctx, releases.CertManager); err != nil {
return fmt.Errorf("installing cert-manager: %w", err)
}
if releases.CSI != nil {
i.log.Debugf("Installing CSI deployments")
if err := i.installer.InstallChart(ctx, *releases.CSI); err != nil {
return fmt.Errorf("installing CSI snapshot CRDs: %w", err)
}
}
if releases.AWSLoadBalancerController != nil {
i.log.Debugf("Installing AWS Load Balancer Controller")
if err := i.installer.InstallChart(ctx, *releases.AWSLoadBalancerController); err != nil {
return fmt.Errorf("installing AWS Load Balancer Controller: %w", err)
}
}
i.log.Debugf("Installing constellation operators")
if err := i.installer.InstallChart(ctx, releases.ConstellationOperators); err != nil {
return fmt.Errorf("installing constellation operators: %w", err)
}
return nil
}
// installer is the interface for installing a single Helm chart.
type installer interface {
InstallChart(context.Context, Release) error
InstallChartWithValues(ctx context.Context, release Release, extraValues map[string]any) error
}
type cloudConfig struct {
Cloud string `json:"cloud,omitempty"`
TenantID string `json:"tenantId,omitempty"`
SubscriptionID string `json:"subscriptionId,omitempty"`
ResourceGroup string `json:"resourceGroup,omitempty"`
Location string `json:"location,omitempty"`
SubnetName string `json:"subnetName,omitempty"`
SecurityGroupName string `json:"securityGroupName,omitempty"`
SecurityGroupResourceGroup string `json:"securityGroupResourceGroup,omitempty"`
LoadBalancerName string `json:"loadBalancerName,omitempty"`
LoadBalancerSku string `json:"loadBalancerSku,omitempty"`
VNetName string `json:"vnetName,omitempty"`
VNetResourceGroup string `json:"vnetResourceGroup,omitempty"`
CloudProviderBackoff bool `json:"cloudProviderBackoff,omitempty"`
UseInstanceMetadata bool `json:"useInstanceMetadata,omitempty"`
VMType string `json:"vmType,omitempty"`
UseManagedIdentityExtension bool `json:"useManagedIdentityExtension,omitempty"`
UserAssignedIdentityID string `json:"userAssignedIdentityID,omitempty"`
}

151
cli/internal/helm/install.go
View file

@ -1,151 +0,0 @@
/*
Copyright (c) Edgeless Systems GmbH
SPDX-License-Identifier: AGPL-3.0-only
*/
package helm
import (
"context"
"fmt"
"strings"
"time"
"github.com/edgelesssys/constellation/v2/internal/constants"
"github.com/edgelesssys/constellation/v2/internal/retry"
"helm.sh/helm/v3/pkg/action"
"helm.sh/helm/v3/pkg/chart"
"helm.sh/helm/v3/pkg/cli"
"k8s.io/apimachinery/pkg/util/wait"
)
const (
// timeout is the maximum time given to the helm Installer.
timeout = 10 * time.Minute
// maximumRetryAttempts is the maximum number of attempts to retry a helm install.
maximumRetryAttempts = 3
)
type debugLog interface {
Debugf(format string, args ...any)
Sync()
}
// Installer is a wrapper for a helm install action.
type Installer struct {
*action.Install
log debugLog
}
// NewInstaller creates a new Installer with the given logger.
func NewInstaller(kubeconfig string, logger debugLog) (*Installer, error) {
settings := cli.New()
settings.KubeConfig = kubeconfig
actionConfig := &action.Configuration{}
if err := actionConfig.Init(settings.RESTClientGetter(), constants.HelmNamespace,
"secret", logger.Debugf); err != nil {
return nil, err
}
action := action.NewInstall(actionConfig)
action.Namespace = constants.HelmNamespace
action.Timeout = timeout
return &Installer{
Install: action,
log: logger,
}, nil
}
// InstallChart is the generic install function for helm charts.
func (h *Installer) InstallChart(ctx context.Context, release Release) error {
return h.InstallChartWithValues(ctx, release, nil)
}
// InstallChartWithValues is the generic install function for helm charts with custom values.
func (h *Installer) InstallChartWithValues(ctx context.Context, release Release, extraValues map[string]any) error {
mergedVals := mergeMaps(release.Values, extraValues)
h.ReleaseName = release.ReleaseName
if err := h.SetWaitMode(release.WaitMode); err != nil {
return err
}
return h.install(ctx, release.Chart, mergedVals)
}
// install tries to install the given chart and aborts after ~5 tries.
// The function will wait 30 seconds before retrying a failed installation attempt.
// After 3 tries, the retrier will be canceled and the function returns with an error.
func (h *Installer) install(ctx context.Context, chart *chart.Chart, values map[string]any) error {
var retries int
retriable := func(err error) bool {
// abort after maximumRetryAttempts tries.
if retries >= maximumRetryAttempts {
return false
}
retries++
// only retry if atomic is set
// otherwise helm doesn't uninstall
// the release on failure
if !h.Atomic {
return false
}
// check if error is retriable
return wait.Interrupted(err) ||
strings.Contains(err.Error(), "connection refused")
}
doer := installDoer{
h,
chart,
values,
h.log,
}
retrier := retry.NewIntervalRetrier(doer, 30*time.Second, retriable)
retryLoopStartTime := time.Now()
if err := retrier.Do(ctx); err != nil {
return fmt.Errorf("helm install: %w", err)
}
retryLoopFinishDuration := time.Since(retryLoopStartTime)
h.log.Debugf("Helm chart %q installation finished after %s", chart.Name(), retryLoopFinishDuration)
return nil
}
// SetWaitMode sets the wait mode of the installer.
func (h *Installer) SetWaitMode(waitMode WaitMode) error {
switch waitMode {
case WaitModeNone:
h.Wait = false
h.Atomic = false
case WaitModeWait:
h.Wait = true
h.Atomic = false
case WaitModeAtomic:
h.Wait = true
h.Atomic = true
default:
return fmt.Errorf("unknown wait mode %q", waitMode)
}
return nil
}
// installDoer is a help struct to enable retrying helm's install action.
type installDoer struct {
Installer *Installer
chart *chart.Chart
values map[string]any
log debugLog
}
// Do logs which chart is installed and tries to install it.
func (i installDoer) Do(ctx context.Context) error {
i.log.Debugf("Trying to install Helm chart %s", i.chart.Name())
if _, err := i.Installer.RunWithContext(ctx, i.chart, i.values); err != nil {
i.log.Debugf("Helm chart installation %s failed: %v", i.chart.Name(), err)
return err
}
return nil
}

97
cli/internal/helm/loader.go
View file

@ -57,9 +57,10 @@ var (
csiInfo = chartInfo{releaseName: "constellation-csi", chartName: "constellation-csi", path: "charts/edgeless/csi"}
)
// ChartLoader loads embedded helm charts.
type ChartLoader struct {
// chartLoader loads embedded helm charts.
type chartLoader struct {
csp cloudprovider.Provider
config *config.Config
joinServiceImage string
keyServiceImage string
ccmImage string // cloud controller manager image
@ -71,11 +72,16 @@ type ChartLoader struct {
constellationOperatorImage string
nodeMaintenanceOperatorImage string
clusterName string
idFile clusterid.File
cliVersion semver.Semver
}
// NewLoader creates a new ChartLoader.
func NewLoader(csp cloudprovider.Provider, k8sVersion versions.ValidK8sVersion, clusterName string) *ChartLoader {
// newLoader creates a new ChartLoader.
func newLoader(config *config.Config, idFile clusterid.File, k8sVersion versions.ValidK8sVersion, cliVersion semver.Semver) *chartLoader {
// TODO(malt3): Allow overriding container image registry + prefix for all images
// (e.g. for air-gapped environments).
var ccmImage, cnmImage string
csp := config.GetProvider()
switch csp {
case cloudprovider.AWS:
ccmImage = versions.VersionConfigs[k8sVersion].CloudControllerManagerImageAWS
@ -87,35 +93,39 @@ func NewLoader(csp cloudprovider.Provider, k8sVersion versions.ValidK8sVersion,
case cloudprovider.OpenStack:
ccmImage = versions.VersionConfigs[k8sVersion].CloudControllerManagerImageOpenStack
}
// TODO(malt3): Allow overriding container image registry + prefix for all images
// (e.g. for air-gapped environments).
return &ChartLoader{
return &chartLoader{
cliVersion: cliVersion,
csp: csp,
joinServiceImage: imageversion.JoinService("", ""),
keyServiceImage: imageversion.KeyService("", ""),
idFile: idFile,
ccmImage: ccmImage,
azureCNMImage: cnmImage,
config: config,
joinServiceImage: imageversion.JoinService("", ""),
keyServiceImage: imageversion.KeyService("", ""),
autoscalerImage: versions.VersionConfigs[k8sVersion].ClusterAutoscalerImage,
verificationServiceImage: imageversion.VerificationService("", ""),
gcpGuestAgentImage: versions.GcpGuestImage,
konnectivityImage: versions.KonnectivityAgentImage,
constellationOperatorImage: imageversion.ConstellationNodeOperator("", ""),
nodeMaintenanceOperatorImage: versions.NodeMaintenanceOperatorImage,
clusterName: clusterName,
}
}
// LoadReleases loads the embedded helm charts and returns them as a HelmReleases object.
func (i *ChartLoader) LoadReleases(
config *config.Config, conformanceMode bool, helmWaitMode WaitMode, masterSecret uri.MasterSecret,
serviceAccURI string, idFile clusterid.File, output terraform.ApplyOutput,
) (*Releases, error) {
// releaseApplyOrder is a list of releases in the order they should be applied.
// makes sure if a release was removed as a dependency from one chart,
// and then added as a new standalone chart (or as a dependency of another chart),
// that the new release is installed after the existing one to avoid name conflicts.
type releaseApplyOrder []Release
// loadReleases loads the embedded helm charts and returns them as a HelmReleases object.
func (i *chartLoader) loadReleases(conformanceMode bool, helmWaitMode WaitMode, masterSecret uri.MasterSecret,
serviceAccURI string, output terraform.ApplyOutput,
) (releaseApplyOrder, error) {
ciliumRelease, err := i.loadRelease(ciliumInfo, helmWaitMode)
if err != nil {
return nil, fmt.Errorf("loading cilium: %w", err)
}
ciliumVals := extraCiliumValues(config.GetProvider(), conformanceMode, output)
ciliumVals := extraCiliumValues(i.config.GetProvider(), conformanceMode, output)
ciliumRelease.Values = mergeMaps(ciliumRelease.Values, ciliumVals)
certManagerRelease, err := i.loadRelease(certManagerInfo, helmWaitMode)
@ -127,46 +137,47 @@ func (i *ChartLoader) LoadReleases(
if err != nil {
return nil, fmt.Errorf("loading operators: %w", err)
}
operatorRelease.Values = mergeMaps(operatorRelease.Values, extraOperatorValues(idFile.UID))
operatorRelease.Values = mergeMaps(operatorRelease.Values, extraOperatorValues(i.idFile.UID))
conServicesRelease, err := i.loadRelease(constellationServicesInfo, helmWaitMode)
if err != nil {
return nil, fmt.Errorf("loading constellation-services: %w", err)
}
svcVals, err := extraConstellationServicesValues(config, masterSecret, idFile.UID, serviceAccURI, output)
svcVals, err := extraConstellationServicesValues(i.config, masterSecret, i.idFile.UID, serviceAccURI, output)
if err != nil {
return nil, fmt.Errorf("extending constellation-services values: %w", err)
}
conServicesRelease.Values = mergeMaps(conServicesRelease.Values, svcVals)
releases := Releases{Cilium: ciliumRelease, CertManager: certManagerRelease, ConstellationOperators: operatorRelease, ConstellationServices: conServicesRelease}
if config.HasProvider(cloudprovider.AWS) {
awsRelease, err := i.loadRelease(awsLBControllerInfo, helmWaitMode)
if err != nil {
return nil, fmt.Errorf("loading aws-services: %w", err)
}
releases.AWSLoadBalancerController = &awsRelease
}
if config.DeployCSIDriver() {
releases := releaseApplyOrder{ciliumRelease, conServicesRelease, certManagerRelease}
if i.config.DeployCSIDriver() {
csiRelease, err := i.loadRelease(csiInfo, helmWaitMode)
if err != nil {
return nil, fmt.Errorf("loading snapshot CRDs: %w", err)
}
extraCSIvals, err := extraCSIValues(config.GetProvider(), serviceAccURI)
extraCSIvals, err := extraCSIValues(i.config.GetProvider(), serviceAccURI)
if err != nil {
return nil, fmt.Errorf("extending CSI values: %w", err)
}
csiRelease.Values = mergeMaps(csiRelease.Values, extraCSIvals)
releases.CSI = &csiRelease
releases = append(releases, csiRelease)
}
return &releases, nil
if i.config.HasProvider(cloudprovider.AWS) {
awsRelease, err := i.loadRelease(awsLBControllerInfo, helmWaitMode)
if err != nil {
return nil, fmt.Errorf("loading aws-services: %w", err)
}
releases = append(releases, awsRelease)
}
releases = append(releases, operatorRelease)
return releases, nil
}
// loadRelease loads the embedded chart and values depending on the given info argument.
// IMPORTANT: .helmignore rules specifying files in subdirectories are not applied (e.g. crds/kustomization.yaml).
func (i *ChartLoader) loadRelease(info chartInfo, helmWaitMode WaitMode) (Release, error) {
func (i *chartLoader) loadRelease(info chartInfo, helmWaitMode WaitMode) (Release, error) {
chart, err := loadChartsDir(helmFS, info.path)
if err != nil {
return Release{}, fmt.Errorf("loading %s chart: %w", info.releaseName, err)
@ -184,23 +195,23 @@ func (i *ChartLoader) loadRelease(info chartInfo, helmWaitMode WaitMode) (Releas
case certManagerInfo.releaseName:
values = i.loadCertManagerValues()
case constellationOperatorsInfo.releaseName:
updateVersions(chart, constants.BinaryVersion())
updateVersions(chart, i.cliVersion)
values = i.loadOperatorsValues()
case constellationServicesInfo.releaseName:
updateVersions(chart, constants.BinaryVersion())
updateVersions(chart, i.cliVersion)
values = i.loadConstellationServicesValues()
case awsLBControllerInfo.releaseName:
values = i.loadAWSLBControllerValues()
case csiInfo.releaseName:
updateVersions(chart, constants.BinaryVersion())
updateVersions(chart, i.cliVersion)
values = i.loadCSIValues()
}
return Release{Chart: chart, Values: values, ReleaseName: info.releaseName, WaitMode: helmWaitMode}, nil
}
func (i *ChartLoader) loadAWSLBControllerValues() map[string]any {
func (i *chartLoader) loadAWSLBControllerValues() map[string]any {
return map[string]any{
"clusterName": i.clusterName,
"clusterName": clusterid.GetClusterName(i.config, i.idFile),
"tolerations": controlPlaneTolerations,
"nodeSelector": controlPlaneNodeSelector,
}
@ -208,7 +219,7 @@ func (i *ChartLoader) loadAWSLBControllerValues() map[string]any {
// loadCertManagerHelper is used to separate the marshalling step from the loading step.
// This reduces the time unit tests take to execute.
func (i *ChartLoader) loadCertManagerValues() map[string]any {
func (i *chartLoader) loadCertManagerValues() map[string]any {
return map[string]any{
"installCRDs": true,
"prometheus": map[string]any{
@ -233,7 +244,7 @@ func (i *ChartLoader) loadCertManagerValues() map[string]any {
// loadOperatorsHelper is used to separate the marshalling step from the loading step.
// This reduces the time unit tests take to execute.
func (i *ChartLoader) loadOperatorsValues() map[string]any {
func (i *chartLoader) loadOperatorsValues() map[string]any {
return map[string]any{
"constellation-operator": map[string]any{
"controllerManager": map[string]any{
@ -256,7 +267,7 @@ func (i *ChartLoader) loadOperatorsValues() map[string]any {
// loadConstellationServicesHelper is used to separate the marshalling step from the loading step.
// This reduces the time unit tests take to execute.
func (i *ChartLoader) loadConstellationServicesValues() map[string]any {
func (i *chartLoader) loadConstellationServicesValues() map[string]any {
return map[string]any{
"global": map[string]any{
"keyServicePort": constants.KeyServicePort,
@ -299,13 +310,13 @@ func (i *ChartLoader) loadConstellationServicesValues() map[string]any {
}
}
func (i *ChartLoader) loadCSIValues() map[string]any {
func (i *chartLoader) loadCSIValues() map[string]any {
return map[string]any{
"tags": i.cspTags(),
}
}
func (i *ChartLoader) cspTags() map[string]any {
func (i *chartLoader) cspTags() map[string]any {
return map[string]any{
i.csp.String(): true,
}

32
cli/internal/helm/loader_test.go
View file

@ -30,6 +30,8 @@ import (
"github.com/edgelesssys/constellation/v2/internal/cloud/gcpshared"
"github.com/edgelesssys/constellation/v2/internal/config"
"github.com/edgelesssys/constellation/v2/internal/kms/uri"
"github.com/edgelesssys/constellation/v2/internal/semver"
"github.com/edgelesssys/constellation/v2/internal/versions"
)
func fakeServiceAccURI(provider cloudprovider.Provider) string {
@ -65,26 +67,34 @@ func TestLoadReleases(t *testing.T) {
assert := assert.New(t)
require := require.New(t)
config := &config.Config{Provider: config.ProviderConfig{GCP: &config.GCPConfig{}}}
chartLoader := ChartLoader{csp: config.GetProvider()}
helmReleases, err := chartLoader.LoadReleases(
config, true, WaitModeAtomic,
k8sVersion := versions.ValidK8sVersion("v1.27.4")
chartLoader := newLoader(config, clusterid.File{UID: "testuid", MeasurementSalt: []byte("measurementSalt")},
k8sVersion, semver.NewFromInt(2, 10, 0, ""))
helmReleases, err := chartLoader.loadReleases(
true, WaitModeAtomic,
uri.MasterSecret{Key: []byte("secret"), Salt: []byte("masterSalt")},
fakeServiceAccURI(cloudprovider.GCP), clusterid.File{UID: "testuid"}, terraform.ApplyOutput{GCP: &terraform.GCPApplyOutput{}},
fakeServiceAccURI(cloudprovider.GCP), terraform.ApplyOutput{GCP: &terraform.GCPApplyOutput{}},
)
require.NoError(err)
chart := helmReleases.ConstellationServices.Chart
assert.NotNil(chart.Dependencies())
for _, release := range helmReleases {
if release.ReleaseName == constellationServicesInfo.releaseName {
assert.NotNil(release.Chart.Dependencies())
}
}
}
func TestLoadAWSLoadBalancerValues(t *testing.T) {
sut := ChartLoader{
sut := chartLoader{
config: &config.Config{Name: "testCluster"},
clusterName: "testCluster",
idFile: clusterid.File{UID: "testuid"},
}
val := sut.loadAWSLBControllerValues()
assert.Equal(t, "testCluster", val["clusterName"])
assert.Equal(t, "testCluster-testuid", val["clusterName"])
// needs to run on control-plane
assert.Contains(t, val["nodeSelector"].(map[string]any), "node-role.kubernetes.io/control-plane")
assert.Contains(t, val["tolerations"].([]map[string]any), map[string]any{"key": "node-role.kubernetes.io/control-plane", "operator": "Exists", "effect": "NoSchedule"})
assert.Contains(t, val["tolerations"].([]map[string]any),
map[string]any{"key": "node-role.kubernetes.io/control-plane", "operator": "Exists", "effect": "NoSchedule"})
}
// TestConstellationServices checks if the rendered constellation-services chart produces the expected yaml files.
@ -146,7 +156,7 @@ func TestConstellationServices(t *testing.T) {
assert := assert.New(t)
require := require.New(t)
chartLoader := ChartLoader{
chartLoader := chartLoader{
csp: tc.config.GetProvider(),
joinServiceImage: "joinServiceImage",
keyServiceImage: "keyServiceImage",
@ -239,7 +249,7 @@ func TestOperators(t *testing.T) {
assert := assert.New(t)
require := require.New(t)
chartLoader := ChartLoader{
chartLoader := chartLoader{
csp: tc.csp,
joinServiceImage: "joinServiceImage",
keyServiceImage: "keyServiceImage",

21
cli/internal/helm/overrides.go
View file

@ -125,6 +125,27 @@ func extraConstellationServicesValues(
return extraVals, nil
}
// cloudConfig is used to marshal the cloud config for the Kubernetes Cloud Controller Manager on Azure.
type cloudConfig struct {
Cloud string `json:"cloud,omitempty"`
TenantID string `json:"tenantId,omitempty"`
SubscriptionID string `json:"subscriptionId,omitempty"`
ResourceGroup string `json:"resourceGroup,omitempty"`
Location string `json:"location,omitempty"`
SubnetName string `json:"subnetName,omitempty"`
SecurityGroupName string `json:"securityGroupName,omitempty"`
SecurityGroupResourceGroup string `json:"securityGroupResourceGroup,omitempty"`
LoadBalancerName string `json:"loadBalancerName,omitempty"`
LoadBalancerSku string `json:"loadBalancerSku,omitempty"`
VNetName string `json:"vnetName,omitempty"`
VNetResourceGroup string `json:"vnetResourceGroup,omitempty"`
CloudProviderBackoff bool `json:"cloudProviderBackoff,omitempty"`
UseInstanceMetadata bool `json:"useInstanceMetadata,omitempty"`
VMType string `json:"vmType,omitempty"`
UseManagedIdentityExtension bool `json:"useManagedIdentityExtension,omitempty"`
UserAssignedIdentityID string `json:"userAssignedIdentityID,omitempty"`
}
// getCCMConfig returns the configuration needed for the Kubernetes Cloud Controller Manager on Azure.
func getCCMConfig(tfOutput terraform.AzureApplyOutput, serviceAccURI string) ([]byte, error) {
creds, err := azureshared.ApplicationCredentialsFromURI(serviceAccURI)

10
cli/internal/helm/release.go
View file

@ -17,16 +17,6 @@ type Release struct {
WaitMode WaitMode
}
// Releases bundles all helm releases to be deployed to Constellation.
type Releases struct {
AWSLoadBalancerController *Release
CSI *Release
Cilium Release
CertManager Release
ConstellationOperators Release
ConstellationServices Release
}
// WaitMode specifies the wait mode for a helm release.
type WaitMode string

79
cli/internal/helm/retryaction.go Normal file
View file

@ -0,0 +1,79 @@
/*
Copyright (c) Edgeless Systems GmbH
SPDX-License-Identifier: AGPL-3.0-only
*/
package helm
import (
"context"
"fmt"
"strings"
"time"
"github.com/edgelesssys/constellation/v2/internal/retry"
"k8s.io/apimachinery/pkg/util/wait"
)
const (
// maximumRetryAttempts is the maximum number of attempts to retry a helm install.
maximumRetryAttempts = 3
)
type retrieableApplier interface {
apply(context.Context) error
ReleaseName() string
IsAtomic() bool
}
// retryApply retries the given retriable action.
func retryApply(ctx context.Context, action retrieableApplier, log debugLog) error {
var retries int
retriable := func(err error) bool {
// abort after maximumRetryAttempts tries.
if retries >= maximumRetryAttempts {
return false
}
retries++
// only retry if atomic is set
// otherwise helm doesn't uninstall
// the release on failure
if !action.IsAtomic() {
return false
}
// check if error is retriable
return wait.Interrupted(err) ||
strings.Contains(err.Error(), "connection refused")
}
doer := applyDoer{
action,
log,
}
retrier := retry.NewIntervalRetrier(doer, 30*time.Second, retriable)
retryLoopStartTime := time.Now()
if err := retrier.Do(ctx); err != nil {
return fmt.Errorf("helm install: %w", err)
}
retryLoopFinishDuration := time.Since(retryLoopStartTime)
log.Debugf("Helm chart %q installation finished after %s", action.ReleaseName(), retryLoopFinishDuration)
return nil
}
// applyDoer is a helper struct to enable retrying helm actions.
type applyDoer struct {
Applier retrieableApplier
log debugLog
}
// Do tries to apply the action.
func (i applyDoer) Do(ctx context.Context) error {
i.log.Debugf("Trying to apply Helm chart %s", i.Applier.ReleaseName())
if err := i.Applier.apply(ctx); err != nil {
i.log.Debugf("Helm chart installation %s failed: %v", i.Applier.ReleaseName(), err)
return err
}
return nil
}

424
cli/internal/helm/upgrade.go
View file

@ -1,424 +0,0 @@
/*
Copyright (c) Edgeless Systems GmbH
SPDX-License-Identifier: AGPL-3.0-only
*/
package helm
import (
"context"
"errors"
"fmt"
"strings"
"time"
"github.com/edgelesssys/constellation/v2/cli/internal/clusterid"
"github.com/edgelesssys/constellation/v2/cli/internal/terraform"
"github.com/edgelesssys/constellation/v2/internal/cloud/cloudprovider"
"github.com/edgelesssys/constellation/v2/internal/compatibility"
"github.com/edgelesssys/constellation/v2/internal/config"
"github.com/edgelesssys/constellation/v2/internal/constants"
"github.com/edgelesssys/constellation/v2/internal/file"
"github.com/edgelesssys/constellation/v2/internal/kms/uri"
"github.com/edgelesssys/constellation/v2/internal/semver"
"github.com/edgelesssys/constellation/v2/internal/versions"
"github.com/spf13/afero"
"helm.sh/helm/v3/pkg/action"
"helm.sh/helm/v3/pkg/chart"
"helm.sh/helm/v3/pkg/cli"
"helm.sh/helm/v3/pkg/release"
apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/runtime/schema"
)
const (
// AllowDestructive is a named bool to signal that destructive actions have been confirmed by the user.
AllowDestructive = true
// DenyDestructive is a named bool to signal that destructive actions have not been confirmed by the user yet.
DenyDestructive = false
)
// ErrConfirmationMissing signals that an action requires user confirmation.
var ErrConfirmationMissing = errors.New("action requires user confirmation")
var errReleaseNotFound = errors.New("release not found")
// UpgradeClient handles interaction with helm and the cluster.
type UpgradeClient struct {
config *action.Configuration
kubectl crdClient
fs file.Handler
actions actionWrapper
log debugLog
}
// NewUpgradeClient returns a newly initialized UpgradeClient for the given namespace.
func NewUpgradeClient(client crdClient, kubeConfigPath, helmNamespace string, log debugLog) (*UpgradeClient, error) {
settings := cli.New()
settings.KubeConfig = kubeConfigPath
actionConfig := &action.Configuration{}
if err := actionConfig.Init(settings.RESTClientGetter(), helmNamespace, "secret", log.Debugf); err != nil {
return nil, fmt.Errorf("initializing config: %w", err)
}
fileHandler := file.NewHandler(afero.NewOsFs())
kubeconfig, err := fileHandler.Read(kubeConfigPath)
if err != nil {
return nil, fmt.Errorf("reading gce config: %w", err)
}
if err := client.Initialize(kubeconfig); err != nil {
return nil, fmt.Errorf("initializing kubectl: %w", err)
}
return &UpgradeClient{
kubectl: client,
fs: fileHandler,
actions: actions{config: actionConfig},
log: log,
}, nil
}
func (c *UpgradeClient) shouldUpgrade(releaseName string, newVersion semver.Semver, force bool) error {
currentVersion, err := c.currentVersion(releaseName)
if err != nil {
return fmt.Errorf("getting version for %s: %w", releaseName, err)
}
c.log.Debugf("Current %s version: %s", releaseName, currentVersion)
c.log.Debugf("New %s version: %s", releaseName, newVersion)
// This may break for cert-manager or cilium if we decide to upgrade more than one minor version at a time.
// Leaving it as is since it is not clear to me what kind of sanity check we could do.
if !force {
if err := newVersion.IsUpgradeTo(currentVersion); err != nil {
return err
}
}
// at this point we conclude that the release should be upgraded. check that this CLI supports the upgrade.
cliVersion := constants.BinaryVersion()
if isCLIVersionedRelease(releaseName) && cliVersion.Compare(newVersion) != 0 {
return fmt.Errorf("this CLI only supports microservice version %s for upgrading", cliVersion.String())
}
c.log.Debugf("Upgrading %s from %s to %s", releaseName, currentVersion, newVersion)
return nil
}
// Upgrade runs a helm-upgrade on all deployments that are managed via Helm.
// If the CLI receives an interrupt signal it will cancel the context.
// Canceling the context will prompt helm to abort and roll back the ongoing upgrade.
func (c *UpgradeClient) Upgrade(ctx context.Context, config *config.Config, idFile clusterid.File, timeout time.Duration,
allowDestructive, force bool, upgradeDir string, conformance bool, helmWaitMode WaitMode, masterSecret uri.MasterSecret,
serviceAccURI string, validK8sVersion versions.ValidK8sVersion, output terraform.ApplyOutput,
) error {
upgradeErrs := []error{}
upgradeReleases := []Release{}
newReleases := []Release{}
clusterName := clusterid.GetClusterName(config, idFile)
helmLoader := NewLoader(config.GetProvider(), validK8sVersion, clusterName)
c.log.Debugf("Created new Helm loader")
releases, err := helmLoader.LoadReleases(config, conformance, helmWaitMode, masterSecret, serviceAccURI, idFile, output)
if err != nil {
return fmt.Errorf("loading releases: %w", err)
}
for _, release := range getManagedReleases(config, releases) {
var invalidUpgrade *compatibility.InvalidUpgradeError
// Get version of the chart embedded in the CLI
// This is the version we are upgrading to
// Since our bundled charts are embedded with version 0.0.0,
// we need to update them to the same version as the CLI
var upgradeVersion semver.Semver
if isCLIVersionedRelease(release.ReleaseName) {
updateVersions(release.Chart, constants.BinaryVersion())
upgradeVersion = config.MicroserviceVersion
} else {
chartVersion, err := semver.New(release.Chart.Metadata.Version)
if err != nil {
return fmt.Errorf("parsing chart version: %w", err)
}
upgradeVersion = chartVersion
}
err = c.shouldUpgrade(release.ReleaseName, upgradeVersion, force)
switch {
case errors.Is(err, errReleaseNotFound):
// if the release is not found, we need to install it
c.log.Debugf("Release %s not found, adding to new releases...", release.ReleaseName)
newReleases = append(newReleases, release)
case errors.As(err, &invalidUpgrade):
c.log.Debugf("Appending to %s upgrade: %s", release.ReleaseName, err)
upgradeErrs = append(upgradeErrs, fmt.Errorf("skipping %s upgrade: %w", release.ReleaseName, err))
case err != nil:
return fmt.Errorf("should upgrade %s: %w", release.ReleaseName, err)
case err == nil:
c.log.Debugf("Adding %s to upgrade releases...", release.ReleaseName)
upgradeReleases = append(upgradeReleases, release)
// Check if installing/upgrading the chart could be destructive
// If so, we don't want to perform any actions,
// unless the user confirms it to be OK.
if !allowDestructive &&
release.ReleaseName == certManagerInfo.releaseName {
return ErrConfirmationMissing
}
}
}
// Backup CRDs and CRs if we are upgrading anything.
if len(upgradeReleases) != 0 {
c.log.Debugf("Creating backup of CRDs and CRs")
crds, err := c.backupCRDs(ctx, upgradeDir)
if err != nil {
return fmt.Errorf("creating CRD backup: %w", err)
}
if err := c.backupCRs(ctx, crds, upgradeDir); err != nil {
return fmt.Errorf("creating CR backup: %w", err)
}
}
for _, release := range upgradeReleases {
c.log.Debugf("Upgrading release %s", release.Chart.Metadata.Name)
if release.ReleaseName == constellationOperatorsInfo.releaseName {
if err := c.updateCRDs(ctx, release.Chart); err != nil {
return fmt.Errorf("updating operator CRDs: %w", err)
}
}
if err := c.upgradeRelease(ctx, timeout, release); err != nil {
return fmt.Errorf("upgrading %s: %w", release.Chart.Metadata.Name, err)
}
}
// Install new releases after upgrading existing ones.
// This makes sure if a release was removed as a dependency from one chart,
// and then added as a new standalone chart (or as a dependency of another chart),
// that the new release is installed without creating naming conflicts.
// If in the future, we require to install a new release before upgrading existing ones,
// it should be done in a separate loop, instead of moving this one up.
for _, release := range newReleases {
c.log.Debugf("Installing new release %s", release.Chart.Metadata.Name)
if err := c.installNewRelease(ctx, timeout, release); err != nil {
return fmt.Errorf("upgrading %s: %w", release.Chart.Metadata.Name, err)
}
}
return errors.Join(upgradeErrs...)
}
func getManagedReleases(config *config.Config, releases *Releases) []Release {
res := []Release{releases.Cilium, releases.CertManager, releases.ConstellationOperators, releases.ConstellationServices}
if config.GetProvider() == cloudprovider.AWS {
res = append(res, *releases.AWSLoadBalancerController)
}
if config.DeployCSIDriver() {
res = append(res, *releases.CSI)
}
return res
}
// Versions queries the cluster for running versions and returns a map of releaseName -> version.
func (c *UpgradeClient) Versions() (ServiceVersions, error) {
ciliumVersion, err := c.currentVersion(ciliumInfo.releaseName)
if err != nil {
return ServiceVersions{}, fmt.Errorf("getting %s version: %w", ciliumInfo.releaseName, err)
}
certManagerVersion, err := c.currentVersion(certManagerInfo.releaseName)
if err != nil {
return ServiceVersions{}, fmt.Errorf("getting %s version: %w", certManagerInfo.releaseName, err)
}
operatorsVersion, err := c.currentVersion(constellationOperatorsInfo.releaseName)
if err != nil {
return ServiceVersions{}, fmt.Errorf("getting %s version: %w", constellationOperatorsInfo.releaseName, err)
}
servicesVersion, err := c.currentVersion(constellationServicesInfo.releaseName)
if err != nil {
return ServiceVersions{}, fmt.Errorf("getting %s version: %w", constellationServicesInfo.releaseName, err)
}
csiVersions, err := c.csiVersions()
if err != nil {
return ServiceVersions{}, fmt.Errorf("getting CSI versions: %w", err)
}
serviceVersions := ServiceVersions{
cilium: ciliumVersion,
certManager: certManagerVersion,
constellationOperators: operatorsVersion,
constellationServices: servicesVersion,
csiVersions: csiVersions,
}
if awsLBVersion, err := c.currentVersion(awsLBControllerInfo.releaseName); err == nil {
serviceVersions.awsLBController = awsLBVersion
} else if !errors.Is(err, errReleaseNotFound) {
return ServiceVersions{}, fmt.Errorf("getting %s version: %w", awsLBControllerInfo.releaseName, err)
}
return serviceVersions, nil
}
// currentVersion returns the version of the currently installed helm release.
func (c *UpgradeClient) currentVersion(release string) (semver.Semver, error) {
rel, err := c.actions.listAction(release)
if err != nil {
return semver.Semver{}, err
}
if len(rel) == 0 {
return semver.Semver{}, errReleaseNotFound
}
if len(rel) > 1 {
return semver.Semver{}, fmt.Errorf("multiple releases found for %s", release)
}
if rel[0] == nil || rel[0].Chart == nil || rel[0].Chart.Metadata == nil {
return semver.Semver{}, fmt.Errorf("received invalid release %s", release)
}
return semver.New(rel[0].Chart.Metadata.Version)
}
func (c *UpgradeClient) csiVersions() (map[string]semver.Semver, error) {
packedChartRelease, err := c.actions.listAction(csiInfo.releaseName)
if err != nil {
return nil, fmt.Errorf("listing %s: %w", csiInfo.releaseName, err)
}
csiVersions := make(map[string]semver.Semver)
// No CSI driver installed
if len(packedChartRelease) == 0 {
return csiVersions, nil
}
if len(packedChartRelease) > 1 {
return nil, fmt.Errorf("multiple releases found for %s", csiInfo.releaseName)
}
if packedChartRelease[0] == nil || packedChartRelease[0].Chart == nil {
return nil, fmt.Errorf("received invalid release %s", csiInfo.releaseName)
}
dependencies := packedChartRelease[0].Chart.Metadata.Dependencies
for _, dep := range dependencies {
var err error
csiVersions[dep.Name], err = semver.New(dep.Version)
if err != nil {
return nil, fmt.Errorf("parsing CSI version %q: %w", dep.Name, err)
}
}
return csiVersions, nil
}
// installNewRelease installs a previously not installed release on the cluster.
func (c *UpgradeClient) installNewRelease(
ctx context.Context, timeout time.Duration, release Release,
) error {
return c.actions.installAction(ctx, release.ReleaseName, release.Chart, release.Values, timeout)
}
// upgradeRelease upgrades a release running on the cluster.
func (c *UpgradeClient) upgradeRelease(
ctx context.Context, timeout time.Duration, release Release,
) error {
return c.actions.upgradeAction(ctx, release.ReleaseName, release.Chart, release.Values, timeout)
}
// GetValues queries the cluster for the values of the given release.
func (c *UpgradeClient) GetValues(release string) (map[string]any, error) {
client := action.NewGetValues(c.config)
// Version corresponds to the releases revision. Specifying a Version <= 0 yields the latest release.
client.Version = 0
values, err := client.Run(release)
if err != nil {
return nil, fmt.Errorf("getting values for %s: %w", release, err)
}
return values, nil
}
// updateCRDs walks through the dependencies of the given chart and applies
// the files in the dependencie's 'crds' folder.
// This function is NOT recursive!
func (c *UpgradeClient) updateCRDs(ctx context.Context, chart *chart.Chart) error {
for _, dep := range chart.Dependencies() {
for _, crdFile := range dep.Files {
if strings.HasPrefix(crdFile.Name, "crds/") {
c.log.Debugf("Updating crd: %s", crdFile.Name)
err := c.kubectl.ApplyCRD(ctx, crdFile.Data)
if err != nil {
return err
}
}
}
}
return nil
}
type crdClient interface {
Initialize(kubeconfig []byte) error
ApplyCRD(ctx context.Context, rawCRD []byte) error
ListCRDs(ctx context.Context) ([]apiextensionsv1.CustomResourceDefinition, error)
ListCRs(ctx context.Context, gvr schema.GroupVersionResource) ([]unstructured.Unstructured, error)
}
type actionWrapper interface {
listAction(release string) ([]*release.Release, error)
getValues(release string) (map[string]any, error)
installAction(ctx context.Context, releaseName string, chart *chart.Chart, values map[string]any, timeout time.Duration) error
upgradeAction(ctx context.Context, releaseName string, chart *chart.Chart, values map[string]any, timeout time.Duration) error
}
type actions struct {
config *action.Configuration
}
// listAction execute a List action by wrapping helm's action package.
// It creates the action, runs it at returns results and errors.
func (a actions) listAction(release string) ([]*release.Release, error) {
action := action.NewList(a.config)
action.Filter = release
return action.Run()
}
func (a actions) getValues(release string) (map[string]any, error) {
client := action.NewGetValues(a.config)
// Version corresponds to the releases revision. Specifying a Version <= 0 yields the latest release.
client.Version = 0
return client.Run(release)
}
func (a actions) upgradeAction(ctx context.Context, releaseName string, chart *chart.Chart, values map[string]any, timeout time.Duration) error {
action := action.NewUpgrade(a.config)
action.Atomic = true
action.Namespace = constants.HelmNamespace
action.ReuseValues = false
action.Timeout = timeout
if _, err := action.RunWithContext(ctx, releaseName, chart, values); err != nil {
return fmt.Errorf("upgrading %s: %w", releaseName, err)
}
return nil
}
func (a actions) installAction(ctx context.Context, releaseName string, chart *chart.Chart, values map[string]any, timeout time.Duration) error {
action := action.NewInstall(a.config)
action.Atomic = true
action.Namespace = constants.HelmNamespace
action.ReleaseName = releaseName
action.Timeout = timeout
if _, err := action.RunWithContext(ctx, chart, values); err != nil {
return fmt.Errorf("installing previously not installed chart %s: %w", chart.Name(), err)
}
return nil
}
// isCLIVersionedRelease checks if the given release is versioned by the CLI,
// meaning that the version of the Helm release is equal to the version of the CLI that installed it.
func isCLIVersionedRelease(releaseName string) bool {
return releaseName == constellationOperatorsInfo.releaseName ||
releaseName == constellationServicesInfo.releaseName ||
releaseName == csiInfo.releaseName
}

112
cli/internal/helm/upgrade_test.go
View file

@ -1,112 +0,0 @@
/*
Copyright (c) Edgeless Systems GmbH
SPDX-License-Identifier: AGPL-3.0-only
*/
package helm
import (
"context"
"testing"
"time"
"github.com/edgelesssys/constellation/v2/internal/compatibility"
"github.com/edgelesssys/constellation/v2/internal/logger"
"github.com/edgelesssys/constellation/v2/internal/semver"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"helm.sh/helm/v3/pkg/chart"
"helm.sh/helm/v3/pkg/release"
)
func TestShouldUpgrade(t *testing.T) {
testCases := map[string]struct {
version string
assertCorrectError func(t *testing.T, err error) bool
wantError bool
}{
"valid upgrade": {
version: "1.9.0",
},
"not a valid upgrade": {
version: "1.0.0",
assertCorrectError: func(t *testing.T, err error) bool {
var target *compatibility.InvalidUpgradeError
return assert.ErrorAs(t, err, &target)
},
wantError: true,
},
}
for name, tc := range testCases {
t.Run(name, func(t *testing.T) {
assert := assert.New(t)
require := require.New(t)
client := UpgradeClient{kubectl: nil, actions: &stubActionWrapper{version: tc.version}, log: logger.NewTest(t)}
chart, err := loadChartsDir(helmFS, certManagerInfo.path)
require.NoError(err)
chartVersion, err := semver.New(chart.Metadata.Version)
require.NoError(err)
err = client.shouldUpgrade(certManagerInfo.releaseName, chartVersion, false)
if tc.wantError {
tc.assertCorrectError(t, err)
return
}
assert.NoError(err)
})
}
}
func TestUpgradeRelease(t *testing.T) {
testCases := map[string]struct {
version string
wantError bool
}{
"allow": {
version: "1.9.0",
},
}
for name, tc := range testCases {
t.Run(name, func(t *testing.T) {
assert := assert.New(t)
require := require.New(t)
client := UpgradeClient{kubectl: nil, actions: &stubActionWrapper{version: tc.version}, log: logger.NewTest(t)}
chart, err := loadChartsDir(helmFS, certManagerInfo.path)
require.NoError(err)
err = client.upgradeRelease(context.Background(), 0, Release{Chart: chart})
if tc.wantError {
assert.Error(err)
return
}
assert.NoError(err)
})
}
}
type stubActionWrapper struct {
version string
}
// listAction returns a list of len 1 with a release that has only it's version set.
func (a *stubActionWrapper) listAction(_ string) ([]*release.Release, error) {
return []*release.Release{{Chart: &chart.Chart{Metadata: &chart.Metadata{Version: a.version}}}}, nil
}
func (a *stubActionWrapper) getValues(_ string) (map[string]any, error) {
return nil, nil
}
func (a *stubActionWrapper) installAction(_ context.Context, _ string, _ *chart.Chart, _ map[string]any, _ time.Duration) error {
return nil
}
func (a *stubActionWrapper) upgradeAction(_ context.Context, _ string, _ *chart.Chart, _ map[string]any, _ time.Duration) error {
return nil
}

142
cli/internal/helm/versionlister.go Normal file
View file

@ -0,0 +1,142 @@
/*
Copyright (c) Edgeless Systems GmbH
SPDX-License-Identifier: AGPL-3.0-only
*/
package helm
import (
"errors"
"fmt"
"github.com/edgelesssys/constellation/v2/internal/semver"
"helm.sh/helm/v3/pkg/action"
"helm.sh/helm/v3/pkg/release"
"k8s.io/client-go/util/retry"
)
// ReleaseVersionClient is a client that can retrieve the version of a helm release.
type ReleaseVersionClient struct {
config *action.Configuration
}
// NewReleaseVersionClient creates a new ReleaseVersionClient.
func NewReleaseVersionClient(kubeConfigPath string, log debugLog) (*ReleaseVersionClient, error) {
config, err := newActionConfig(kubeConfigPath, log)
if err != nil {
return nil, err
}
return &ReleaseVersionClient{
config: config,
}, nil
}
// listAction execute a List action by wrapping helm's action package.
// It creates the action, runs it at returns results and errors.
func (c ReleaseVersionClient) listAction(release string) (res []*release.Release, err error) {
action := action.NewList(c.config)
action.Filter = release
// during init, the kube API might not yet be reachable, so we retry
err = retry.OnError(retry.DefaultBackoff, func(err error) bool {
return err != nil
}, func() error {
res, err = action.Run()
return err
})
return
}
// Versions queries the cluster for running versions and returns a map of releaseName -> version.
func (c ReleaseVersionClient) Versions() (ServiceVersions, error) {
ciliumVersion, err := c.currentVersion(ciliumInfo.releaseName)
if err != nil {
return ServiceVersions{}, fmt.Errorf("getting %s version: %w", ciliumInfo.releaseName, err)
}
certManagerVersion, err := c.currentVersion(certManagerInfo.releaseName)
if err != nil {
return ServiceVersions{}, fmt.Errorf("getting %s version: %w", certManagerInfo.releaseName, err)
}
operatorsVersion, err := c.currentVersion(constellationOperatorsInfo.releaseName)
if err != nil {
return ServiceVersions{}, fmt.Errorf("getting %s version: %w", constellationOperatorsInfo.releaseName, err)
}
servicesVersion, err := c.currentVersion(constellationServicesInfo.releaseName)
if err != nil {
return ServiceVersions{}, fmt.Errorf("getting %s version: %w", constellationServicesInfo.releaseName, err)
}
csiVersions, err := c.csiVersions()
if err != nil {
return ServiceVersions{}, fmt.Errorf("getting CSI versions: %w", err)
}
serviceVersions := ServiceVersions{
cilium: ciliumVersion,
certManager: certManagerVersion,
constellationOperators: operatorsVersion,
constellationServices: servicesVersion,
csiVersions: csiVersions,
}
if awsLBVersion, err := c.currentVersion(awsLBControllerInfo.releaseName); err == nil {
serviceVersions.awsLBController = awsLBVersion
} else if !errors.Is(err, errReleaseNotFound) {
return ServiceVersions{}, fmt.Errorf("getting %s version: %w", awsLBControllerInfo.releaseName, err)
}
return serviceVersions, nil
}
// currentVersion returns the version of the currently installed helm release.
// If the CSI chart is not installed, no error is returned because the user can configure if the chart should be installed.
func (c ReleaseVersionClient) currentVersion(release string) (semver.Semver, error) {
rel, err := c.listAction(release)
if err != nil {
return semver.Semver{}, err
}
if len(rel) == 0 {
return semver.Semver{}, errReleaseNotFound
}
if len(rel) > 1 {
return semver.Semver{}, fmt.Errorf("multiple releases found for %s", release)
}
if rel[0] == nil || rel[0].Chart == nil || rel[0].Chart.Metadata == nil {
return semver.Semver{}, fmt.Errorf("received invalid release %s", release)
}
return semver.New(rel[0].Chart.Metadata.Version)
}
// csi versions needs special handling because all versions of its subcharts should be gathered.
func (c ReleaseVersionClient) csiVersions() (map[string]semver.Semver, error) {
packedChartRelease, err := c.listAction(csiInfo.releaseName)
if err != nil {
return nil, fmt.Errorf("listing %s: %w", csiInfo.releaseName, err)
}
csiVersions := make(map[string]semver.Semver)
// No CSI driver installed
if len(packedChartRelease) == 0 {
return csiVersions, nil
}
if len(packedChartRelease) > 1 {
return nil, fmt.Errorf("multiple releases found for %s", csiInfo.releaseName)
}
if packedChartRelease[0] == nil || packedChartRelease[0].Chart == nil {
return nil, fmt.Errorf("received invalid release %s", csiInfo.releaseName)
}
dependencies := packedChartRelease[0].Chart.Metadata.Dependencies
for _, dep := range dependencies {
var err error
csiVersions[dep.Name], err = semver.New(dep.Version)
if err != nil {
return nil, fmt.Errorf("parsing CSI version %q: %w", dep.Name, err)
}
}
return csiVersions, nil
}