Git-Mirrors/constellation
1
0
Fork 0
mirror of https://github.com/edgelesssys/constellation.git synced 2024-12-28 00:49:26 -05:00
Code Issues Packages Projects Releases Wiki Activity

ref: decouple helm from config (#2669)

Browse Source
This commit is contained in:
Adrian Stobbe 2023-12-01 12:51:51 +01:00 committed by GitHub
parent 8532d1ff02
commit c2d1a7b7fb
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
10 changed files with 73 additions and 52 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
cli/internal/cmd/applyhelm.go
View File

@ -34,6 +34,7 @@ func (a *applyCmd) runHelmApply(
}
options := helm.Options{
DeployCSIDriver: conf.DeployCSIDriver(),
Force: a.flags.force,
Conformance: a.flags.conformance,
HelmWaitMode: a.flags.helmWaitMode,
@ -52,7 +53,8 @@ func (a *applyCmd) runHelmApply(
}
a.log.Debugf("Preparing Helm charts")
executor, includesUpgrades, err := helmApplier.PrepareApply(conf, stateFile, options, serviceAccURI, masterSecret)
executor, includesUpgrades, err := helmApplier.PrepareApply(conf.GetProvider(), conf.GetAttestationConfig().GetVariant(),
conf.KubernetesVersion, conf.MicroserviceVersion, stateFile, options, serviceAccURI, masterSecret, conf.Provider.OpenStack)
if errors.Is(err, helm.ErrConfirmationMissing) {
if !a.flags.yes {
cmd.PrintErrln("WARNING: Upgrading cert-manager will destroy all custom resources you have manually created that are based on the current version of cert-manager.")
@ -66,7 +68,8 @@ func (a *applyCmd) runHelmApply(
}
}
options.AllowDestructive = helm.AllowDestructive
executor, includesUpgrades, err = helmApplier.PrepareApply(conf, stateFile, options, serviceAccURI, masterSecret)
executor, includesUpgrades, err = helmApplier.PrepareApply(conf.GetProvider(), conf.GetAttestationConfig().GetVariant(),
conf.KubernetesVersion, conf.MicroserviceVersion, stateFile, options, serviceAccURI, masterSecret, conf.Provider.OpenStack)
}
var upgradeErr *compatibility.InvalidUpgradeError
if err != nil {

10
cli/internal/cmd/init.go
View File

@ -23,13 +23,17 @@ import (
"sigs.k8s.io/yaml"
"github.com/edgelesssys/constellation/v2/bootstrapper/initproto"
"github.com/edgelesssys/constellation/v2/internal/attestation/variant"
"github.com/edgelesssys/constellation/v2/internal/cloud/cloudprovider"
"github.com/edgelesssys/constellation/v2/internal/config"
"github.com/edgelesssys/constellation/v2/internal/constants"
"github.com/edgelesssys/constellation/v2/internal/file"
"github.com/edgelesssys/constellation/v2/internal/grpc/grpclog"
"github.com/edgelesssys/constellation/v2/internal/helm"
"github.com/edgelesssys/constellation/v2/internal/kms/uri"
"github.com/edgelesssys/constellation/v2/internal/semver"
"github.com/edgelesssys/constellation/v2/internal/state"
"github.com/edgelesssys/constellation/v2/internal/versions"
)
// NewInitCmd returns a new cobra.Command for the init command.
@ -270,7 +274,9 @@ func (e *nonRetriableError) Unwrap() error {
}
type helmApplier interface {
PrepareApply(conf *config.Config, stateFile *state.State,
flags helm.Options, serviceAccURI string, masterSecret uri.MasterSecret) (
PrepareApply(
csp cloudprovider.Provider, attestationVariant variant.Variant, k8sVersion versions.ValidK8sVersion, microserviceVersion semver.Semver, stateFile *state.State,
flags helm.Options, serviceAccURI string, masterSecret uri.MasterSecret, openStackCfg *config.OpenStackConfig,
) (
helm.Applier, bool, error)
}

2
cli/internal/cmd/init_test.go
View File

@ -313,7 +313,7 @@ type stubApplier struct {
err error
}
func (s stubApplier) PrepareApply(_ *config.Config, _ *state.State, _ helm.Options, _ string, _ uri.MasterSecret) (helm.Applier, bool, error) {
func (s stubApplier) PrepareApply(_ cloudprovider.Provider, _ variant.Variant, _ versions.ValidK8sVersion, _ semver.Semver, _ *state.State, _ helm.Options, _ string, _ uri.MasterSecret, _ *config.OpenStackConfig) (helm.Applier, bool, error) {
return stubRunner{}, false, s.err
}

6
cli/internal/cmd/upgradeapply_test.go
View File

@ -361,9 +361,9 @@ type mockApplier struct {
mock.Mock
}
func (m *mockApplier) PrepareApply(cfg *config.Config, stateFile *state.State,
helmOpts helm.Options, str string, masterSecret uri.MasterSecret,
func (m *mockApplier) PrepareApply(csp cloudprovider.Provider, variant variant.Variant, k8sVersion versions.ValidK8sVersion, microserviceVersion semver.Semver, stateFile *state.State,
helmOpts helm.Options, str string, masterSecret uri.MasterSecret, openStackCfg *config.OpenStackConfig,
) (helm.Applier, bool, error) {
args := m.Called(cfg, stateFile, helmOpts, str, masterSecret)
args := m.Called(csp, variant, k8sVersion, microserviceVersion, stateFile, helmOpts, str, masterSecret, openStackCfg)
return args.Get(0).(helm.Applier), args.Bool(1), args.Error(2)
}

3
internal/helm/BUILD.bazel
View File

@ -457,6 +457,7 @@ go_library(
importpath = "github.com/edgelesssys/constellation/v2/internal/helm",
visibility = ["//:__subpackages__"],
deps = [
"//internal/attestation/variant",
"//internal/cloud/azureshared",
"//internal/cloud/cloudprovider",
"//internal/cloud/gcpshared",
@ -501,6 +502,7 @@ go_test(
embed = [":helm"],
deps = [
"//internal/attestation/measurements",
"//internal/attestation/variant",
"//internal/cloud/azureshared",
"//internal/cloud/cloudprovider",
"//internal/cloud/gcpshared",
@ -510,6 +512,7 @@ go_test(
"//internal/logger",
"//internal/semver",
"//internal/state",
"//internal/versions",
"@com_github_pkg_errors//:errors",
"@com_github_stretchr_testify//assert",
"@com_github_stretchr_testify//mock",

22
internal/helm/helm.go
View File

@ -33,6 +33,8 @@ import (
"fmt"
"time"
"github.com/edgelesssys/constellation/v2/internal/attestation/variant"
"github.com/edgelesssys/constellation/v2/internal/cloud/cloudprovider"
"github.com/edgelesssys/constellation/v2/internal/config"
"github.com/edgelesssys/constellation/v2/internal/constants"
"github.com/edgelesssys/constellation/v2/internal/file"
@ -40,6 +42,7 @@ import (
"github.com/edgelesssys/constellation/v2/internal/kubernetes/kubectl"
"github.com/edgelesssys/constellation/v2/internal/semver"
"github.com/edgelesssys/constellation/v2/internal/state"
"github.com/edgelesssys/constellation/v2/internal/versions"
)
const (
@ -80,36 +83,37 @@ func NewClient(kubeConfig []byte, log debugLog) (*Client, error) {
type Options struct {
Conformance bool
HelmWaitMode WaitMode
DeployCSIDriver bool
AllowDestructive bool
Force bool
ApplyTimeout time.Duration
}
// PrepareApply loads the charts and returns the executor to apply them.
// TODO(elchead): remove validK8sVersion by putting ValidK8sVersion into config.Config, see AB#3374.
func (h Client) PrepareApply(
conf *config.Config, stateFile *state.State,
flags Options, serviceAccURI string, masterSecret uri.MasterSecret,
csp cloudprovider.Provider, attestationVariant variant.Variant, k8sVersion versions.ValidK8sVersion,
microserviceVersion semver.Semver, stateFile *state.State, flags Options, serviceAccURI string,
masterSecret uri.MasterSecret, openStackCfg *config.OpenStackConfig,
) (Applier, bool, error) {
releases, err := h.loadReleases(conf, masterSecret, stateFile, flags, serviceAccURI)
releases, err := h.loadReleases(csp, attestationVariant, k8sVersion, masterSecret, stateFile, flags, serviceAccURI, openStackCfg)
if err != nil {
return nil, false, fmt.Errorf("loading Helm releases: %w", err)
}
h.log.Debugf("Loaded Helm releases")
actions, includesUpgrades, err := h.factory.GetActions(
releases, conf.MicroserviceVersion, flags.Force, flags.AllowDestructive, flags.ApplyTimeout,
releases, microserviceVersion, flags.Force, flags.AllowDestructive, flags.ApplyTimeout,
)
return &ChartApplyExecutor{actions: actions, log: h.log}, includesUpgrades, err
}
func (h Client) loadReleases(
conf *config.Config, secret uri.MasterSecret,
stateFile *state.State, flags Options, serviceAccURI string,
csp cloudprovider.Provider, attestationVariant variant.Variant, k8sVersion versions.ValidK8sVersion, secret uri.MasterSecret,
stateFile *state.State, flags Options, serviceAccURI string, openStackCfg *config.OpenStackConfig,
) ([]release, error) {
helmLoader := newLoader(conf, stateFile, h.cliVersion)
helmLoader := newLoader(csp, attestationVariant, k8sVersion, stateFile, h.cliVersion)
h.log.Debugf("Created new Helm loader")
return helmLoader.loadReleases(flags.Conformance, flags.HelmWaitMode, secret, serviceAccURI)
return helmLoader.loadReleases(flags.Conformance, flags.DeployCSIDriver, flags.HelmWaitMode, secret, serviceAccURI, openStackCfg)
}
// Applier runs the Helm actions.

12
internal/helm/helm_test.go
View File

@ -10,13 +10,14 @@ import (
"errors"
"testing"
"github.com/edgelesssys/constellation/v2/internal/attestation/variant"
"github.com/edgelesssys/constellation/v2/internal/cloud/cloudprovider"
"github.com/edgelesssys/constellation/v2/internal/compatibility"
"github.com/edgelesssys/constellation/v2/internal/config"
"github.com/edgelesssys/constellation/v2/internal/kms/uri"
"github.com/edgelesssys/constellation/v2/internal/logger"
"github.com/edgelesssys/constellation/v2/internal/semver"
"github.com/edgelesssys/constellation/v2/internal/state"
"github.com/edgelesssys/constellation/v2/internal/versions"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/mock"
"helm.sh/helm/v3/pkg/action"
@ -121,6 +122,7 @@ func TestMergeMaps(t *testing.T) {
func TestHelmApply(t *testing.T) {
cliVersion := semver.NewFromInt(1, 99, 0, "")
csp := cloudprovider.AWS // using AWS since it has an additional chart: aws-load-balancer-controller
attestationVariant := variant.AWSSEVSNP{}
microserviceCharts := []string{
"constellation-services",
"constellation-operators",
@ -171,11 +173,9 @@ func TestHelmApply(t *testing.T) {
},
}
cfg := config.Default()
cfg.RemoveProviderAndAttestationExcept(csp)
cfg.MicroserviceVersion = cliVersion
log := logger.NewTest(t)
options := Options{
DeployCSIDriver: true,
Conformance: false,
HelmWaitMode: WaitModeWait,
AllowDestructive: true,
@ -206,12 +206,12 @@ func TestHelmApply(t *testing.T) {
helmListVersion(lister, "aws-load-balancer-controller", awsLbVersion)
options.AllowDestructive = tc.allowDestructive
ex, includesUpgrade, err := sut.PrepareApply(cfg,
ex, includesUpgrade, err := sut.PrepareApply(csp, attestationVariant, versions.Default, cliVersion,
state.New().
SetInfrastructure(state.Infrastructure{UID: "testuid"}).
SetClusterValues(state.ClusterValues{MeasurementSalt: []byte{0x41}}),
options, fakeServiceAccURI(csp),
uri.MasterSecret{Key: []byte("secret"), Salt: []byte("masterSalt")})
uri.MasterSecret{Key: []byte("secret"), Salt: []byte("masterSalt")}, nil)
var upgradeErr *compatibility.InvalidUpgradeError
if tc.expectError {
assert.Error(t, err)

24
internal/helm/loader.go
View File

@ -19,6 +19,7 @@ import (
"helm.sh/helm/v3/pkg/chart"
"helm.sh/helm/v3/pkg/chart/loader"
"github.com/edgelesssys/constellation/v2/internal/attestation/variant"
"github.com/edgelesssys/constellation/v2/internal/cloud/cloudprovider"
"github.com/edgelesssys/constellation/v2/internal/config"
"github.com/edgelesssys/constellation/v2/internal/constants"
@ -59,7 +60,7 @@ var (
// chartLoader loads embedded helm charts.
type chartLoader struct {
csp cloudprovider.Provider
config *config.Config
attestationVariant variant.Variant
joinServiceImage string
keyServiceImage string
ccmImage string // cloud controller manager image
@ -76,12 +77,10 @@ type chartLoader struct {
}
// newLoader creates a new ChartLoader.
func newLoader(config *config.Config, stateFile *state.State, cliVersion semver.Semver) *chartLoader {
func newLoader(csp cloudprovider.Provider, attestationVariant variant.Variant, k8sVersion versions.ValidK8sVersion, stateFile *state.State, cliVersion semver.Semver) *chartLoader {
// TODO(malt3): Allow overriding container image registry + prefix for all images
// (e.g. for air-gapped environments).
var ccmImage, cnmImage string
csp := config.GetProvider()
k8sVersion := config.KubernetesVersion
switch csp {
case cloudprovider.AWS:
ccmImage = versions.VersionConfigs[k8sVersion].CloudControllerManagerImageAWS
@ -96,10 +95,10 @@ func newLoader(config *config.Config, stateFile *state.State, cliVersion semver.
return &chartLoader{
cliVersion: cliVersion,
csp: csp,
attestationVariant: attestationVariant,
stateFile: stateFile,
ccmImage: ccmImage,
azureCNMImage: cnmImage,
config: config,
joinServiceImage: imageversion.JoinService("", ""),
keyServiceImage: imageversion.KeyService("", ""),
autoscalerImage: versions.VersionConfigs[k8sVersion].ClusterAutoscalerImage,
@ -118,14 +117,14 @@ func newLoader(config *config.Config, stateFile *state.State, cliVersion semver.
type releaseApplyOrder []release
// loadReleases loads the embedded helm charts and returns them as a HelmReleases object.
func (i *chartLoader) loadReleases(conformanceMode bool, helmWaitMode WaitMode, masterSecret uri.MasterSecret,
serviceAccURI string,
func (i *chartLoader) loadReleases(conformanceMode, deployCSIDriver bool, helmWaitMode WaitMode, masterSecret uri.MasterSecret,
serviceAccURI string, openStackCfg *config.OpenStackConfig,
) (releaseApplyOrder, error) {
ciliumRelease, err := i.loadRelease(ciliumInfo, helmWaitMode)
if err != nil {
return nil, fmt.Errorf("loading cilium: %w", err)
}
ciliumVals := extraCiliumValues(i.config.GetProvider(), conformanceMode, i.stateFile.Infrastructure)
ciliumVals := extraCiliumValues(i.csp, conformanceMode, i.stateFile.Infrastructure)
ciliumRelease.values = mergeMaps(ciliumRelease.values, ciliumVals)
certManagerRelease, err := i.loadRelease(certManagerInfo, helmWaitMode)
@ -144,26 +143,27 @@ func (i *chartLoader) loadReleases(conformanceMode bool, helmWaitMode WaitMode,
return nil, fmt.Errorf("loading constellation-services: %w", err)
}
svcVals, err := extraConstellationServicesValues(i.config, masterSecret, serviceAccURI, i.stateFile.Infrastructure)
svcVals, err := extraConstellationServicesValues(i.csp, i.attestationVariant, masterSecret,
serviceAccURI, i.stateFile.Infrastructure, openStackCfg)
if err != nil {
return nil, fmt.Errorf("extending constellation-services values: %w", err)
}
conServicesRelease.values = mergeMaps(conServicesRelease.values, svcVals)
releases := releaseApplyOrder{ciliumRelease, conServicesRelease, certManagerRelease}
if i.config.DeployCSIDriver() {
if deployCSIDriver {
csiRelease, err := i.loadRelease(csiInfo, helmWaitMode)
if err != nil {
return nil, fmt.Errorf("loading snapshot CRDs: %w", err)
}
extraCSIvals, err := extraCSIValues(i.config.GetProvider(), serviceAccURI)
extraCSIvals, err := extraCSIValues(i.csp, serviceAccURI)
if err != nil {
return nil, fmt.Errorf("extending CSI values: %w", err)
}
csiRelease.values = mergeMaps(csiRelease.values, extraCSIvals)
releases = append(releases, csiRelease)
}
if i.config.HasProvider(cloudprovider.AWS) {
if i.csp == cloudprovider.AWS {
awsRelease, err := i.loadRelease(awsLBControllerInfo, helmWaitMode)
if err != nil {
return nil, fmt.Errorf("loading aws-services: %w", err)

14
internal/helm/loader_test.go
View File

@ -23,6 +23,7 @@ import (
"helm.sh/helm/v3/pkg/engine"
"github.com/edgelesssys/constellation/v2/internal/attestation/measurements"
"github.com/edgelesssys/constellation/v2/internal/attestation/variant"
"github.com/edgelesssys/constellation/v2/internal/cloud/azureshared"
"github.com/edgelesssys/constellation/v2/internal/cloud/cloudprovider"
"github.com/edgelesssys/constellation/v2/internal/cloud/gcpshared"
@ -30,6 +31,7 @@ import (
"github.com/edgelesssys/constellation/v2/internal/kms/uri"
"github.com/edgelesssys/constellation/v2/internal/semver"
"github.com/edgelesssys/constellation/v2/internal/state"
"github.com/edgelesssys/constellation/v2/internal/versions"
)
func fakeServiceAccURI(provider cloudprovider.Provider) string {
@ -64,9 +66,8 @@ func fakeServiceAccURI(provider cloudprovider.Provider) string {
func TestLoadReleases(t *testing.T) {
assert := assert.New(t)
require := require.New(t)
config := &config.Config{Provider: config.ProviderConfig{GCP: &config.GCPConfig{}}}
chartLoader := newLoader(
config,
cloudprovider.GCP, variant.GCPSEVES{}, versions.Default,
state.New().
SetInfrastructure(state.Infrastructure{
GCP: &state.GCP{
@ -78,9 +79,9 @@ func TestLoadReleases(t *testing.T) {
semver.NewFromInt(2, 10, 0, ""),
)
helmReleases, err := chartLoader.loadReleases(
true, WaitModeAtomic,
true, false, WaitModeAtomic,
uri.MasterSecret{Key: []byte("secret"), Salt: []byte("masterSalt")},
fakeServiceAccURI(cloudprovider.GCP),
fakeServiceAccURI(cloudprovider.GCP), nil,
)
require.NoError(err)
for _, release := range helmReleases {
@ -92,7 +93,6 @@ func TestLoadReleases(t *testing.T) {
func TestLoadAWSLoadBalancerValues(t *testing.T) {
sut := chartLoader{
config: &config.Config{Name: "testCluster"},
clusterName: "testCluster",
stateFile: state.New().SetInfrastructure(state.Infrastructure{UID: "testuid", Name: "testCluster-testuid"}),
}
@ -180,14 +180,14 @@ func TestConstellationServices(t *testing.T) {
values := chartLoader.loadConstellationServicesValues()
serviceAccURI := fakeServiceAccURI(tc.config.GetProvider())
extraVals, err := extraConstellationServicesValues(
tc.config, uri.MasterSecret{
tc.config.GetProvider(), tc.config.GetAttestationConfig().GetVariant(), uri.MasterSecret{
Key: []byte("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"),
Salt: []byte("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"),
}, serviceAccURI, state.Infrastructure{
UID: "uid",
Azure: &state.Azure{},
GCP: &state.GCP{},
})
}, tc.config.Provider.OpenStack)
require.NoError(err)
values = mergeMaps(values, extraVals)

25
internal/helm/overrides.go
View File

@ -13,6 +13,7 @@ import (
"encoding/json"
"fmt"
"github.com/edgelesssys/constellation/v2/internal/attestation/variant"
"github.com/edgelesssys/constellation/v2/internal/cloud/azureshared"
"github.com/edgelesssys/constellation/v2/internal/cloud/cloudprovider"
"github.com/edgelesssys/constellation/v2/internal/cloud/gcpshared"
@ -64,14 +65,15 @@ func extraCiliumValues(provider cloudprovider.Provider, conformanceMode bool, ou
// extraConstellationServicesValues extends the given values map by some values depending on user input.
// Values set inside this function are only applied during init, not during upgrade.
func extraConstellationServicesValues(
cfg *config.Config, masterSecret uri.MasterSecret, serviceAccURI string, output state.Infrastructure,
csp cloudprovider.Provider, attestationVariant variant.Variant, masterSecret uri.MasterSecret, serviceAccURI string,
output state.Infrastructure, openStackCfg *config.OpenStackConfig,
) (map[string]any, error) {
extraVals := map[string]any{}
extraVals["join-service"] = map[string]any{
"attestationVariant": cfg.GetAttestationConfig().GetVariant().String(),
"attestationVariant": attestationVariant.String(),
}
extraVals["verification-service"] = map[string]any{
"attestationVariant": cfg.GetAttestationConfig().GetVariant().String(),
"attestationVariant": attestationVariant.String(),
}
extraVals["konnectivity"] = map[string]any{
"loadBalancerIP": output.ClusterEndpoint,
@ -81,19 +83,22 @@ func extraConstellationServicesValues(
"masterSecret": base64.StdEncoding.EncodeToString(masterSecret.Key),
"salt": base64.StdEncoding.EncodeToString(masterSecret.Salt),
}
switch cfg.GetProvider() {
switch csp {
case cloudprovider.OpenStack:
extraVals["openstack"] = map[string]any{
"deployYawolLoadBalancer": cfg.DeployYawolLoadBalancer(),
if openStackCfg == nil {
return nil, fmt.Errorf("no OpenStack config")
}
if cfg.DeployYawolLoadBalancer() {
extraVals["openstack"] = map[string]any{
"deployYawolLoadBalancer": openStackCfg.DeployYawolLoadBalancer != nil && *openStackCfg.DeployYawolLoadBalancer,
}
if openStackCfg.DeployYawolLoadBalancer != nil && *openStackCfg.DeployYawolLoadBalancer {
extraVals["yawol-controller"] = map[string]any{
"yawolOSSecretName": "yawolkey",
// has to be larger than ~30s to account for slow OpenStack API calls.
"openstackTimeout": "1m",
"yawolFloatingID": cfg.Provider.OpenStack.FloatingIPPoolID,
"yawolFlavorID": cfg.Provider.OpenStack.YawolFlavorID,
"yawolImageID": cfg.Provider.OpenStack.YawolImageID,
"yawolFloatingID": openStackCfg.FloatingIPPoolID,
"yawolFlavorID": openStackCfg.YawolFlavorID,
"yawolImageID": openStackCfg.YawolImageID,
}
}
case cloudprovider.GCP: