From 70ce195a5f8c283cbfa48396f2ef6516d7a4a594 Mon Sep 17 00:00:00 2001 From: Adrian Stobbe Date: Thu, 3 Aug 2023 13:54:48 +0200 Subject: [PATCH] cli: unify chart value setup (#2153) --- cli/internal/cmd/init.go | 40 ++++-- cli/internal/cmd/init_test.go | 32 ++++- cli/internal/helm/BUILD.bazel | 7 +- cli/internal/helm/init.go | 155 +---------------------- cli/internal/helm/install.go | 11 +- cli/internal/helm/loader.go | 125 +++---------------- cli/internal/helm/loader_test.go | 48 ++++++-- cli/internal/helm/overrides.go | 182 ++++++++++++++++++++++++++++ cli/internal/helm/release.go | 4 +- cli/internal/terraform/terraform.go | 3 + 10 files changed, 310 insertions(+), 297 deletions(-) create mode 100644 cli/internal/helm/overrides.go diff --git a/cli/internal/cmd/init.go b/cli/internal/cmd/init.go index 005c854c7..1c701c526 100644 --- a/cli/internal/cmd/init.go +++ b/cli/internal/cmd/init.go @@ -37,6 +37,7 @@ import ( "github.com/edgelesssys/constellation/v2/cli/internal/cloudcmd" "github.com/edgelesssys/constellation/v2/cli/internal/clusterid" "github.com/edgelesssys/constellation/v2/cli/internal/helm" + "github.com/edgelesssys/constellation/v2/cli/internal/terraform" "github.com/edgelesssys/constellation/v2/internal/cloud/azureshared" "github.com/edgelesssys/constellation/v2/internal/cloud/cloudprovider" "github.com/edgelesssys/constellation/v2/internal/cloud/gcpshared" @@ -78,6 +79,11 @@ type initCmd struct { masterSecret uri.MasterSecret fh *file.Handler helmInstaller helm.Initializer + tfClient showClusterer +} + +type showClusterer interface { + ShowCluster(ctx context.Context, provider cloudprovider.Provider) (terraform.ApplyOutput, error) } // runInitialize runs the initialize command. @@ -105,7 +111,11 @@ func runInitialize(cmd *cobra.Command, _ []string) error { if err != nil { return fmt.Errorf("creating Helm installer: %w", err) } - i := &initCmd{log: log, spinner: spinner, merger: &kubeconfigMerger{log: log}, fh: &fileHandler, helmInstaller: helmInstaller} + tfClient, err := terraform.New(ctx, constants.TerraformWorkingDir) + if err != nil { + return fmt.Errorf("creating Terraform client: %w", err) + } + i := &initCmd{log: log, spinner: spinner, merger: &kubeconfigMerger{log: log}, fh: &fileHandler, helmInstaller: helmInstaller, tfClient: tfClient} fetcher := attestationconfigapi.NewFetcher() return i.initialize(cmd, newDialer, fileHandler, license.NewClient(), fetcher) } @@ -183,17 +193,6 @@ func (i *initCmd) initialize(cmd *cobra.Command, newDialer func(validator atls.V clusterName := clusterid.GetClusterName(conf, idFile) i.log.Debugf("Setting cluster name to %s", clusterName) - helmLoader := helm.NewLoader(provider, k8sVersion, clusterName) - i.log.Debugf("Created new Helm loader") - releases, err := helmLoader.LoadReleases(conf, flags.conformance, flags.helmWaitMode, masterSecret.Key, masterSecret.Salt) - if err != nil { - return fmt.Errorf("loading Helm charts: %w", err) - } - i.log.Debugf("Loaded Helm deployments") - if err != nil { - return fmt.Errorf("loading Helm charts: %w", err) - } - cmd.PrintErrln("Note: If you just created the cluster, it can take a few minutes to connect.") i.spinner.Start("Connecting ", false) req := &initproto.InitRequest{ @@ -230,7 +229,22 @@ func (i *initCmd) initialize(cmd *cobra.Command, newDialer func(validator atls.V if err != nil { return err } - if err := i.helmInstaller.Install(cmd.Context(), provider, masterSecret, idFile, serviceAccURI, releases); err != nil { + + helmLoader := helm.NewLoader(provider, k8sVersion, clusterName) + i.log.Debugf("Created new Helm loader") + output, err := i.tfClient.ShowCluster(cmd.Context(), conf.GetProvider()) + if err != nil { + return fmt.Errorf("getting Terraform output: %w", err) + } + releases, err := helmLoader.LoadReleases(conf, flags.conformance, flags.helmWaitMode, masterSecret.Key, masterSecret.Salt, serviceAccURI, idFile, output) + if err != nil { + return fmt.Errorf("loading Helm charts: %w", err) + } + i.log.Debugf("Loaded Helm deployments") + if err != nil { + return fmt.Errorf("loading Helm charts: %w", err) + } + if err := i.helmInstaller.Install(cmd.Context(), releases); err != nil { return fmt.Errorf("installing Helm charts: %w", err) } cmd.Println(bufferedOutput.String()) diff --git a/cli/internal/cmd/init_test.go b/cli/internal/cmd/init_test.go index 1a1ac03c7..e693ab5ab 100644 --- a/cli/internal/cmd/init_test.go +++ b/cli/internal/cmd/init_test.go @@ -22,6 +22,7 @@ import ( "github.com/edgelesssys/constellation/v2/bootstrapper/initproto" "github.com/edgelesssys/constellation/v2/cli/internal/clusterid" "github.com/edgelesssys/constellation/v2/cli/internal/helm" + "github.com/edgelesssys/constellation/v2/cli/internal/terraform" "github.com/edgelesssys/constellation/v2/internal/atls" "github.com/edgelesssys/constellation/v2/internal/attestation/measurements" "github.com/edgelesssys/constellation/v2/internal/attestation/variant" @@ -54,7 +55,16 @@ func TestInitArgumentValidation(t *testing.T) { func TestInitialize(t *testing.T) { gcpServiceAccKey := &gcpshared.ServiceAccountKey{ - Type: "service_account", + Type: "service_account", + ProjectID: "project_id", + PrivateKeyID: "key_id", + PrivateKey: "key", + ClientEmail: "client_email", + ClientID: "client_id", + AuthURI: "auth_uri", + TokenURI: "token_uri", + AuthProviderX509CertURL: "cert", + ClientX509CertURL: "client_cert", } testInitResp := &initproto.InitSuccessResponse{ Kubeconfig: []byte("kubeconfig"), @@ -175,7 +185,7 @@ func TestInitialize(t *testing.T) { ctx, cancel := context.WithTimeout(ctx, 4*time.Second) defer cancel() cmd.SetContext(ctx) - i := &initCmd{log: logger.NewTest(t), spinner: &nopSpinner{}, helmInstaller: &stubHelmInstaller{}} + i := &initCmd{log: logger.NewTest(t), spinner: &nopSpinner{}, helmInstaller: &stubHelmInstaller{}, tfClient: &stubShowCluster{}} err := i.initialize(cmd, newDialer, fileHandler, &stubLicenseClient{}, stubAttestationFetcher{}) if tc.wantErr { @@ -670,9 +680,19 @@ func (c stubInitClient) Recv() (*initproto.InitResponse, error) { type stubHelmInstaller struct{} -func (i *stubHelmInstaller) Install(_ context.Context, _ cloudprovider.Provider, _ uri.MasterSecret, - _ clusterid.File, - _ string, _ *helm.Releases, -) error { +func (i *stubHelmInstaller) Install(_ context.Context, _ *helm.Releases) error { return nil } + +type stubShowCluster struct{} + +func (s *stubShowCluster) ShowCluster(_ context.Context, csp cloudprovider.Provider) (terraform.ApplyOutput, error) { + res := terraform.ApplyOutput{} + switch csp { + case cloudprovider.Azure: + res.Azure = &terraform.AzureApplyOutput{} + case cloudprovider.GCP: + res.GCP = &terraform.GCPApplyOutput{} + } + return res, nil +} diff --git a/cli/internal/helm/BUILD.bazel b/cli/internal/helm/BUILD.bazel index 08df2735f..4e99761f4 100644 --- a/cli/internal/helm/BUILD.bazel +++ b/cli/internal/helm/BUILD.bazel @@ -10,6 +10,7 @@ go_library( "init.go", "install.go", "loader.go", + "overrides.go", "release.go", "serviceversion.go", "upgrade.go", @@ -430,7 +431,6 @@ go_library( "//internal/config", "//internal/constants", "//internal/file", - "//internal/kms/uri", "//internal/retry", "//internal/semver", "//internal/versions", @@ -449,7 +449,6 @@ go_library( "@sh_helm_helm_v3//pkg/action", "@sh_helm_helm_v3//pkg/chart", "@sh_helm_helm_v3//pkg/chart/loader", - "@sh_helm_helm_v3//pkg/chartutil", "@sh_helm_helm_v3//pkg/cli", "@sh_helm_helm_v3//pkg/release", ], @@ -467,9 +466,12 @@ go_test( embed = [":helm"], deps = [ "//cli/internal/clusterid", + "//cli/internal/terraform", "//internal/attestation/idkeydigest", "//internal/attestation/measurements", + "//internal/cloud/azureshared", "//internal/cloud/cloudprovider", + "//internal/cloud/gcpshared", "//internal/compatibility", "//internal/config", "//internal/file", @@ -485,7 +487,6 @@ go_test( "@io_k8s_apimachinery//pkg/runtime/schema", "@io_k8s_sigs_yaml//:yaml", "@sh_helm_helm_v3//pkg/chart", - "@sh_helm_helm_v3//pkg/chart/loader", "@sh_helm_helm_v3//pkg/chartutil", "@sh_helm_helm_v3//pkg/engine", "@sh_helm_helm_v3//pkg/release", diff --git a/cli/internal/helm/init.go b/cli/internal/helm/init.go index 99b6503ab..144c63855 100644 --- a/cli/internal/helm/init.go +++ b/cli/internal/helm/init.go @@ -7,27 +7,15 @@ package helm import ( "context" - "encoding/base64" - "encoding/json" "fmt" "time" - "github.com/edgelesssys/constellation/v2/cli/internal/clusterid" - "github.com/edgelesssys/constellation/v2/cli/internal/terraform" - "github.com/edgelesssys/constellation/v2/internal/cloud/azureshared" - "github.com/edgelesssys/constellation/v2/internal/cloud/cloudprovider" - "github.com/edgelesssys/constellation/v2/internal/cloud/gcpshared" - "github.com/edgelesssys/constellation/v2/internal/cloud/openstack" "github.com/edgelesssys/constellation/v2/internal/constants" - "github.com/edgelesssys/constellation/v2/internal/kms/uri" ) // Initializer installs all Helm charts required for a constellation cluster. type Initializer interface { - Install(ctx context.Context, provider cloudprovider.Provider, masterSecret uri.MasterSecret, - idFile clusterid.File, - serviceAccURI string, releases *Releases, - ) error + Install(ctx context.Context, releases *Releases) error } type initializationClient struct { @@ -45,20 +33,9 @@ func NewInitializer(log debugLog) (Initializer, error) { } // Install installs all Helm charts required for a constellation cluster. -func (h initializationClient) Install(ctx context.Context, provider cloudprovider.Provider, masterSecret uri.MasterSecret, - idFile clusterid.File, - serviceAccURI string, releases *Releases, +func (h initializationClient) Install(ctx context.Context, releases *Releases, ) error { - tfClient, err := terraform.New(ctx, constants.TerraformWorkingDir) - if err != nil { - return fmt.Errorf("creating Terraform client: %w", err) - } - output, err := tfClient.ShowCluster(ctx, provider) - if err != nil { - return fmt.Errorf("getting Terraform output: %w", err) - } - ciliumVals := setupCiliumVals(provider, output) - if err := h.installer.InstallChartWithValues(ctx, releases.Cilium, ciliumVals); err != nil { + if err := h.installer.InstallChart(ctx, releases.Cilium); err != nil { return fmt.Errorf("installing Cilium: %w", err) } h.log.Debugf("Waiting for Cilium to become ready") @@ -81,11 +58,7 @@ func (h initializationClient) Install(ctx context.Context, provider cloudprovide } h.log.Debugf("Installing microservices") - serviceVals, err := setupMicroserviceVals(provider, masterSecret.Salt, idFile.UID, serviceAccURI, output) - if err != nil { - return fmt.Errorf("setting up microservice values: %w", err) - } - if err := h.installer.InstallChartWithValues(ctx, releases.ConstellationServices, serviceVals); err != nil { + if err := h.installer.InstallChart(ctx, releases.ConstellationServices); err != nil { return fmt.Errorf("installing microservices: %w", err) } @@ -95,22 +68,8 @@ func (h initializationClient) Install(ctx context.Context, provider cloudprovide } if releases.CSI != nil { - var csiVals map[string]any - if provider == cloudprovider.OpenStack { - creds, err := openstack.AccountKeyFromURI(serviceAccURI) - if err != nil { - return err - } - cinderIni := creds.CloudINI().CinderCSIConfiguration() - csiVals = map[string]any{ - "cinder-config": map[string]any{ - "secretData": cinderIni, - }, - } - } - h.log.Debugf("Installing CSI deployments") - if err := h.installer.InstallChartWithValues(ctx, *releases.CSI, csiVals); err != nil { + if err := h.installer.InstallChart(ctx, *releases.CSI); err != nil { return fmt.Errorf("installing CSI snapshot CRDs: %w", err) } } @@ -123,8 +82,7 @@ func (h initializationClient) Install(ctx context.Context, provider cloudprovide } h.log.Debugf("Installing constellation operators") - operatorVals := setupOperatorVals(ctx, idFile.UID) - if err := h.installer.InstallChartWithValues(ctx, releases.ConstellationOperators, operatorVals); err != nil { + if err := h.installer.InstallChart(ctx, releases.ConstellationOperators); err != nil { return fmt.Errorf("installing constellation operators: %w", err) } return nil @@ -136,82 +94,6 @@ type installer interface { InstallChartWithValues(ctx context.Context, release Release, extraValues map[string]any) error } -// TODO(malt3): switch over to DNS name on AWS and Azure -// soon as every apiserver certificate of every control-plane node -// has the dns endpoint in its SAN list. -func setupCiliumVals(provider cloudprovider.Provider, output terraform.ApplyOutput) map[string]any { - vals := map[string]any{ - "k8sServiceHost": output.IP, - "k8sServicePort": constants.KubernetesPort, - } - if provider == cloudprovider.GCP { - vals["ipv4NativeRoutingCIDR"] = output.GCP.IPCidrPod - vals["strictModeCIDR"] = output.GCP.IPCidrPod - } - return vals -} - -// setupMicroserviceVals returns the values for the microservice chart. -func setupMicroserviceVals(provider cloudprovider.Provider, measurementSalt []byte, uid, serviceAccURI string, output terraform.ApplyOutput) (map[string]any, error) { - extraVals := map[string]any{ - "join-service": map[string]any{ - "measurementSalt": base64.StdEncoding.EncodeToString(measurementSalt), - }, - "verification-service": map[string]any{ - "loadBalancerIP": output.IP, - }, - "konnectivity": map[string]any{ - "loadBalancerIP": output.IP, - }, - } - switch provider { - case cloudprovider.GCP: - serviceAccountKey, err := gcpshared.ServiceAccountKeyFromURI(serviceAccURI) - if err != nil { - return nil, fmt.Errorf("getting service account key: %w", err) - } - rawKey, err := json.Marshal(serviceAccountKey) - if err != nil { - return nil, fmt.Errorf("marshaling service account key: %w", err) - } - if output.GCP == nil { - return nil, fmt.Errorf("no GCP output from Terraform") - } - extraVals["ccm"] = map[string]any{ - "GCP": map[string]any{ - "projectID": output.GCP.ProjectID, - "uid": uid, - "secretData": string(rawKey), - "subnetworkPodCIDR": output.GCP.IPCidrPod, - }, - } - case cloudprovider.Azure: - if output.Azure == nil { - return nil, fmt.Errorf("no Azure output from Terraform") - } - ccmConfig, err := getCCMConfig(*output.Azure, serviceAccURI) - if err != nil { - return nil, fmt.Errorf("getting Azure CCM config: %w", err) - } - extraVals["ccm"] = map[string]any{ - "Azure": map[string]any{ - "azureConfig": string(ccmConfig), - }, - } - } - - return extraVals, nil -} - -// setupOperatorVals returns the values for the constellation-operator chart. -func setupOperatorVals(_ context.Context, uid string) map[string]any { - return map[string]any{ - "constellation-operator": map[string]any{ - "constellationUID": uid, - }, - } -} - type cloudConfig struct { Cloud string `json:"cloud,omitempty"` TenantID string `json:"tenantId,omitempty"` @@ -231,28 +113,3 @@ type cloudConfig struct { UseManagedIdentityExtension bool `json:"useManagedIdentityExtension,omitempty"` UserAssignedIdentityID string `json:"userAssignedIdentityID,omitempty"` } - -// GetCCMConfig returns the configuration needed for the Kubernetes Cloud Controller Manager on Azure. -func getCCMConfig(tfOutput terraform.AzureApplyOutput, serviceAccURI string) ([]byte, error) { - creds, err := azureshared.ApplicationCredentialsFromURI(serviceAccURI) - if err != nil { - return nil, fmt.Errorf("getting service account key: %w", err) - } - useManagedIdentityExtension := creds.PreferredAuthMethod == azureshared.AuthMethodUserAssignedIdentity - config := cloudConfig{ - Cloud: "AzurePublicCloud", - TenantID: creds.TenantID, - SubscriptionID: tfOutput.SubscriptionID, - ResourceGroup: tfOutput.ResourceGroup, - LoadBalancerSku: "standard", - SecurityGroupName: tfOutput.NetworkSecurityGroupName, - LoadBalancerName: tfOutput.LoadBalancerName, - UseInstanceMetadata: true, - VMType: "vmss", - Location: creds.Location, - UseManagedIdentityExtension: useManagedIdentityExtension, - UserAssignedIdentityID: tfOutput.UserAssignedIdentity, - } - - return json.Marshal(config) -} diff --git a/cli/internal/helm/install.go b/cli/internal/helm/install.go index 14b91dadf..02851e339 100644 --- a/cli/internal/helm/install.go +++ b/cli/internal/helm/install.go @@ -7,7 +7,6 @@ SPDX-License-Identifier: AGPL-3.0-only package helm import ( - "bytes" "context" "fmt" "strings" @@ -17,7 +16,6 @@ import ( "github.com/edgelesssys/constellation/v2/internal/retry" "helm.sh/helm/v3/pkg/action" "helm.sh/helm/v3/pkg/chart" - "helm.sh/helm/v3/pkg/chart/loader" "helm.sh/helm/v3/pkg/cli" "k8s.io/apimachinery/pkg/util/wait" ) @@ -79,7 +77,7 @@ func (h *Installer) InstallChartWithValues(ctx context.Context, release Release, // install tries to install the given chart and aborts after ~5 tries. // The function will wait 30 seconds before retrying a failed installation attempt. // After 3 tries, the retrier will be canceled and the function returns with an error. -func (h *Installer) install(ctx context.Context, chartRaw []byte, values map[string]any) error { +func (h *Installer) install(ctx context.Context, chart *chart.Chart, values map[string]any) error { var retries int retriable := func(err error) bool { // abort after maximumRetryAttempts tries. @@ -97,13 +95,6 @@ func (h *Installer) install(ctx context.Context, chartRaw []byte, values map[str return wait.Interrupted(err) || strings.Contains(err.Error(), "connection refused") } - - reader := bytes.NewReader(chartRaw) - chart, err := loader.LoadArchive(reader) - if err != nil { - return fmt.Errorf("helm load archive: %w", err) - } - doer := installDoer{ h, chart, diff --git a/cli/internal/helm/loader.go b/cli/internal/helm/loader.go index 0d659cd60..c4c48e885 100644 --- a/cli/internal/helm/loader.go +++ b/cli/internal/helm/loader.go @@ -9,11 +9,8 @@ package helm import ( "bytes" "embed" - "encoding/base64" - "encoding/json" "fmt" "io/fs" - "os" "path/filepath" "strings" @@ -21,9 +18,10 @@ import ( "helm.sh/helm/pkg/ignore" "helm.sh/helm/v3/pkg/chart" "helm.sh/helm/v3/pkg/chart/loader" - "helm.sh/helm/v3/pkg/chartutil" + "github.com/edgelesssys/constellation/v2/cli/internal/clusterid" "github.com/edgelesssys/constellation/v2/cli/internal/helm/imageversion" + "github.com/edgelesssys/constellation/v2/cli/internal/terraform" "github.com/edgelesssys/constellation/v2/internal/cloud/cloudprovider" "github.com/edgelesssys/constellation/v2/internal/config" "github.com/edgelesssys/constellation/v2/internal/constants" @@ -108,12 +106,13 @@ func NewLoader(csp cloudprovider.Provider, k8sVersion versions.ValidK8sVersion, } // LoadReleases loads the embedded helm charts and returns them as a HelmReleases object. -func (i *ChartLoader) LoadReleases(config *config.Config, conformanceMode bool, helmWaitMode WaitMode, masterSecret, salt []byte) (*Releases, error) { +func (i *ChartLoader) LoadReleases(config *config.Config, conformanceMode bool, helmWaitMode WaitMode, masterSecret, salt []byte, serviceAccURI string, idFile clusterid.File, output terraform.ApplyOutput) (*Releases, error) { ciliumRelease, err := i.loadRelease(ciliumInfo, helmWaitMode) if err != nil { return nil, fmt.Errorf("loading cilium: %w", err) } - extendCiliumValues(ciliumRelease.Values, conformanceMode) + ciliumVals := extraCiliumValues(config.GetProvider(), conformanceMode, output) + ciliumRelease.Values = mergeMaps(ciliumRelease.Values, ciliumVals) certManagerRelease, err := i.loadRelease(certManagerInfo, helmWaitMode) if err != nil { @@ -124,14 +123,17 @@ func (i *ChartLoader) LoadReleases(config *config.Config, conformanceMode bool, if err != nil { return nil, fmt.Errorf("loading operators: %w", err) } + operatorRelease.Values = mergeMaps(operatorRelease.Values, extraOperatorValues(idFile.UID)) conServicesRelease, err := i.loadRelease(constellationServicesInfo, helmWaitMode) if err != nil { return nil, fmt.Errorf("loading constellation-services: %w", err) } - if err := extendConstellationServicesValues(conServicesRelease.Values, config, masterSecret, salt); err != nil { + svcVals, err := extraConstellationServicesValues(config, masterSecret, salt, idFile.UID, serviceAccURI, output) + if err != nil { return nil, fmt.Errorf("extending constellation-services values: %w", err) } + conServicesRelease.Values = mergeMaps(conServicesRelease.Values, svcVals) releases := Releases{Cilium: ciliumRelease, CertManager: certManagerRelease, ConstellationOperators: operatorRelease, ConstellationServices: conServicesRelease} if config.HasProvider(cloudprovider.AWS) { @@ -143,11 +145,16 @@ func (i *ChartLoader) LoadReleases(config *config.Config, conformanceMode bool, } if config.DeployCSIDriver() { - csi, err := i.loadRelease(csiInfo, helmWaitMode) + csiRelease, err := i.loadRelease(csiInfo, helmWaitMode) if err != nil { return nil, fmt.Errorf("loading snapshot CRDs: %w", err) } - releases.CSI = &csi + extraCSIvals, err := extraCSIValues(config.GetProvider(), serviceAccURI) + if err != nil { + return nil, fmt.Errorf("extending CSI values: %w", err) + } + csiRelease.Values = mergeMaps(csiRelease.Values, extraCSIvals) + releases.CSI = &csiRelease } return &releases, nil } @@ -183,13 +190,7 @@ func (i *ChartLoader) loadRelease(info chartInfo, helmWaitMode WaitMode) (Releas updateVersions(chart, constants.BinaryVersion()) values = i.loadCSIValues() } - - chartRaw, err := i.marshalChart(chart) - if err != nil { - return Release{}, fmt.Errorf("packaging %s chart: %w", info.releaseName, err) - } - - return Release{Chart: chartRaw, Values: values, ReleaseName: info.releaseName, WaitMode: helmWaitMode}, nil + return Release{Chart: chart, Values: values, ReleaseName: info.releaseName, WaitMode: helmWaitMode}, nil } func (i *ChartLoader) loadAWSLBControllerValues() map[string]any { @@ -200,22 +201,6 @@ func (i *ChartLoader) loadAWSLBControllerValues() map[string]any { } } -// extendCiliumValues extends the given values map by some values depending on user input. -// This extra step of separating the application of user input is necessary since service upgrades should -// reuse user input from the init step. However, we can't rely on reuse-values, because -// during upgrades we all values need to be set locally as they might have changed. -// Also, the charts are not rendered correctly without all of these values. -func extendCiliumValues(in map[string]any, conformanceMode bool) { - if conformanceMode { - in["kubeProxyReplacementHealthzBindAddr"] = "" - in["kubeProxyReplacement"] = "partial" - in["sessionAffinity"] = true - in["cni"] = map[string]any{ - "chainingMode": "portmap", - } - } -} - // loadCertManagerHelper is used to separate the marshalling step from the loading step. // This reduces the time unit tests take to execute. func (i *ChartLoader) loadCertManagerValues() map[string]any { @@ -321,59 +306,6 @@ func (i *ChartLoader) cspTags() map[string]any { } } -// extendConstellationServicesValues extends the given values map by some values depending on user input. -// Values set inside this function are only applied during init, not during upgrade. -func extendConstellationServicesValues( - in map[string]any, cfg *config.Config, masterSecret, salt []byte, -) error { - keyServiceValues, ok := in["key-service"].(map[string]any) - if !ok { - return errors.New("missing 'key-service' key") - } - keyServiceValues["masterSecret"] = base64.StdEncoding.EncodeToString(masterSecret) - keyServiceValues["salt"] = base64.StdEncoding.EncodeToString(salt) - - joinServiceVals, ok := in["join-service"].(map[string]any) - if !ok { - return errors.New("invalid join-service values") - } - joinServiceVals["attestationVariant"] = cfg.GetAttestationConfig().GetVariant().String() - - // attestation config is updated separately during upgrade, - // so we only set them in Helm during init. - attestationConfigJSON, err := json.Marshal(cfg.GetAttestationConfig()) - if err != nil { - return fmt.Errorf("marshalling measurements: %w", err) - } - joinServiceVals["attestationConfig"] = string(attestationConfigJSON) - - verifyServiceVals, ok := in["verification-service"].(map[string]any) - if !ok { - return errors.New("invalid verification-service values") - } - verifyServiceVals["attestationVariant"] = cfg.GetAttestationConfig().GetVariant().String() - - csp := cfg.GetProvider() - switch csp { - case cloudprovider.OpenStack: - in["openstack"] = map[string]any{ - "deployYawolLoadBalancer": cfg.DeployYawolLoadBalancer(), - } - if cfg.DeployYawolLoadBalancer() { - in["yawol-controller"] = map[string]any{ - "yawolOSSecretName": "yawolkey", - // has to be larger than ~30s to account for slow OpenStack API calls. - "openstackTimeout": "1m", - "yawolFloatingID": cfg.Provider.OpenStack.FloatingIPPoolID, - "yawolFlavorID": cfg.Provider.OpenStack.YawolFlavorID, - "yawolImageID": cfg.Provider.OpenStack.YawolImageID, - } - } - } - - return nil -} - // updateVersions changes all versions of direct dependencies that are set to "0.0.0" to newVersion. func updateVersions(chart *chart.Chart, newVersion semver.Semver) { chart.Metadata.Version = newVersion.String() @@ -392,29 +324,6 @@ func updateVersions(chart *chart.Chart, newVersion semver.Semver) { } } -// marshalChart takes a Chart object, packages it to a temporary file and returns the content of that file. -// We currently need to take this approach of marshaling as dependencies are not marshaled correctly with json.Marshal. -// This stems from the fact that chart.Chart does not export the dependencies property. -func (i *ChartLoader) marshalChart(chart *chart.Chart) ([]byte, error) { - // A separate tmpdir path is necessary since during unit testing multiple go routines are accessing the same path, possibly deleting files for other routines. - tmpDirPath, err := os.MkdirTemp("", "*") - defer os.Remove(tmpDirPath) - if err != nil { - return nil, fmt.Errorf("creating tmp dir: %w", err) - } - - path, err := chartutil.Save(chart, tmpDirPath) - defer os.Remove(path) - if err != nil { - return nil, fmt.Errorf("chartutil save: %w", err) - } - chartRaw, err := os.ReadFile(path) - if err != nil { - return nil, fmt.Errorf("reading packaged chart: %w", err) - } - return chartRaw, nil -} - // taken from loader.LoadDir from the helm go module // loadChartsDir loads from a directory. // diff --git a/cli/internal/helm/loader_test.go b/cli/internal/helm/loader_test.go index b6950f7c0..e8446dd1b 100644 --- a/cli/internal/helm/loader_test.go +++ b/cli/internal/helm/loader_test.go @@ -20,27 +20,56 @@ import ( "github.com/pkg/errors" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "helm.sh/helm/v3/pkg/chart/loader" "helm.sh/helm/v3/pkg/chartutil" "helm.sh/helm/v3/pkg/engine" + "github.com/edgelesssys/constellation/v2/cli/internal/clusterid" + "github.com/edgelesssys/constellation/v2/cli/internal/terraform" "github.com/edgelesssys/constellation/v2/internal/attestation/idkeydigest" "github.com/edgelesssys/constellation/v2/internal/attestation/measurements" + "github.com/edgelesssys/constellation/v2/internal/cloud/azureshared" "github.com/edgelesssys/constellation/v2/internal/cloud/cloudprovider" + "github.com/edgelesssys/constellation/v2/internal/cloud/gcpshared" "github.com/edgelesssys/constellation/v2/internal/config" ) +func fakeServiceAccURI(provider cloudprovider.Provider) string { + switch provider { + case cloudprovider.GCP: + cred := gcpshared.ServiceAccountKey{ + Type: "service_account", + ProjectID: "project_id", + PrivateKeyID: "key_id", + PrivateKey: "key", + ClientEmail: "client_email", + ClientID: "client_id", + AuthURI: "auth_uri", + TokenURI: "token_uri", + AuthProviderX509CertURL: "cert", + ClientX509CertURL: "client_cert", + } + return cred.ToCloudServiceAccountURI() + case cloudprovider.Azure: + creds := azureshared.ApplicationCredentials{ + TenantID: "TenantID", + Location: "Location", + PreferredAuthMethod: azureshared.AuthMethodUserAssignedIdentity, + UamiResourceID: "uid", + } + return creds.ToCloudServiceAccountURI() + default: + return "" + } +} + func TestLoadReleases(t *testing.T) { assert := assert.New(t) require := require.New(t) - config := &config.Config{Provider: config.ProviderConfig{GCP: &config.GCPConfig{}}} chartLoader := ChartLoader{csp: config.GetProvider()} - helmReleases, err := chartLoader.LoadReleases(config, true, WaitModeAtomic, []byte("secret"), []byte("salt")) - require.NoError(err) - reader := bytes.NewReader(helmReleases.ConstellationServices.Chart) - chart, err := loader.LoadArchive(reader) + helmReleases, err := chartLoader.LoadReleases(config, true, WaitModeAtomic, []byte("secret"), []byte("salt"), fakeServiceAccURI(cloudprovider.GCP), clusterid.File{UID: "testuid"}, terraform.ApplyOutput{GCP: &terraform.GCPApplyOutput{}}) require.NoError(err) + chart := helmReleases.ConstellationServices.Chart assert.NotNil(chart.Dependencies()) } @@ -146,8 +175,13 @@ func TestConstellationServices(t *testing.T) { chart, err := loadChartsDir(helmFS, constellationServicesInfo.path) require.NoError(err) values := chartLoader.loadConstellationServicesValues() - err = extendConstellationServicesValues(values, tc.config, []byte("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"), []byte("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa")) + serviceAccURI := fakeServiceAccURI(tc.config.GetProvider()) + extraVals, err := extraConstellationServicesValues(tc.config, []byte("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"), []byte("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"), "uid", serviceAccURI, terraform.ApplyOutput{ + Azure: &terraform.AzureApplyOutput{}, + GCP: &terraform.GCPApplyOutput{}, + }) require.NoError(err) + values = mergeMaps(values, extraVals) options := chartutil.ReleaseOptions{ Name: "testRelease", diff --git a/cli/internal/helm/overrides.go b/cli/internal/helm/overrides.go new file mode 100644 index 000000000..cdffbef30 --- /dev/null +++ b/cli/internal/helm/overrides.go @@ -0,0 +1,182 @@ +/* +Copyright (c) Edgeless Systems GmbH + +SPDX-License-Identifier: AGPL-3.0-only +*/ +/* +Overrides contains helm values that are dynamically injected into the helm charts. +*/ +package helm + +import ( + "encoding/base64" + "encoding/json" + "fmt" + + "github.com/edgelesssys/constellation/v2/cli/internal/terraform" + "github.com/edgelesssys/constellation/v2/internal/cloud/azureshared" + "github.com/edgelesssys/constellation/v2/internal/cloud/cloudprovider" + "github.com/edgelesssys/constellation/v2/internal/cloud/gcpshared" + "github.com/edgelesssys/constellation/v2/internal/cloud/openstack" + "github.com/edgelesssys/constellation/v2/internal/config" + "github.com/edgelesssys/constellation/v2/internal/constants" +) + +// TODO(malt3): switch over to DNS name on AWS and Azure +// soon as every apiserver certificate of every control-plane node +// has the dns endpoint in its SAN list. +// extraCiliumValues extends the given values map by some values depending on user input. +// This extra step of separating the application of user input is necessary since service upgrades should +// reuse user input from the init step. However, we can't rely on reuse-values, because +// during upgrades we all values need to be set locally as they might have changed. +// Also, the charts are not rendered correctly without all of these values. +func extraCiliumValues(provider cloudprovider.Provider, conformanceMode bool, output terraform.ApplyOutput) map[string]any { + extraVals := map[string]any{} + if conformanceMode { + extraVals["kubeProxyReplacementHealthzBindAddr"] = "" + extraVals["kubeProxyReplacement"] = "partial" + extraVals["sessionAffinity"] = true + extraVals["cni"] = map[string]any{ + "chainingMode": "portmap", + } + } + + extraVals["k8sServiceHost"] = output.IP + extraVals["k8sServicePort"] = constants.KubernetesPort + if provider == cloudprovider.GCP { + extraVals["ipv4NativeRoutingCIDR"] = output.GCP.IPCidrPod + extraVals["strictModeCIDR"] = output.GCP.IPCidrPod + } + return extraVals +} + +// extraConstellationServicesValues extends the given values map by some values depending on user input. +// Values set inside this function are only applied during init, not during upgrade. +func extraConstellationServicesValues(cfg *config.Config, masterSecret, salt []byte, uid, serviceAccURI string, output terraform.ApplyOutput, +) (map[string]any, error) { + attestationConfigJSON, err := json.Marshal(cfg.GetAttestationConfig()) + if err != nil { + return nil, fmt.Errorf("marshalling measurements: %w", err) + } + extraVals := map[string]any{} + extraVals["join-service"] = map[string]any{ + "measurementSalt": base64.StdEncoding.EncodeToString(salt), + "attestationVariant": cfg.GetAttestationConfig().GetVariant().String(), + "attestationConfig": string(attestationConfigJSON), + } + extraVals["verification-service"] = map[string]any{ + "attestationVariant": cfg.GetAttestationConfig().GetVariant().String(), + "loadBalancerIP": output.IP, + } + extraVals["konnectivity"] = map[string]any{ + "loadBalancerIP": output.IP, + } + + extraVals["key-service"] = map[string]any{ + "masterSecret": base64.StdEncoding.EncodeToString(masterSecret), + "salt": base64.StdEncoding.EncodeToString(salt), + } + switch cfg.GetProvider() { + case cloudprovider.OpenStack: + extraVals["openstack"] = map[string]any{ + "deployYawolLoadBalancer": cfg.DeployYawolLoadBalancer(), + } + if cfg.DeployYawolLoadBalancer() { + extraVals["yawol-controller"] = map[string]any{ + "yawolOSSecretName": "yawolkey", + // has to be larger than ~30s to account for slow OpenStack API calls. + "openstackTimeout": "1m", + "yawolFloatingID": cfg.Provider.OpenStack.FloatingIPPoolID, + "yawolFlavorID": cfg.Provider.OpenStack.YawolFlavorID, + "yawolImageID": cfg.Provider.OpenStack.YawolImageID, + } + } + case cloudprovider.GCP: + serviceAccountKey, err := gcpshared.ServiceAccountKeyFromURI(serviceAccURI) + if err != nil { + return nil, fmt.Errorf("getting service account key: %w", err) + } + rawKey, err := json.Marshal(serviceAccountKey) + if err != nil { + return nil, fmt.Errorf("marshaling service account key: %w", err) + } + if output.GCP == nil { + return nil, fmt.Errorf("no GCP output from Terraform") + } + extraVals["ccm"] = map[string]any{ + "GCP": map[string]any{ + "projectID": output.GCP.ProjectID, + "uid": uid, + "secretData": string(rawKey), + "subnetworkPodCIDR": output.GCP.IPCidrPod, + }, + } + case cloudprovider.Azure: + if output.Azure == nil { + return nil, fmt.Errorf("no Azure output from Terraform") + } + ccmConfig, err := getCCMConfig(*output.Azure, serviceAccURI) + if err != nil { + return nil, fmt.Errorf("getting Azure CCM config: %w", err) + } + extraVals["ccm"] = map[string]any{ + "Azure": map[string]any{ + "azureConfig": string(ccmConfig), + }, + } + } + + return extraVals, nil +} + +// getCCMConfig returns the configuration needed for the Kubernetes Cloud Controller Manager on Azure. +func getCCMConfig(tfOutput terraform.AzureApplyOutput, serviceAccURI string) ([]byte, error) { + creds, err := azureshared.ApplicationCredentialsFromURI(serviceAccURI) + if err != nil { + return nil, fmt.Errorf("getting service account key: %w", err) + } + useManagedIdentityExtension := creds.PreferredAuthMethod == azureshared.AuthMethodUserAssignedIdentity + config := cloudConfig{ + Cloud: "AzurePublicCloud", + TenantID: creds.TenantID, + SubscriptionID: tfOutput.SubscriptionID, + ResourceGroup: tfOutput.ResourceGroup, + LoadBalancerSku: "standard", + SecurityGroupName: tfOutput.NetworkSecurityGroupName, + LoadBalancerName: tfOutput.LoadBalancerName, + UseInstanceMetadata: true, + VMType: "vmss", + Location: creds.Location, + UseManagedIdentityExtension: useManagedIdentityExtension, + UserAssignedIdentityID: tfOutput.UserAssignedIdentity, + } + + return json.Marshal(config) +} + +// extraOperatorValues returns the values for the constellation-operator chart. +func extraOperatorValues(uid string) map[string]any { + return map[string]any{ + "constellation-operator": map[string]any{ + "constellationUID": uid, + }, + } +} + +// extraCSIValues returns the values for the csi chart. +func extraCSIValues(provider cloudprovider.Provider, serviceAccURI string) (map[string]any, error) { + var csiVals map[string]any + if provider == cloudprovider.OpenStack { + creds, err := openstack.AccountKeyFromURI(serviceAccURI) + if err != nil { + return nil, err + } + cinderIni := creds.CloudINI().CinderCSIConfiguration() + csiVals = map[string]any{ + "cinder-config": map[string]any{ + "secretData": cinderIni, + }, + } + } + return csiVals, nil +} diff --git a/cli/internal/helm/release.go b/cli/internal/helm/release.go index ccbe7646d..90f8b5ccf 100644 --- a/cli/internal/helm/release.go +++ b/cli/internal/helm/release.go @@ -7,9 +7,11 @@ SPDX-License-Identifier: AGPL-3.0-only // Package helm provides types and functions shared across services. package helm +import "helm.sh/helm/v3/pkg/chart" + // Release bundles all information necessary to create a helm release. type Release struct { - Chart []byte + Chart *chart.Chart Values map[string]any ReleaseName string WaitMode WaitMode diff --git a/cli/internal/terraform/terraform.go b/cli/internal/terraform/terraform.go index 2af8d2635..15be30964 100644 --- a/cli/internal/terraform/terraform.go +++ b/cli/internal/terraform/terraform.go @@ -166,6 +166,9 @@ func (c *Client) ShowCluster(ctx context.Context, provider cloudprovider.Provide if err != nil { return ApplyOutput{}, fmt.Errorf("terraform show: %w", err) } + if tfState.Values == nil { + return ApplyOutput{}, errors.New("terraform show: no values returned") + } ipOutput, ok := tfState.Values.Outputs["ip"] if !ok {