Git-Mirrors/constellation
1
0
Fork 0
mirror of https://github.com/edgelesssys/constellation.git synced 2025-01-22 21:31:14 -05:00
Code Issues Packages Projects Releases Wiki Activity

cli: unify chart value setup (#2153)

Browse Source
This commit is contained in:
Adrian Stobbe 2023-08-03 13:54:48 +02:00 committed by GitHub
parent 5119d843f1
commit 70ce195a5f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
10 changed files with 310 additions and 297 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

40
cli/internal/cmd/init.go
View File

@ -37,6 +37,7 @@ import (
"github.com/edgelesssys/constellation/v2/cli/internal/cloudcmd"
"github.com/edgelesssys/constellation/v2/cli/internal/clusterid"
"github.com/edgelesssys/constellation/v2/cli/internal/helm"
"github.com/edgelesssys/constellation/v2/cli/internal/terraform"
"github.com/edgelesssys/constellation/v2/internal/cloud/azureshared"
"github.com/edgelesssys/constellation/v2/internal/cloud/cloudprovider"
"github.com/edgelesssys/constellation/v2/internal/cloud/gcpshared"
@ -78,6 +79,11 @@ type initCmd struct {
masterSecret uri.MasterSecret
fh *file.Handler
helmInstaller helm.Initializer
tfClient showClusterer
}
type showClusterer interface {
ShowCluster(ctx context.Context, provider cloudprovider.Provider) (terraform.ApplyOutput, error)
}
// runInitialize runs the initialize command.
@ -105,7 +111,11 @@ func runInitialize(cmd *cobra.Command, _ []string) error {
if err != nil {
return fmt.Errorf("creating Helm installer: %w", err)
}
i := &initCmd{log: log, spinner: spinner, merger: &kubeconfigMerger{log: log}, fh: &fileHandler, helmInstaller: helmInstaller}
tfClient, err := terraform.New(ctx, constants.TerraformWorkingDir)
if err != nil {
return fmt.Errorf("creating Terraform client: %w", err)
}
i := &initCmd{log: log, spinner: spinner, merger: &kubeconfigMerger{log: log}, fh: &fileHandler, helmInstaller: helmInstaller, tfClient: tfClient}
fetcher := attestationconfigapi.NewFetcher()
return i.initialize(cmd, newDialer, fileHandler, license.NewClient(), fetcher)
}
@ -183,17 +193,6 @@ func (i *initCmd) initialize(cmd *cobra.Command, newDialer func(validator atls.V
clusterName := clusterid.GetClusterName(conf, idFile)
i.log.Debugf("Setting cluster name to %s", clusterName)
helmLoader := helm.NewLoader(provider, k8sVersion, clusterName)
i.log.Debugf("Created new Helm loader")
releases, err := helmLoader.LoadReleases(conf, flags.conformance, flags.helmWaitMode, masterSecret.Key, masterSecret.Salt)
if err != nil {
return fmt.Errorf("loading Helm charts: %w", err)
}
i.log.Debugf("Loaded Helm deployments")
if err != nil {
return fmt.Errorf("loading Helm charts: %w", err)
}
cmd.PrintErrln("Note: If you just created the cluster, it can take a few minutes to connect.")
i.spinner.Start("Connecting ", false)
req := &initproto.InitRequest{
@ -230,7 +229,22 @@ func (i *initCmd) initialize(cmd *cobra.Command, newDialer func(validator atls.V
if err != nil {
return err
}
if err := i.helmInstaller.Install(cmd.Context(), provider, masterSecret, idFile, serviceAccURI, releases); err != nil {
helmLoader := helm.NewLoader(provider, k8sVersion, clusterName)
i.log.Debugf("Created new Helm loader")
output, err := i.tfClient.ShowCluster(cmd.Context(), conf.GetProvider())
if err != nil {
return fmt.Errorf("getting Terraform output: %w", err)
}
releases, err := helmLoader.LoadReleases(conf, flags.conformance, flags.helmWaitMode, masterSecret.Key, masterSecret.Salt, serviceAccURI, idFile, output)
if err != nil {
return fmt.Errorf("loading Helm charts: %w", err)
}
i.log.Debugf("Loaded Helm deployments")
if err != nil {
return fmt.Errorf("loading Helm charts: %w", err)
}
if err := i.helmInstaller.Install(cmd.Context(), releases); err != nil {
return fmt.Errorf("installing Helm charts: %w", err)
}
cmd.Println(bufferedOutput.String())

32
cli/internal/cmd/init_test.go
View File

@ -22,6 +22,7 @@ import (
"github.com/edgelesssys/constellation/v2/bootstrapper/initproto"
"github.com/edgelesssys/constellation/v2/cli/internal/clusterid"
"github.com/edgelesssys/constellation/v2/cli/internal/helm"
"github.com/edgelesssys/constellation/v2/cli/internal/terraform"
"github.com/edgelesssys/constellation/v2/internal/atls"
"github.com/edgelesssys/constellation/v2/internal/attestation/measurements"
"github.com/edgelesssys/constellation/v2/internal/attestation/variant"
@ -54,7 +55,16 @@ func TestInitArgumentValidation(t *testing.T) {
func TestInitialize(t *testing.T) {
gcpServiceAccKey := &gcpshared.ServiceAccountKey{
Type: "service_account",
Type: "service_account",
ProjectID: "project_id",
PrivateKeyID: "key_id",
PrivateKey: "key",
ClientEmail: "client_email",
ClientID: "client_id",
AuthURI: "auth_uri",
TokenURI: "token_uri",
AuthProviderX509CertURL: "cert",
ClientX509CertURL: "client_cert",
}
testInitResp := &initproto.InitSuccessResponse{
Kubeconfig: []byte("kubeconfig"),
@ -175,7 +185,7 @@ func TestInitialize(t *testing.T) {
ctx, cancel := context.WithTimeout(ctx, 4*time.Second)
defer cancel()
cmd.SetContext(ctx)
i := &initCmd{log: logger.NewTest(t), spinner: &nopSpinner{}, helmInstaller: &stubHelmInstaller{}}
i := &initCmd{log: logger.NewTest(t), spinner: &nopSpinner{}, helmInstaller: &stubHelmInstaller{}, tfClient: &stubShowCluster{}}
err := i.initialize(cmd, newDialer, fileHandler, &stubLicenseClient{}, stubAttestationFetcher{})
if tc.wantErr {
@ -670,9 +680,19 @@ func (c stubInitClient) Recv() (*initproto.InitResponse, error) {
type stubHelmInstaller struct{}
func (i *stubHelmInstaller) Install(_ context.Context, _ cloudprovider.Provider, _ uri.MasterSecret,
_ clusterid.File,
_ string, _ *helm.Releases,
) error {
func (i *stubHelmInstaller) Install(_ context.Context, _ *helm.Releases) error {
return nil
}
type stubShowCluster struct{}
func (s *stubShowCluster) ShowCluster(_ context.Context, csp cloudprovider.Provider) (terraform.ApplyOutput, error) {
res := terraform.ApplyOutput{}
switch csp {
case cloudprovider.Azure:
res.Azure = &terraform.AzureApplyOutput{}
case cloudprovider.GCP:
res.GCP = &terraform.GCPApplyOutput{}
}
return res, nil
}

7
cli/internal/helm/BUILD.bazel
View File

@ -10,6 +10,7 @@ go_library(
"init.go",
"install.go",
"loader.go",
"overrides.go",
"release.go",
"serviceversion.go",
"upgrade.go",
@ -430,7 +431,6 @@ go_library(
"//internal/config",
"//internal/constants",
"//internal/file",
"//internal/kms/uri",
"//internal/retry",
"//internal/semver",
"//internal/versions",
@ -449,7 +449,6 @@ go_library(
"@sh_helm_helm_v3//pkg/action",
"@sh_helm_helm_v3//pkg/chart",
"@sh_helm_helm_v3//pkg/chart/loader",
"@sh_helm_helm_v3//pkg/chartutil",
"@sh_helm_helm_v3//pkg/cli",
"@sh_helm_helm_v3//pkg/release",
],
@ -467,9 +466,12 @@ go_test(
embed = [":helm"],
deps = [
"//cli/internal/clusterid",
"//cli/internal/terraform",
"//internal/attestation/idkeydigest",
"//internal/attestation/measurements",
"//internal/cloud/azureshared",
"//internal/cloud/cloudprovider",
"//internal/cloud/gcpshared",
"//internal/compatibility",
"//internal/config",
"//internal/file",
@ -485,7 +487,6 @@ go_test(
"@io_k8s_apimachinery//pkg/runtime/schema",
"@io_k8s_sigs_yaml//:yaml",
"@sh_helm_helm_v3//pkg/chart",
"@sh_helm_helm_v3//pkg/chart/loader",
"@sh_helm_helm_v3//pkg/chartutil",
"@sh_helm_helm_v3//pkg/engine",
"@sh_helm_helm_v3//pkg/release",

155
cli/internal/helm/init.go
View File

@ -7,27 +7,15 @@ package helm
import (
"context"
"encoding/base64"
"encoding/json"
"fmt"
"time"
"github.com/edgelesssys/constellation/v2/cli/internal/clusterid"
"github.com/edgelesssys/constellation/v2/cli/internal/terraform"
"github.com/edgelesssys/constellation/v2/internal/cloud/azureshared"
"github.com/edgelesssys/constellation/v2/internal/cloud/cloudprovider"
"github.com/edgelesssys/constellation/v2/internal/cloud/gcpshared"
"github.com/edgelesssys/constellation/v2/internal/cloud/openstack"
"github.com/edgelesssys/constellation/v2/internal/constants"
"github.com/edgelesssys/constellation/v2/internal/kms/uri"
)
// Initializer installs all Helm charts required for a constellation cluster.
type Initializer interface {
Install(ctx context.Context, provider cloudprovider.Provider, masterSecret uri.MasterSecret,
idFile clusterid.File,
serviceAccURI string, releases *Releases,
) error
Install(ctx context.Context, releases *Releases) error
}
type initializationClient struct {
@ -45,20 +33,9 @@ func NewInitializer(log debugLog) (Initializer, error) {
}
// Install installs all Helm charts required for a constellation cluster.
func (h initializationClient) Install(ctx context.Context, provider cloudprovider.Provider, masterSecret uri.MasterSecret,
idFile clusterid.File,
serviceAccURI string, releases *Releases,
func (h initializationClient) Install(ctx context.Context, releases *Releases,
) error {
tfClient, err := terraform.New(ctx, constants.TerraformWorkingDir)
if err != nil {
return fmt.Errorf("creating Terraform client: %w", err)
}
output, err := tfClient.ShowCluster(ctx, provider)
if err != nil {
return fmt.Errorf("getting Terraform output: %w", err)
}
ciliumVals := setupCiliumVals(provider, output)
if err := h.installer.InstallChartWithValues(ctx, releases.Cilium, ciliumVals); err != nil {
if err := h.installer.InstallChart(ctx, releases.Cilium); err != nil {
return fmt.Errorf("installing Cilium: %w", err)
}
h.log.Debugf("Waiting for Cilium to become ready")
@ -81,11 +58,7 @@ func (h initializationClient) Install(ctx context.Context, provider cloudprovide
}
h.log.Debugf("Installing microservices")
serviceVals, err := setupMicroserviceVals(provider, masterSecret.Salt, idFile.UID, serviceAccURI, output)
if err != nil {
return fmt.Errorf("setting up microservice values: %w", err)
}
if err := h.installer.InstallChartWithValues(ctx, releases.ConstellationServices, serviceVals); err != nil {
if err := h.installer.InstallChart(ctx, releases.ConstellationServices); err != nil {
return fmt.Errorf("installing microservices: %w", err)
}
@ -95,22 +68,8 @@ func (h initializationClient) Install(ctx context.Context, provider cloudprovide
}
if releases.CSI != nil {
var csiVals map[string]any
if provider == cloudprovider.OpenStack {
creds, err := openstack.AccountKeyFromURI(serviceAccURI)
if err != nil {
return err
}
cinderIni := creds.CloudINI().CinderCSIConfiguration()
csiVals = map[string]any{
"cinder-config": map[string]any{
"secretData": cinderIni,
},
}
}
h.log.Debugf("Installing CSI deployments")
if err := h.installer.InstallChartWithValues(ctx, *releases.CSI, csiVals); err != nil {
if err := h.installer.InstallChart(ctx, *releases.CSI); err != nil {
return fmt.Errorf("installing CSI snapshot CRDs: %w", err)
}
}
@ -123,8 +82,7 @@ func (h initializationClient) Install(ctx context.Context, provider cloudprovide
}
h.log.Debugf("Installing constellation operators")
operatorVals := setupOperatorVals(ctx, idFile.UID)
if err := h.installer.InstallChartWithValues(ctx, releases.ConstellationOperators, operatorVals); err != nil {
if err := h.installer.InstallChart(ctx, releases.ConstellationOperators); err != nil {
return fmt.Errorf("installing constellation operators: %w", err)
}
return nil
@ -136,82 +94,6 @@ type installer interface {
InstallChartWithValues(ctx context.Context, release Release, extraValues map[string]any) error
}
// TODO(malt3): switch over to DNS name on AWS and Azure
// soon as every apiserver certificate of every control-plane node
// has the dns endpoint in its SAN list.
func setupCiliumVals(provider cloudprovider.Provider, output terraform.ApplyOutput) map[string]any {
vals := map[string]any{
"k8sServiceHost": output.IP,
"k8sServicePort": constants.KubernetesPort,
}
if provider == cloudprovider.GCP {
vals["ipv4NativeRoutingCIDR"] = output.GCP.IPCidrPod
vals["strictModeCIDR"] = output.GCP.IPCidrPod
}
return vals
}
// setupMicroserviceVals returns the values for the microservice chart.
func setupMicroserviceVals(provider cloudprovider.Provider, measurementSalt []byte, uid, serviceAccURI string, output terraform.ApplyOutput) (map[string]any, error) {
extraVals := map[string]any{
"join-service": map[string]any{
"measurementSalt": base64.StdEncoding.EncodeToString(measurementSalt),
},
"verification-service": map[string]any{
"loadBalancerIP": output.IP,
},
"konnectivity": map[string]any{
"loadBalancerIP": output.IP,
},
}
switch provider {
case cloudprovider.GCP:
serviceAccountKey, err := gcpshared.ServiceAccountKeyFromURI(serviceAccURI)
if err != nil {
return nil, fmt.Errorf("getting service account key: %w", err)
}
rawKey, err := json.Marshal(serviceAccountKey)
if err != nil {
return nil, fmt.Errorf("marshaling service account key: %w", err)
}
if output.GCP == nil {
return nil, fmt.Errorf("no GCP output from Terraform")
}
extraVals["ccm"] = map[string]any{
"GCP": map[string]any{
"projectID": output.GCP.ProjectID,
"uid": uid,
"secretData": string(rawKey),
"subnetworkPodCIDR": output.GCP.IPCidrPod,
},
}
case cloudprovider.Azure:
if output.Azure == nil {
return nil, fmt.Errorf("no Azure output from Terraform")
}
ccmConfig, err := getCCMConfig(*output.Azure, serviceAccURI)
if err != nil {
return nil, fmt.Errorf("getting Azure CCM config: %w", err)
}
extraVals["ccm"] = map[string]any{
"Azure": map[string]any{
"azureConfig": string(ccmConfig),
},
}
}
return extraVals, nil
}
// setupOperatorVals returns the values for the constellation-operator chart.
func setupOperatorVals(_ context.Context, uid string) map[string]any {
return map[string]any{
"constellation-operator": map[string]any{
"constellationUID": uid,
},
}
}
type cloudConfig struct {
Cloud string `json:"cloud,omitempty"`
TenantID string `json:"tenantId,omitempty"`
@ -231,28 +113,3 @@ type cloudConfig struct {
UseManagedIdentityExtension bool `json:"useManagedIdentityExtension,omitempty"`
UserAssignedIdentityID string `json:"userAssignedIdentityID,omitempty"`
}
// GetCCMConfig returns the configuration needed for the Kubernetes Cloud Controller Manager on Azure.
func getCCMConfig(tfOutput terraform.AzureApplyOutput, serviceAccURI string) ([]byte, error) {
creds, err := azureshared.ApplicationCredentialsFromURI(serviceAccURI)
if err != nil {
return nil, fmt.Errorf("getting service account key: %w", err)
}
useManagedIdentityExtension := creds.PreferredAuthMethod == azureshared.AuthMethodUserAssignedIdentity
config := cloudConfig{
Cloud: "AzurePublicCloud",
TenantID: creds.TenantID,
SubscriptionID: tfOutput.SubscriptionID,
ResourceGroup: tfOutput.ResourceGroup,
LoadBalancerSku: "standard",
SecurityGroupName: tfOutput.NetworkSecurityGroupName,
LoadBalancerName: tfOutput.LoadBalancerName,
UseInstanceMetadata: true,
VMType: "vmss",
Location: creds.Location,
UseManagedIdentityExtension: useManagedIdentityExtension,
UserAssignedIdentityID: tfOutput.UserAssignedIdentity,
}
return json.Marshal(config)
}

11
cli/internal/helm/install.go
View File

@ -7,7 +7,6 @@ SPDX-License-Identifier: AGPL-3.0-only
package helm
import (
"bytes"
"context"
"fmt"
"strings"
@ -17,7 +16,6 @@ import (
"github.com/edgelesssys/constellation/v2/internal/retry"
"helm.sh/helm/v3/pkg/action"
"helm.sh/helm/v3/pkg/chart"
"helm.sh/helm/v3/pkg/chart/loader"
"helm.sh/helm/v3/pkg/cli"
"k8s.io/apimachinery/pkg/util/wait"
)
@ -79,7 +77,7 @@ func (h *Installer) InstallChartWithValues(ctx context.Context, release Release,
// install tries to install the given chart and aborts after ~5 tries.
// The function will wait 30 seconds before retrying a failed installation attempt.
// After 3 tries, the retrier will be canceled and the function returns with an error.
func (h *Installer) install(ctx context.Context, chartRaw []byte, values map[string]any) error {
func (h *Installer) install(ctx context.Context, chart *chart.Chart, values map[string]any) error {
var retries int
retriable := func(err error) bool {
// abort after maximumRetryAttempts tries.
@ -97,13 +95,6 @@ func (h *Installer) install(ctx context.Context, chartRaw []byte, values map[str
return wait.Interrupted(err) ||
strings.Contains(err.Error(), "connection refused")
}
reader := bytes.NewReader(chartRaw)
chart, err := loader.LoadArchive(reader)
if err != nil {
return fmt.Errorf("helm load archive: %w", err)
}
doer := installDoer{
h,
chart,

125
cli/internal/helm/loader.go
View File

@ -9,11 +9,8 @@ package helm
import (
"bytes"
"embed"
"encoding/base64"
"encoding/json"
"fmt"
"io/fs"
"os"
"path/filepath"
"strings"
@ -21,9 +18,10 @@ import (
"helm.sh/helm/pkg/ignore"
"helm.sh/helm/v3/pkg/chart"
"helm.sh/helm/v3/pkg/chart/loader"
"helm.sh/helm/v3/pkg/chartutil"
"github.com/edgelesssys/constellation/v2/cli/internal/clusterid"
"github.com/edgelesssys/constellation/v2/cli/internal/helm/imageversion"
"github.com/edgelesssys/constellation/v2/cli/internal/terraform"
"github.com/edgelesssys/constellation/v2/internal/cloud/cloudprovider"
"github.com/edgelesssys/constellation/v2/internal/config"
"github.com/edgelesssys/constellation/v2/internal/constants"
@ -108,12 +106,13 @@ func NewLoader(csp cloudprovider.Provider, k8sVersion versions.ValidK8sVersion,
}
// LoadReleases loads the embedded helm charts and returns them as a HelmReleases object.
func (i *ChartLoader) LoadReleases(config *config.Config, conformanceMode bool, helmWaitMode WaitMode, masterSecret, salt []byte) (*Releases, error) {
func (i *ChartLoader) LoadReleases(config *config.Config, conformanceMode bool, helmWaitMode WaitMode, masterSecret, salt []byte, serviceAccURI string, idFile clusterid.File, output terraform.ApplyOutput) (*Releases, error) {
ciliumRelease, err := i.loadRelease(ciliumInfo, helmWaitMode)
if err != nil {
return nil, fmt.Errorf("loading cilium: %w", err)
}
extendCiliumValues(ciliumRelease.Values, conformanceMode)
ciliumVals := extraCiliumValues(config.GetProvider(), conformanceMode, output)
ciliumRelease.Values = mergeMaps(ciliumRelease.Values, ciliumVals)
certManagerRelease, err := i.loadRelease(certManagerInfo, helmWaitMode)
if err != nil {
@ -124,14 +123,17 @@ func (i *ChartLoader) LoadReleases(config *config.Config, conformanceMode bool,
if err != nil {
return nil, fmt.Errorf("loading operators: %w", err)
}
operatorRelease.Values = mergeMaps(operatorRelease.Values, extraOperatorValues(idFile.UID))
conServicesRelease, err := i.loadRelease(constellationServicesInfo, helmWaitMode)
if err != nil {
return nil, fmt.Errorf("loading constellation-services: %w", err)
}
if err := extendConstellationServicesValues(conServicesRelease.Values, config, masterSecret, salt); err != nil {
svcVals, err := extraConstellationServicesValues(config, masterSecret, salt, idFile.UID, serviceAccURI, output)
if err != nil {
return nil, fmt.Errorf("extending constellation-services values: %w", err)
}
conServicesRelease.Values = mergeMaps(conServicesRelease.Values, svcVals)
releases := Releases{Cilium: ciliumRelease, CertManager: certManagerRelease, ConstellationOperators: operatorRelease, ConstellationServices: conServicesRelease}
if config.HasProvider(cloudprovider.AWS) {
@ -143,11 +145,16 @@ func (i *ChartLoader) LoadReleases(config *config.Config, conformanceMode bool,
}
if config.DeployCSIDriver() {
csi, err := i.loadRelease(csiInfo, helmWaitMode)
csiRelease, err := i.loadRelease(csiInfo, helmWaitMode)
if err != nil {
return nil, fmt.Errorf("loading snapshot CRDs: %w", err)
}
releases.CSI = &csi
extraCSIvals, err := extraCSIValues(config.GetProvider(), serviceAccURI)
if err != nil {
return nil, fmt.Errorf("extending CSI values: %w", err)
}
csiRelease.Values = mergeMaps(csiRelease.Values, extraCSIvals)
releases.CSI = &csiRelease
}
return &releases, nil
}
@ -183,13 +190,7 @@ func (i *ChartLoader) loadRelease(info chartInfo, helmWaitMode WaitMode) (Releas
updateVersions(chart, constants.BinaryVersion())
values = i.loadCSIValues()
}
chartRaw, err := i.marshalChart(chart)
if err != nil {
return Release{}, fmt.Errorf("packaging %s chart: %w", info.releaseName, err)
}
return Release{Chart: chartRaw, Values: values, ReleaseName: info.releaseName, WaitMode: helmWaitMode}, nil
return Release{Chart: chart, Values: values, ReleaseName: info.releaseName, WaitMode: helmWaitMode}, nil
}
func (i *ChartLoader) loadAWSLBControllerValues() map[string]any {
@ -200,22 +201,6 @@ func (i *ChartLoader) loadAWSLBControllerValues() map[string]any {
}
}
// extendCiliumValues extends the given values map by some values depending on user input.
// This extra step of separating the application of user input is necessary since service upgrades should
// reuse user input from the init step. However, we can't rely on reuse-values, because
// during upgrades we all values need to be set locally as they might have changed.
// Also, the charts are not rendered correctly without all of these values.
func extendCiliumValues(in map[string]any, conformanceMode bool) {
if conformanceMode {
in["kubeProxyReplacementHealthzBindAddr"] = ""
in["kubeProxyReplacement"] = "partial"
in["sessionAffinity"] = true
in["cni"] = map[string]any{
"chainingMode": "portmap",
}
}
}
// loadCertManagerHelper is used to separate the marshalling step from the loading step.
// This reduces the time unit tests take to execute.
func (i *ChartLoader) loadCertManagerValues() map[string]any {
@ -321,59 +306,6 @@ func (i *ChartLoader) cspTags() map[string]any {
}
}
// extendConstellationServicesValues extends the given values map by some values depending on user input.
// Values set inside this function are only applied during init, not during upgrade.
func extendConstellationServicesValues(
in map[string]any, cfg *config.Config, masterSecret, salt []byte,
) error {
keyServiceValues, ok := in["key-service"].(map[string]any)
if !ok {
return errors.New("missing 'key-service' key")
}
keyServiceValues["masterSecret"] = base64.StdEncoding.EncodeToString(masterSecret)
keyServiceValues["salt"] = base64.StdEncoding.EncodeToString(salt)
joinServiceVals, ok := in["join-service"].(map[string]any)
if !ok {
return errors.New("invalid join-service values")
}
joinServiceVals["attestationVariant"] = cfg.GetAttestationConfig().GetVariant().String()
// attestation config is updated separately during upgrade,
// so we only set them in Helm during init.
attestationConfigJSON, err := json.Marshal(cfg.GetAttestationConfig())
if err != nil {
return fmt.Errorf("marshalling measurements: %w", err)
}
joinServiceVals["attestationConfig"] = string(attestationConfigJSON)
verifyServiceVals, ok := in["verification-service"].(map[string]any)
if !ok {
return errors.New("invalid verification-service values")
}
verifyServiceVals["attestationVariant"] = cfg.GetAttestationConfig().GetVariant().String()
csp := cfg.GetProvider()
switch csp {
case cloudprovider.OpenStack:
in["openstack"] = map[string]any{
"deployYawolLoadBalancer": cfg.DeployYawolLoadBalancer(),
}
if cfg.DeployYawolLoadBalancer() {
in["yawol-controller"] = map[string]any{
"yawolOSSecretName": "yawolkey",
// has to be larger than ~30s to account for slow OpenStack API calls.
"openstackTimeout": "1m",
"yawolFloatingID": cfg.Provider.OpenStack.FloatingIPPoolID,
"yawolFlavorID": cfg.Provider.OpenStack.YawolFlavorID,
"yawolImageID": cfg.Provider.OpenStack.YawolImageID,
}
}
}
return nil
}
// updateVersions changes all versions of direct dependencies that are set to "0.0.0" to newVersion.
func updateVersions(chart *chart.Chart, newVersion semver.Semver) {
chart.Metadata.Version = newVersion.String()
@ -392,29 +324,6 @@ func updateVersions(chart *chart.Chart, newVersion semver.Semver) {
}
}
// marshalChart takes a Chart object, packages it to a temporary file and returns the content of that file.
// We currently need to take this approach of marshaling as dependencies are not marshaled correctly with json.Marshal.
// This stems from the fact that chart.Chart does not export the dependencies property.
func (i *ChartLoader) marshalChart(chart *chart.Chart) ([]byte, error) {
// A separate tmpdir path is necessary since during unit testing multiple go routines are accessing the same path, possibly deleting files for other routines.
tmpDirPath, err := os.MkdirTemp("", "*")
defer os.Remove(tmpDirPath)
if err != nil {
return nil, fmt.Errorf("creating tmp dir: %w", err)
}
path, err := chartutil.Save(chart, tmpDirPath)
defer os.Remove(path)
if err != nil {
return nil, fmt.Errorf("chartutil save: %w", err)
}
chartRaw, err := os.ReadFile(path)
if err != nil {
return nil, fmt.Errorf("reading packaged chart: %w", err)
}
return chartRaw, nil
}
// taken from loader.LoadDir from the helm go module
// loadChartsDir loads from a directory.
//

48
cli/internal/helm/loader_test.go
View File

@ -20,27 +20,56 @@ import (
"github.com/pkg/errors"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"helm.sh/helm/v3/pkg/chart/loader"
"helm.sh/helm/v3/pkg/chartutil"
"helm.sh/helm/v3/pkg/engine"
"github.com/edgelesssys/constellation/v2/cli/internal/clusterid"
"github.com/edgelesssys/constellation/v2/cli/internal/terraform"
"github.com/edgelesssys/constellation/v2/internal/attestation/idkeydigest"
"github.com/edgelesssys/constellation/v2/internal/attestation/measurements"
"github.com/edgelesssys/constellation/v2/internal/cloud/azureshared"
"github.com/edgelesssys/constellation/v2/internal/cloud/cloudprovider"
"github.com/edgelesssys/constellation/v2/internal/cloud/gcpshared"
"github.com/edgelesssys/constellation/v2/internal/config"
)
func fakeServiceAccURI(provider cloudprovider.Provider) string {
switch provider {
case cloudprovider.GCP:
cred := gcpshared.ServiceAccountKey{
Type: "service_account",
ProjectID: "project_id",
PrivateKeyID: "key_id",
PrivateKey: "key",
ClientEmail: "client_email",
ClientID: "client_id",
AuthURI: "auth_uri",
TokenURI: "token_uri",
AuthProviderX509CertURL: "cert",
ClientX509CertURL: "client_cert",
}
return cred.ToCloudServiceAccountURI()
case cloudprovider.Azure:
creds := azureshared.ApplicationCredentials{
TenantID: "TenantID",
Location: "Location",
PreferredAuthMethod: azureshared.AuthMethodUserAssignedIdentity,
UamiResourceID: "uid",
}
return creds.ToCloudServiceAccountURI()
default:
return ""
}
}
func TestLoadReleases(t *testing.T) {
assert := assert.New(t)
require := require.New(t)
config := &config.Config{Provider: config.ProviderConfig{GCP: &config.GCPConfig{}}}
chartLoader := ChartLoader{csp: config.GetProvider()}
helmReleases, err := chartLoader.LoadReleases(config, true, WaitModeAtomic, []byte("secret"), []byte("salt"))
require.NoError(err)
reader := bytes.NewReader(helmReleases.ConstellationServices.Chart)
chart, err := loader.LoadArchive(reader)
helmReleases, err := chartLoader.LoadReleases(config, true, WaitModeAtomic, []byte("secret"), []byte("salt"), fakeServiceAccURI(cloudprovider.GCP), clusterid.File{UID: "testuid"}, terraform.ApplyOutput{GCP: &terraform.GCPApplyOutput{}})
require.NoError(err)
chart := helmReleases.ConstellationServices.Chart
assert.NotNil(chart.Dependencies())
}
@ -146,8 +175,13 @@ func TestConstellationServices(t *testing.T) {
chart, err := loadChartsDir(helmFS, constellationServicesInfo.path)
require.NoError(err)
values := chartLoader.loadConstellationServicesValues()
err = extendConstellationServicesValues(values, tc.config, []byte("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"), []byte("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"))
serviceAccURI := fakeServiceAccURI(tc.config.GetProvider())
extraVals, err := extraConstellationServicesValues(tc.config, []byte("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"), []byte("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"), "uid", serviceAccURI, terraform.ApplyOutput{
Azure: &terraform.AzureApplyOutput{},
GCP: &terraform.GCPApplyOutput{},
})
require.NoError(err)
values = mergeMaps(values, extraVals)
options := chartutil.ReleaseOptions{
Name: "testRelease",

182
cli/internal/helm/overrides.go Normal file
View File

@ -0,0 +1,182 @@
/*
Copyright (c) Edgeless Systems GmbH
SPDX-License-Identifier: AGPL-3.0-only
*/
/*
Overrides contains helm values that are dynamically injected into the helm charts.
*/
package helm
import (
"encoding/base64"
"encoding/json"
"fmt"
"github.com/edgelesssys/constellation/v2/cli/internal/terraform"
"github.com/edgelesssys/constellation/v2/internal/cloud/azureshared"
"github.com/edgelesssys/constellation/v2/internal/cloud/cloudprovider"
"github.com/edgelesssys/constellation/v2/internal/cloud/gcpshared"
"github.com/edgelesssys/constellation/v2/internal/cloud/openstack"
"github.com/edgelesssys/constellation/v2/internal/config"
"github.com/edgelesssys/constellation/v2/internal/constants"
)
// TODO(malt3): switch over to DNS name on AWS and Azure
// soon as every apiserver certificate of every control-plane node
// has the dns endpoint in its SAN list.
// extraCiliumValues extends the given values map by some values depending on user input.
// This extra step of separating the application of user input is necessary since service upgrades should
// reuse user input from the init step. However, we can't rely on reuse-values, because
// during upgrades we all values need to be set locally as they might have changed.
// Also, the charts are not rendered correctly without all of these values.
func extraCiliumValues(provider cloudprovider.Provider, conformanceMode bool, output terraform.ApplyOutput) map[string]any {
extraVals := map[string]any{}
if conformanceMode {
extraVals["kubeProxyReplacementHealthzBindAddr"] = ""
extraVals["kubeProxyReplacement"] = "partial"
extraVals["sessionAffinity"] = true
extraVals["cni"] = map[string]any{
"chainingMode": "portmap",
}
}
extraVals["k8sServiceHost"] = output.IP
extraVals["k8sServicePort"] = constants.KubernetesPort
if provider == cloudprovider.GCP {
extraVals["ipv4NativeRoutingCIDR"] = output.GCP.IPCidrPod
extraVals["strictModeCIDR"] = output.GCP.IPCidrPod
}
return extraVals
}
// extraConstellationServicesValues extends the given values map by some values depending on user input.
// Values set inside this function are only applied during init, not during upgrade.
func extraConstellationServicesValues(cfg *config.Config, masterSecret, salt []byte, uid, serviceAccURI string, output terraform.ApplyOutput,
) (map[string]any, error) {
attestationConfigJSON, err := json.Marshal(cfg.GetAttestationConfig())
if err != nil {
return nil, fmt.Errorf("marshalling measurements: %w", err)
}
extraVals := map[string]any{}
extraVals["join-service"] = map[string]any{
"measurementSalt": base64.StdEncoding.EncodeToString(salt),
"attestationVariant": cfg.GetAttestationConfig().GetVariant().String(),
"attestationConfig": string(attestationConfigJSON),
}
extraVals["verification-service"] = map[string]any{
"attestationVariant": cfg.GetAttestationConfig().GetVariant().String(),
"loadBalancerIP": output.IP,
}
extraVals["konnectivity"] = map[string]any{
"loadBalancerIP": output.IP,
}
extraVals["key-service"] = map[string]any{
"masterSecret": base64.StdEncoding.EncodeToString(masterSecret),
"salt": base64.StdEncoding.EncodeToString(salt),
}
switch cfg.GetProvider() {
case cloudprovider.OpenStack:
extraVals["openstack"] = map[string]any{
"deployYawolLoadBalancer": cfg.DeployYawolLoadBalancer(),
}
if cfg.DeployYawolLoadBalancer() {
extraVals["yawol-controller"] = map[string]any{
"yawolOSSecretName": "yawolkey",
// has to be larger than ~30s to account for slow OpenStack API calls.
"openstackTimeout": "1m",
"yawolFloatingID": cfg.Provider.OpenStack.FloatingIPPoolID,
"yawolFlavorID": cfg.Provider.OpenStack.YawolFlavorID,
"yawolImageID": cfg.Provider.OpenStack.YawolImageID,
}
}
case cloudprovider.GCP:
serviceAccountKey, err := gcpshared.ServiceAccountKeyFromURI(serviceAccURI)
if err != nil {
return nil, fmt.Errorf("getting service account key: %w", err)
}
rawKey, err := json.Marshal(serviceAccountKey)
if err != nil {
return nil, fmt.Errorf("marshaling service account key: %w", err)
}
if output.GCP == nil {
return nil, fmt.Errorf("no GCP output from Terraform")
}
extraVals["ccm"] = map[string]any{
"GCP": map[string]any{
"projectID": output.GCP.ProjectID,
"uid": uid,
"secretData": string(rawKey),
"subnetworkPodCIDR": output.GCP.IPCidrPod,
},
}
case cloudprovider.Azure:
if output.Azure == nil {
return nil, fmt.Errorf("no Azure output from Terraform")
}
ccmConfig, err := getCCMConfig(*output.Azure, serviceAccURI)
if err != nil {
return nil, fmt.Errorf("getting Azure CCM config: %w", err)
}
extraVals["ccm"] = map[string]any{
"Azure": map[string]any{
"azureConfig": string(ccmConfig),
},
}
}
return extraVals, nil
}
// getCCMConfig returns the configuration needed for the Kubernetes Cloud Controller Manager on Azure.
func getCCMConfig(tfOutput terraform.AzureApplyOutput, serviceAccURI string) ([]byte, error) {
creds, err := azureshared.ApplicationCredentialsFromURI(serviceAccURI)
if err != nil {
return nil, fmt.Errorf("getting service account key: %w", err)
}
useManagedIdentityExtension := creds.PreferredAuthMethod == azureshared.AuthMethodUserAssignedIdentity
config := cloudConfig{
Cloud: "AzurePublicCloud",
TenantID: creds.TenantID,
SubscriptionID: tfOutput.SubscriptionID,
ResourceGroup: tfOutput.ResourceGroup,
LoadBalancerSku: "standard",
SecurityGroupName: tfOutput.NetworkSecurityGroupName,
LoadBalancerName: tfOutput.LoadBalancerName,
UseInstanceMetadata: true,
VMType: "vmss",
Location: creds.Location,
UseManagedIdentityExtension: useManagedIdentityExtension,
UserAssignedIdentityID: tfOutput.UserAssignedIdentity,
}
return json.Marshal(config)
}
// extraOperatorValues returns the values for the constellation-operator chart.
func extraOperatorValues(uid string) map[string]any {
return map[string]any{
"constellation-operator": map[string]any{
"constellationUID": uid,
},
}
}
// extraCSIValues returns the values for the csi chart.
func extraCSIValues(provider cloudprovider.Provider, serviceAccURI string) (map[string]any, error) {
var csiVals map[string]any
if provider == cloudprovider.OpenStack {
creds, err := openstack.AccountKeyFromURI(serviceAccURI)
if err != nil {
return nil, err
}
cinderIni := creds.CloudINI().CinderCSIConfiguration()
csiVals = map[string]any{
"cinder-config": map[string]any{
"secretData": cinderIni,
},
}
}
return csiVals, nil
}

4
cli/internal/helm/release.go
View File

@ -7,9 +7,11 @@ SPDX-License-Identifier: AGPL-3.0-only
// Package helm provides types and functions shared across services.
package helm
import "helm.sh/helm/v3/pkg/chart"
// Release bundles all information necessary to create a helm release.
type Release struct {
Chart []byte
Chart *chart.Chart
Values map[string]any
ReleaseName string
WaitMode WaitMode

3
cli/internal/terraform/terraform.go
View File

@ -166,6 +166,9 @@ func (c *Client) ShowCluster(ctx context.Context, provider cloudprovider.Provide
if err != nil {
return ApplyOutput{}, fmt.Errorf("terraform show: %w", err)
}
if tfState.Values == nil {
return ApplyOutput{}, errors.New("terraform show: no values returned")
}
ipOutput, ok := tfState.Values.Outputs["ip"]
if !ok {