From 091e3b2b2bef94da8d93c0b9cff41f9a02e7c395 Mon Sep 17 00:00:00 2001 From: Otto Bittner Date: Wed, 26 Oct 2022 10:37:10 +0200 Subject: [PATCH] AB#2538: deploy CCM via Helm Also move helmloader interface/stubs --- CMakeLists.txt | 6 +- .../internal/kubernetes/k8sapi/constants.go | 1 - .../k8sapi/kubectl/client/client_test.go | 5 - .../resources/cloud_controller_manager.go | 179 ------------------ .../cloud_controller_manager_test.go | 29 --- .../internal/kubernetes/kubernetes.go | 115 +++++++---- .../internal/kubernetes/kubernetes_test.go | 27 +-- cli/internal/cmd/helmloader.go | 21 -- cli/internal/cmd/init.go | 6 +- cli/internal/cmd/init_test.go | 9 + .../constellation-services/Chart.yaml | 16 ++ .../charts/ccm/.helmignore | 23 +++ .../charts/ccm/Chart.yaml | 5 + .../charts/ccm/templates/aws-daemonset.yaml | 63 ++++++ .../charts/ccm/templates/azure-daemonset.yaml | 73 +++++++ .../charts/ccm/templates/azure-secret.yaml | 9 + .../ccm/templates/clusterrolebinding.yaml | 12 ++ .../charts/ccm/templates/gcp-cm.yaml | 9 + .../charts/ccm/templates/gcp-daemonset.yaml | 84 ++++++++ .../charts/ccm/templates/gcp-secret.yaml | 9 + .../charts/ccm/templates/serviceaccount.yaml | 5 + .../charts/ccm/values.schema.json | 105 ++++++++++ .../charts/ccm/values.yaml | 0 .../constellation-services/values.yaml | 7 + cli/internal/helm/loader.go | 66 ++++++- cli/internal/helm/loader_test.go | 3 +- 26 files changed, 579 insertions(+), 308 deletions(-) delete mode 100644 bootstrapper/internal/kubernetes/k8sapi/resources/cloud_controller_manager.go delete mode 100644 bootstrapper/internal/kubernetes/k8sapi/resources/cloud_controller_manager_test.go delete mode 100644 cli/internal/cmd/helmloader.go create mode 100644 cli/internal/helm/charts/edgeless/constellation-services/charts/ccm/.helmignore create mode 100644 cli/internal/helm/charts/edgeless/constellation-services/charts/ccm/Chart.yaml create mode 100644 cli/internal/helm/charts/edgeless/constellation-services/charts/ccm/templates/aws-daemonset.yaml create mode 100644 cli/internal/helm/charts/edgeless/constellation-services/charts/ccm/templates/azure-daemonset.yaml create mode 100644 cli/internal/helm/charts/edgeless/constellation-services/charts/ccm/templates/azure-secret.yaml create mode 100644 cli/internal/helm/charts/edgeless/constellation-services/charts/ccm/templates/clusterrolebinding.yaml create mode 100644 cli/internal/helm/charts/edgeless/constellation-services/charts/ccm/templates/gcp-cm.yaml create mode 100644 cli/internal/helm/charts/edgeless/constellation-services/charts/ccm/templates/gcp-daemonset.yaml create mode 100644 cli/internal/helm/charts/edgeless/constellation-services/charts/ccm/templates/gcp-secret.yaml create mode 100644 cli/internal/helm/charts/edgeless/constellation-services/charts/ccm/templates/serviceaccount.yaml create mode 100644 cli/internal/helm/charts/edgeless/constellation-services/charts/ccm/values.schema.json create mode 100644 cli/internal/helm/charts/edgeless/constellation-services/charts/ccm/values.yaml diff --git a/CMakeLists.txt b/CMakeLists.txt index ba1252158..fdb05bfb1 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -63,7 +63,9 @@ add_test(NAME integration-node-operator COMMAND make test WORKING_DIRECTORY ${CM add_test(NAME integration-csi COMMAND bash -c "go test -tags integration -c ./test/ && sudo ./test.test -test.v" WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}/csi) add_test(NAME integration-dm COMMAND bash -c "go test -tags integration -c ./test/ && sudo ./test.test -test.v" WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}/disk-mapper/internal) add_test(NAME integration-license COMMAND bash -c "go test -tags integration" WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}/internal/license) -add_test(NAME helm-lint COMMAND bash -c "helm lint * --set kms.image='ghcr.io/edgelesssys/constellation/kms:latest' --set join-service.csp='QEMU' \ +add_test(NAME helm-lint COMMAND bash -c "helm lint * --set tags.GCP=true --set kms.image='ghcr.io/edgelesssys/constellation/kms:latest' --set join-service.csp='GCP' \ --set join-service.enforcedPCRs='[]' --set join-service.image='ghcr.io/edgelesssys/constellation/join-service:latest' --set join-service.measurements='[]' \ - --set join-service.measurementSalt='deadbeef' --set kms.salt='aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa' --set kms.masterSecret='aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa'" + --set join-service.measurementSalt='deadbeef' --set kms.salt='aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa' --set kms.masterSecret='aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa' \ + --set ccm.csp='GCP' --set ccm.subnetworkCIDR='192.0.2.0/24' --set ccm.GCP.image='ghcr.io/edgelesssys/cloud-provider-gcp:latest' --set ccm.GCP.projectID='demoproject-581925' \ + --set ccm.GCP.uid='foobar' --set ccm.GCP.secretData='jsonstring'" WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}/cli/internal/helm/charts/edgeless/) diff --git a/bootstrapper/internal/kubernetes/k8sapi/constants.go b/bootstrapper/internal/kubernetes/k8sapi/constants.go index f4b016034..472aba533 100644 --- a/bootstrapper/internal/kubernetes/k8sapi/constants.go +++ b/bootstrapper/internal/kubernetes/k8sapi/constants.go @@ -14,5 +14,4 @@ const ( kubeletPath = "/run/state/bin/kubelet" kubeletServicePath = "/usr/lib/systemd/system/kubelet.service" executablePerm = 0o544 - systemdUnitPerm = 0o644 ) diff --git a/bootstrapper/internal/kubernetes/k8sapi/kubectl/client/client_test.go b/bootstrapper/internal/kubernetes/k8sapi/kubectl/client/client_test.go index f9bffc3d6..a52809cd4 100644 --- a/bootstrapper/internal/kubernetes/k8sapi/kubectl/client/client_test.go +++ b/bootstrapper/internal/kubernetes/k8sapi/kubectl/client/client_test.go @@ -280,11 +280,6 @@ func TestGetObjects(t *testing.T) { resourcesYAML: string(nginxDeplYAML), wantErr: false, }, - "GetObjects works on cloud-controller-manager deployment": { - wantResources: resources.NewDefaultCloudControllerManagerDeployment("someProvider", "someImage", "somePath", "someCIDR", nil, nil, nil, nil), - resourcesYAML: string(nginxDeplYAML), - wantErr: false, - }, "GetObjects Marshal failure detected": { wantResources: &unmarshableResource{}, resourcesYAML: string(nginxDeplYAML), diff --git a/bootstrapper/internal/kubernetes/k8sapi/resources/cloud_controller_manager.go b/bootstrapper/internal/kubernetes/k8sapi/resources/cloud_controller_manager.go deleted file mode 100644 index cff53c37c..000000000 --- a/bootstrapper/internal/kubernetes/k8sapi/resources/cloud_controller_manager.go +++ /dev/null @@ -1,179 +0,0 @@ -/* -Copyright (c) Edgeless Systems GmbH - -SPDX-License-Identifier: AGPL-3.0-only -*/ - -package resources - -import ( - "fmt" - - "github.com/edgelesssys/constellation/v2/internal/kubernetes" - apps "k8s.io/api/apps/v1" - k8s "k8s.io/api/core/v1" - rbac "k8s.io/api/rbac/v1" - meta "k8s.io/apimachinery/pkg/apis/meta/v1" -) - -type CloudControllerManagerDeployment struct { - ServiceAccount k8s.ServiceAccount - ClusterRoleBinding rbac.ClusterRoleBinding - DaemonSet apps.DaemonSet -} - -// references: -// https://raw.githubusercontent.com/kubernetes/website/main/content/en/examples/admin/cloud/ccm-example.yaml -// https://kubernetes.io/docs/tasks/administer-cluster/running-cloud-controller/#cloud-controller-manager - -// NewDefaultCloudControllerManagerDeployment creates a new *cloudControllerManagerDeployment, customized for the CSP. -func NewDefaultCloudControllerManagerDeployment(cloudProvider, image, path, podCIDR string, extraArgs []string, extraVolumes []k8s.Volume, extraVolumeMounts []k8s.VolumeMount, env []k8s.EnvVar) *CloudControllerManagerDeployment { - command := []string{ - path, - fmt.Sprintf("--cloud-provider=%s", cloudProvider), - "--leader-elect=true", - fmt.Sprintf("--cluster-cidr=%s", podCIDR), - "-v=2", - } - command = append(command, extraArgs...) - volumes := []k8s.Volume{ - { - Name: "etckubernetes", - VolumeSource: k8s.VolumeSource{ - HostPath: &k8s.HostPathVolumeSource{Path: "/etc/kubernetes"}, - }, - }, - { - Name: "etcssl", - VolumeSource: k8s.VolumeSource{ - HostPath: &k8s.HostPathVolumeSource{Path: "/etc/ssl"}, - }, - }, - { - Name: "etcpki", - VolumeSource: k8s.VolumeSource{ - HostPath: &k8s.HostPathVolumeSource{Path: "/etc/pki"}, - }, - }, - } - volumes = append(volumes, extraVolumes...) - volumeMounts := []k8s.VolumeMount{ - { - MountPath: "/etc/kubernetes", - Name: "etckubernetes", - ReadOnly: true, - }, - { - MountPath: "/etc/ssl", - Name: "etcssl", - ReadOnly: true, - }, - { - MountPath: "/etc/pki", - Name: "etcpki", - ReadOnly: true, - }, - } - volumeMounts = append(volumeMounts, extraVolumeMounts...) - - return &CloudControllerManagerDeployment{ - ServiceAccount: k8s.ServiceAccount{ - TypeMeta: meta.TypeMeta{ - APIVersion: "v1", - Kind: "ServiceAccount", - }, - ObjectMeta: meta.ObjectMeta{ - Name: "cloud-controller-manager", - Namespace: "kube-system", - }, - }, - ClusterRoleBinding: rbac.ClusterRoleBinding{ - TypeMeta: meta.TypeMeta{ - APIVersion: "rbac.authorization.k8s.io/v1", - Kind: "ClusterRoleBinding", - }, - ObjectMeta: meta.ObjectMeta{ - Name: "system:cloud-controller-manager", - }, - RoleRef: rbac.RoleRef{ - APIGroup: "rbac.authorization.k8s.io", - Kind: "ClusterRole", - Name: "cluster-admin", - }, - Subjects: []rbac.Subject{ - { - Kind: "ServiceAccount", - Name: "cloud-controller-manager", - Namespace: "kube-system", - }, - }, - }, - DaemonSet: apps.DaemonSet{ - TypeMeta: meta.TypeMeta{ - APIVersion: "apps/v1", - Kind: "DaemonSet", - }, - ObjectMeta: meta.ObjectMeta{ - Labels: map[string]string{ - "k8s-app": "cloud-controller-manager", - }, - Name: "cloud-controller-manager", - Namespace: "kube-system", - }, - Spec: apps.DaemonSetSpec{ - Selector: &meta.LabelSelector{ - MatchLabels: map[string]string{ - "k8s-app": "cloud-controller-manager", - }, - }, - Template: k8s.PodTemplateSpec{ - ObjectMeta: meta.ObjectMeta{ - Labels: map[string]string{ - "k8s-app": "cloud-controller-manager", - }, - }, - Spec: k8s.PodSpec{ - ServiceAccountName: "cloud-controller-manager", - Containers: []k8s.Container{ - { - Name: "cloud-controller-manager", - Image: image, - Command: command, - VolumeMounts: volumeMounts, - Env: env, - }, - }, - Volumes: volumes, - Tolerations: []k8s.Toleration{ - { - Key: "node.cloudprovider.kubernetes.io/uninitialized", - Value: "true", - Effect: k8s.TaintEffectNoSchedule, - }, - { - Key: "node-role.kubernetes.io/master", - Effect: k8s.TaintEffectNoSchedule, - }, - { - Key: "node-role.kubernetes.io/control-plane", - Operator: k8s.TolerationOpExists, - Effect: k8s.TaintEffectNoSchedule, - }, - { - Key: "node.kubernetes.io/not-ready", - Effect: k8s.TaintEffectNoSchedule, - }, - }, - NodeSelector: map[string]string{ - "node-role.kubernetes.io/control-plane": "", - }, - }, - }, - }, - }, - } -} - -func (c *CloudControllerManagerDeployment) Marshal() ([]byte, error) { - return kubernetes.MarshalK8SResources(c) -} diff --git a/bootstrapper/internal/kubernetes/k8sapi/resources/cloud_controller_manager_test.go b/bootstrapper/internal/kubernetes/k8sapi/resources/cloud_controller_manager_test.go deleted file mode 100644 index decab949c..000000000 --- a/bootstrapper/internal/kubernetes/k8sapi/resources/cloud_controller_manager_test.go +++ /dev/null @@ -1,29 +0,0 @@ -/* -Copyright (c) Edgeless Systems GmbH - -SPDX-License-Identifier: AGPL-3.0-only -*/ - -package resources - -import ( - "testing" - - "github.com/edgelesssys/constellation/v2/internal/kubernetes" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - k8s "k8s.io/api/core/v1" -) - -func TestCloudControllerMarshalUnmarshal(t *testing.T) { - require := require.New(t) - assert := assert.New(t) - - cloudControllerManagerDepl := NewDefaultCloudControllerManagerDeployment("dummy-cloudprovider", "some-image:latest", "/dummy_path", "192.0.2.0/24", []string{}, []k8s.Volume{}, []k8s.VolumeMount{}, nil) - data, err := cloudControllerManagerDepl.Marshal() - require.NoError(err) - - var recreated CloudControllerManagerDeployment - require.NoError(kubernetes.UnmarshalK8SResources(data, &recreated)) - assert.Equal(cloudControllerManagerDepl, &recreated) -} diff --git a/bootstrapper/internal/kubernetes/kubernetes.go b/bootstrapper/internal/kubernetes/kubernetes.go index 37fd9f74e..d1d8f2017 100644 --- a/bootstrapper/internal/kubernetes/kubernetes.go +++ b/bootstrapper/internal/kubernetes/kubernetes.go @@ -11,6 +11,7 @@ import ( "encoding/base64" "encoding/hex" "encoding/json" + "errors" "fmt" "net" "strconv" @@ -18,9 +19,11 @@ import ( "github.com/edgelesssys/constellation/v2/bootstrapper/internal/kubernetes/k8sapi" "github.com/edgelesssys/constellation/v2/bootstrapper/internal/kubernetes/k8sapi/resources" + "github.com/edgelesssys/constellation/v2/internal/cloud/cloudprovider" "github.com/edgelesssys/constellation/v2/internal/cloud/metadata" "github.com/edgelesssys/constellation/v2/internal/constants" "github.com/edgelesssys/constellation/v2/internal/deploy/helm" + "github.com/edgelesssys/constellation/v2/internal/gcpshared" "github.com/edgelesssys/constellation/v2/internal/logger" "github.com/edgelesssys/constellation/v2/internal/role" "github.com/edgelesssys/constellation/v2/internal/versions" @@ -201,7 +204,10 @@ func (k *KubeWrapper) InitCluster( return nil, fmt.Errorf("setting up konnectivity: %w", err) } - extraVals := setupExtraVals(k.initialMeasurementsJSON, idKeyDigest, measurementSalt) + extraVals, err := k.setupExtraVals(ctx, k.initialMeasurementsJSON, idKeyDigest, measurementSalt, subnetworkPodCIDR, cloudServiceAccountURI) + if err != nil { + return nil, fmt.Errorf("setting up extraVals: %w", err) + } if err = k.helmClient.InstallConstellationServices(ctx, helmReleases.ConstellationServices, extraVals); err != nil { return nil, fmt.Errorf("installing constellation-services: %w", err) @@ -211,9 +217,6 @@ func (k *KubeWrapper) InitCluster( return nil, fmt.Errorf("failed to setup internal ConfigMap: %w", err) } - if err := k.setupCCM(ctx, subnetworkPodCIDR, cloudServiceAccountURI, instance, k8sVersion); err != nil { - return nil, fmt.Errorf("setting up cloud controller manager: %w", err) - } if err := k.setupCloudNodeManager(k8sVersion); err != nil { return nil, fmt.Errorf("setting up cloud node manager: %w", err) } @@ -328,34 +331,6 @@ func (k *KubeWrapper) GetKubeconfig() ([]byte, error) { return k.kubeconfigReader.ReadKubeconfig() } -func (k *KubeWrapper) setupCCM(ctx context.Context, subnetworkPodCIDR, cloudServiceAccountURI string, instance metadata.InstanceMetadata, k8sVersion versions.ValidK8sVersion) error { - if !k.cloudControllerManager.Supported() { - return nil - } - ccmConfigMaps, err := k.cloudControllerManager.ConfigMaps() - if err != nil { - return fmt.Errorf("defining ConfigMaps for CCM: %w", err) - } - ccmSecrets, err := k.cloudControllerManager.Secrets(ctx, instance.ProviderID, cloudServiceAccountURI) - if err != nil { - return fmt.Errorf("defining Secrets for CCM: %w", err) - } - ccmImage, err := k.cloudControllerManager.Image(k8sVersion) - if err != nil { - return fmt.Errorf("defining Image for CCM: %w", err) - } - - cloudControllerManagerConfiguration := resources.NewDefaultCloudControllerManagerDeployment( - k.cloudControllerManager.Name(), ccmImage, k.cloudControllerManager.Path(), subnetworkPodCIDR, - k.cloudControllerManager.ExtraArgs(), k.cloudControllerManager.Volumes(), k.cloudControllerManager.VolumeMounts(), k.cloudControllerManager.Env(), - ) - if err := k.clusterUtil.SetupCloudControllerManager(k.client, cloudControllerManagerConfiguration, ccmConfigMaps, ccmSecrets); err != nil { - return fmt.Errorf("setting up cloud-controller-manager: %w", err) - } - - return nil -} - func (k *KubeWrapper) setupCloudNodeManager(k8sVersion versions.ValidK8sVersion) error { if !k.cloudNodeManager.Supported() { return nil @@ -498,12 +473,82 @@ func getIPAddr() (string, error) { // setupExtraVals create a helm values map for consumption by helm-install. // Will move to a more dedicated place once that place becomes apparent. -func setupExtraVals(initialMeasurementsJSON []byte, idkeydigest []byte, measurementSalt []byte) map[string]any { - return map[string]any{ +func (k *KubeWrapper) setupExtraVals(ctx context.Context, initialMeasurementsJSON []byte, idkeydigest []byte, measurementSalt []byte, subnetworkCIDR string, cloudServiceAccountURI string) (map[string]any, error) { + extraVals := map[string]any{ "join-service": map[string]any{ "measurements": string(initialMeasurementsJSON), - "idkeydigest": hex.EncodeToString(idkeydigest), "measurementSalt": base64.StdEncoding.EncodeToString(measurementSalt), }, + "ccm": map[string]any{ + "subnetworkCIDR": subnetworkCIDR, + }, } + + instance, err := k.providerMetadata.Self(ctx) + if err != nil { + return nil, fmt.Errorf("retrieving current instance: %w", err) + } + + switch cloudprovider.FromString(k.cloudProvider) { + case cloudprovider.GCP: + { + uid, err := k.providerMetadata.UID(ctx) + if err != nil { + return nil, fmt.Errorf("getting uid: %w", err) + } + + projectID, _, _, err := gcpshared.SplitProviderID(instance.ProviderID) + if err != nil { + return nil, fmt.Errorf("splitting providerID: %w", err) + } + + serviceAccountKey, err := gcpshared.ServiceAccountKeyFromURI(cloudServiceAccountURI) + if err != nil { + return nil, fmt.Errorf("getting service account key: %w", err) + } + rawKey, err := json.Marshal(serviceAccountKey) + if err != nil { + return nil, fmt.Errorf("marshaling service account key: %w", err) + } + + ccmVals, ok := extraVals["ccm"].(map[string]any) + if !ok { + return nil, errors.New("invalid ccm values") + } + ccmVals["GCP"] = map[string]any{ + "projectID": projectID, + "uid": uid, + "secretData": string(rawKey), + } + } + case cloudprovider.Azure: + { + // TODO: After refactoring the ProviderMetadata interface this section should be rewritten. + // Currently, we have to rely on the Secrets(..) method, as GetNetworkSecurityGroupName & GetLoadBalancerName + // rely on Azure specific API endpoints. + ccmSecrets, err := k.cloudControllerManager.Secrets(ctx, instance.ProviderID, cloudServiceAccountURI) + if err != nil { + return nil, fmt.Errorf("creating ccm secret: %w", err) + } + if len(ccmSecrets) < 1 { + return nil, errors.New("missing secret") + } + rawConfig := ccmSecrets[0].Data["azure.json"] + + ccmVals, ok := extraVals["ccm"].(map[string]any) + if !ok { + return nil, errors.New("invalid ccm values") + } + ccmVals["Azure"] = map[string]any{ + "azureConfig": string(rawConfig), + } + + joinVals, ok := extraVals["join-service"].(map[string]any) + if !ok { + return nil, errors.New("invalid join-service values") + } + joinVals["idkeydigest"] = hex.EncodeToString(idkeydigest) + } + } + return extraVals, nil } diff --git a/bootstrapper/internal/kubernetes/kubernetes_test.go b/bootstrapper/internal/kubernetes/kubernetes_test.go index f9050d007..539935f56 100644 --- a/bootstrapper/internal/kubernetes/kubernetes_test.go +++ b/bootstrapper/internal/kubernetes/kubernetes_test.go @@ -191,7 +191,7 @@ func TestInitCluster(t *testing.T) { wantErr: true, k8sVersion: versions.Default, }, - "kubeadm init fails when setting up the join service": { + "kubeadm init fails when setting up constellation-services chart": { clusterUtil: stubClusterUtil{}, helmClient: stubHelmClient{servicesError: someErr}, kubeconfigReader: &stubKubeconfigReader{ @@ -204,18 +204,6 @@ func TestInitCluster(t *testing.T) { wantErr: true, k8sVersion: versions.Default, }, - "kubeadm init fails when setting the cloud contoller manager": { - clusterUtil: stubClusterUtil{setupCloudControllerManagerError: someErr}, - kubeconfigReader: &stubKubeconfigReader{ - Kubeconfig: []byte("someKubeconfig"), - }, - providerMetadata: &stubProviderMetadata{}, - CloudControllerManager: &stubCloudControllerManager{SupportedResp: true}, - CloudNodeManager: &stubCloudNodeManager{}, - ClusterAutoscaler: &stubClusterAutoscaler{}, - wantErr: true, - k8sVersion: versions.Default, - }, "kubeadm init fails when setting the cloud node manager": { clusterUtil: stubClusterUtil{setupCloudNodeManagerError: someErr}, kubeconfigReader: &stubKubeconfigReader{ @@ -252,19 +240,6 @@ func TestInitCluster(t *testing.T) { wantErr: true, k8sVersion: versions.Default, }, - "kubeadm init fails when setting up the kms": { - clusterUtil: stubClusterUtil{}, - helmClient: stubHelmClient{servicesError: someErr}, - kubeconfigReader: &stubKubeconfigReader{ - Kubeconfig: []byte("someKubeconfig"), - }, - providerMetadata: &stubProviderMetadata{SupportedResp: false}, - CloudControllerManager: &stubCloudControllerManager{}, - CloudNodeManager: &stubCloudNodeManager{SupportedResp: false}, - ClusterAutoscaler: &stubClusterAutoscaler{}, - wantErr: true, - k8sVersion: versions.Default, - }, "kubeadm init fails when setting up konnectivity": { clusterUtil: stubClusterUtil{setupKonnectivityError: someErr}, kubeconfigReader: &stubKubeconfigReader{ diff --git a/cli/internal/cmd/helmloader.go b/cli/internal/cmd/helmloader.go deleted file mode 100644 index cc547abce..000000000 --- a/cli/internal/cmd/helmloader.go +++ /dev/null @@ -1,21 +0,0 @@ -/* -Copyright (c) Edgeless Systems GmbH - -SPDX-License-Identifier: AGPL-3.0-only -*/ - -package cmd - -import "github.com/edgelesssys/constellation/v2/internal/cloud/cloudprovider" - -type helmLoader interface { - Load(csp cloudprovider.Provider, conformanceMode bool, masterSecret []byte, salt []byte, enforcedPCRs []uint32, enforceIDKeyDigest bool) ([]byte, error) -} - -type stubHelmLoader struct { - loadErr error -} - -func (d *stubHelmLoader) Load(csp cloudprovider.Provider, conformanceMode bool, masterSecret []byte, salt []byte, enforcedPCRs []uint32, enforceIDKeyDigest bool) ([]byte, error) { - return nil, d.loadErr -} diff --git a/cli/internal/cmd/init.go b/cli/internal/cmd/init.go index 34c9ea2d9..5d52cfe1c 100644 --- a/cli/internal/cmd/init.go +++ b/cli/internal/cmd/init.go @@ -126,7 +126,7 @@ func initialize(cmd *cobra.Command, newDialer func(validator *cloudcmd.Validator return fmt.Errorf("parsing or generating master secret from file %s: %w", flags.masterSecretPath, err) } - helmDeployments, err := helmLoader.Load(provider, flags.conformance, masterSecret.Key, masterSecret.Salt, getEnforcedPCRs(provider, config), getEnforceIDKeyDigest(provider, config)) + helmDeployments, err := helmLoader.Load(provider, flags.conformance, masterSecret.Key, masterSecret.Salt, getEnforcedPCRs(provider, config), getEnforceIDKeyDigest(provider, config), k8sVersion) if err != nil { return fmt.Errorf("loading Helm charts: %w", err) } @@ -368,3 +368,7 @@ func getMarshaledServiceAccountURI(provider cloudprovider.Provider, config *conf type grpcDialer interface { Dial(ctx context.Context, target string) (*grpc.ClientConn, error) } + +type helmLoader interface { + Load(csp cloudprovider.Provider, conformanceMode bool, masterSecret []byte, salt []byte, enforcedPCRs []uint32, enforceIDKeyDigest bool, k8sVersion versions.ValidK8sVersion) ([]byte, error) +} diff --git a/cli/internal/cmd/init_test.go b/cli/internal/cmd/init_test.go index 37a02cb32..88c304876 100644 --- a/cli/internal/cmd/init_test.go +++ b/cli/internal/cmd/init_test.go @@ -31,6 +31,7 @@ import ( "github.com/edgelesssys/constellation/v2/internal/grpc/testdialer" "github.com/edgelesssys/constellation/v2/internal/license" "github.com/edgelesssys/constellation/v2/internal/oid" + "github.com/edgelesssys/constellation/v2/internal/versions" "github.com/spf13/afero" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -503,3 +504,11 @@ func (c *stubLicenseClient) QuotaCheck(ctx context.Context, checkRequest license Quota: 25, }, nil } + +type stubHelmLoader struct { + loadErr error +} + +func (d *stubHelmLoader) Load(csp cloudprovider.Provider, conformanceMode bool, masterSecret []byte, salt []byte, enforcedPCRs []uint32, enforceIDKeyDigest bool, k8sVersion versions.ValidK8sVersion) ([]byte, error) { + return nil, d.loadErr +} diff --git a/cli/internal/helm/charts/edgeless/constellation-services/Chart.yaml b/cli/internal/helm/charts/edgeless/constellation-services/Chart.yaml index afd5ece1a..1eaefd701 100644 --- a/cli/internal/helm/charts/edgeless/constellation-services/Chart.yaml +++ b/cli/internal/helm/charts/edgeless/constellation-services/Chart.yaml @@ -7,5 +7,21 @@ version: 2.2.0-pre dependencies: - name: kms version: 2.2.0-pre + tags: + - Azure + - GCP + - AWS + - QEMU - name: join-service version: 2.2.0-pre + tags: + - Azure + - GCP + - AWS + - QEMU + - name: ccm + version: 2.2.0-pre + tags: + - Azure + - GCP + - AWS diff --git a/cli/internal/helm/charts/edgeless/constellation-services/charts/ccm/.helmignore b/cli/internal/helm/charts/edgeless/constellation-services/charts/ccm/.helmignore new file mode 100644 index 000000000..0e8a0eb36 --- /dev/null +++ b/cli/internal/helm/charts/edgeless/constellation-services/charts/ccm/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/cli/internal/helm/charts/edgeless/constellation-services/charts/ccm/Chart.yaml b/cli/internal/helm/charts/edgeless/constellation-services/charts/ccm/Chart.yaml new file mode 100644 index 000000000..56a7519da --- /dev/null +++ b/cli/internal/helm/charts/edgeless/constellation-services/charts/ccm/Chart.yaml @@ -0,0 +1,5 @@ +apiVersion: v2 +name: ccm +description: A Helm chart to deploy the cloud controller manager. +type: application +version: 2.2.0-pre diff --git a/cli/internal/helm/charts/edgeless/constellation-services/charts/ccm/templates/aws-daemonset.yaml b/cli/internal/helm/charts/edgeless/constellation-services/charts/ccm/templates/aws-daemonset.yaml new file mode 100644 index 000000000..515eec82e --- /dev/null +++ b/cli/internal/helm/charts/edgeless/constellation-services/charts/ccm/templates/aws-daemonset.yaml @@ -0,0 +1,63 @@ +{{ if eq .Values.csp "AWS" }} +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: cloud-controller-manager + namespace: {{ .Release.Namespace }} + labels: + k8s-app: cloud-controller-manager +spec: + selector: + matchLabels: + k8s-app: cloud-controller-manager + template: + metadata: + labels: + k8s-app: cloud-controller-manager + spec: + containers: + - name: cloud-controller-manager + image: {{ .Values.AWS.image }} + command: + - /aws-cloud-controller-manager + - --cloud-provider=aws + - --leader-elect=true + - --cluster-cidr={{ .Values.subnetworkCIDR }} + - -v=2 + volumeMounts: + - name: etckubernetes + mountPath: /etc/kubernetes + readOnly: true + - name: etcssl + mountPath: /etc/ssl + readOnly: true + - name: etcpki + mountPath: /etc/pki + readOnly: true + resources: {} + nodeSelector: + node-role.kubernetes.io/control-plane: "" + serviceAccountName: cloud-controller-manager + tolerations: + - effect: NoSchedule + key: node.cloudprovider.kubernetes.io/uninitialized + value: "true" + - effect: NoSchedule + key: node-role.kubernetes.io/master + - effect: NoSchedule + key: node-role.kubernetes.io/control-plane + operator: Exists + - effect: NoSchedule + key: node.kubernetes.io/not-ready + volumes: + - name: etckubernetes + hostPath: + path: /etc/kubernetes + - name: etcssl + hostPath: + path: /etc/ssl + - name: etcpki + hostPath: + path: /etc/pki + updateStrategy: {} +{{ end }} diff --git a/cli/internal/helm/charts/edgeless/constellation-services/charts/ccm/templates/azure-daemonset.yaml b/cli/internal/helm/charts/edgeless/constellation-services/charts/ccm/templates/azure-daemonset.yaml new file mode 100644 index 000000000..ba5ffcb74 --- /dev/null +++ b/cli/internal/helm/charts/edgeless/constellation-services/charts/ccm/templates/azure-daemonset.yaml @@ -0,0 +1,73 @@ +{{ if eq .Values.csp "Azure" }} +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: cloud-controller-manager + namespace: {{ .Release.Namespace}} + labels: + k8s-app: cloud-controller-manager +spec: + selector: + matchLabels: + k8s-app: cloud-controller-manager + template: + metadata: + labels: + k8s-app: cloud-controller-manager + spec: + containers: + - name: cloud-controller-manager + image: {{ .Values.Azure.image }} + command: + - cloud-controller-manager + - --cloud-provider=azure + - --leader-elect=true + - --cluster-cidr={{ .Values.subnetworkCIDR }} + - -v=2 + - --controllers=*,-cloud-node + - --cloud-config=/etc/azure/azure.json + - --allocate-node-cidrs=false + - --configure-cloud-routes=true + volumeMounts: + - name: etckubernetes + mountPath: /etc/kubernetes + readOnly: true + - name: etcssl + mountPath: /etc/ssl + readOnly: true + - name: etcpki + mountPath: /etc/pki + readOnly: true + - name: azureconfig + mountPath: /etc/azure + readOnly: true + resources: {} + nodeSelector: + node-role.kubernetes.io/control-plane: "" + serviceAccountName: cloud-controller-manager + tolerations: + - effect: NoSchedule + key: node.cloudprovider.kubernetes.io/uninitialized + value: "true" + - effect: NoSchedule + key: node-role.kubernetes.io/master + - effect: NoSchedule + key: node-role.kubernetes.io/control-plane + operator: Exists + - effect: NoSchedule + key: node.kubernetes.io/not-ready + volumes: + - name: etckubernetes + hostPath: + path: /etc/kubernetes + - name: etcssl + hostPath: + path: /etc/ssl + - name: etcpki + hostPath: + path: /etc/pki + - name: azureconfig + secret: + secretName: azureconfig + updateStrategy: {} +{{ end }} diff --git a/cli/internal/helm/charts/edgeless/constellation-services/charts/ccm/templates/azure-secret.yaml b/cli/internal/helm/charts/edgeless/constellation-services/charts/ccm/templates/azure-secret.yaml new file mode 100644 index 000000000..103db8cc1 --- /dev/null +++ b/cli/internal/helm/charts/edgeless/constellation-services/charts/ccm/templates/azure-secret.yaml @@ -0,0 +1,9 @@ +{{ if eq .Values.csp "Azure" }} +apiVersion: v1 +kind: Secret +metadata: + name: azureconfig + namespace: {{ .Release.Namespace }} +data: + azure.json: {{ .Values.Azure.azureConfig | b64enc }} +{{ end }} diff --git a/cli/internal/helm/charts/edgeless/constellation-services/charts/ccm/templates/clusterrolebinding.yaml b/cli/internal/helm/charts/edgeless/constellation-services/charts/ccm/templates/clusterrolebinding.yaml new file mode 100644 index 000000000..b84c9682b --- /dev/null +++ b/cli/internal/helm/charts/edgeless/constellation-services/charts/ccm/templates/clusterrolebinding.yaml @@ -0,0 +1,12 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: system:cloud-controller-manager +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cluster-admin +subjects: + - kind: ServiceAccount + name: cloud-controller-manager + namespace: {{ .Release.Namespace }} diff --git a/cli/internal/helm/charts/edgeless/constellation-services/charts/ccm/templates/gcp-cm.yaml b/cli/internal/helm/charts/edgeless/constellation-services/charts/ccm/templates/gcp-cm.yaml new file mode 100644 index 000000000..1c460da69 --- /dev/null +++ b/cli/internal/helm/charts/edgeless/constellation-services/charts/ccm/templates/gcp-cm.yaml @@ -0,0 +1,9 @@ +{{ if eq .Values.csp "GCP" }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: gceconf + namespace: {{ .Release.Namespace }} +data: + gce.conf: "[global]\nproject-id = {{.Values.GCP.projectID }}\nuse-metadata-server = true\nnode-tags = constellation-{{ .Values.GCP.uid }}\n" +{{ end }} diff --git a/cli/internal/helm/charts/edgeless/constellation-services/charts/ccm/templates/gcp-daemonset.yaml b/cli/internal/helm/charts/edgeless/constellation-services/charts/ccm/templates/gcp-daemonset.yaml new file mode 100644 index 000000000..325169a0d --- /dev/null +++ b/cli/internal/helm/charts/edgeless/constellation-services/charts/ccm/templates/gcp-daemonset.yaml @@ -0,0 +1,84 @@ +{{ if eq .Values.csp "GCP" }} +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: cloud-controller-manager + namespace: {{ .Release.Namespace }} + labels: + k8s-app: cloud-controller-manager +spec: + selector: + matchLabels: + k8s-app: cloud-controller-manager + template: + metadata: + labels: + k8s-app: cloud-controller-manager + spec: + containers: + - name: cloud-controller-manager + image: {{ .Values.GCP.image }} + command: + - /cloud-controller-manager + - --cloud-provider=gce + - --leader-elect=true + - --cluster-cidr={{ .Values.subnetworkCIDR }} + - -v=2 + - --use-service-account-credentials + - --controllers=cloud-node,cloud-node-lifecycle,nodeipam,service,route + - --cloud-config=/etc/gce/gce.conf + - --cidr-allocator-type=CloudAllocator + - --allocate-node-cidrs=true + - --configure-cloud-routes=false + volumeMounts: + - mountPath: /etc/kubernetes + name: etckubernetes + readOnly: true + - mountPath: /etc/ssl + name: etcssl + readOnly: true + - mountPath: /etc/pki + name: etcpki + readOnly: true + - mountPath: /etc/gce + name: gceconf + readOnly: true + - mountPath: /var/secrets/google + name: gcekey + readOnly: true + resources: {} + env: + - name: GOOGLE_APPLICATION_CREDENTIALS + value: /var/secrets/google/key.json + serviceAccountName: cloud-controller-manager + nodeSelector: + node-role.kubernetes.io/control-plane: "" + tolerations: + - effect: NoSchedule + key: node.cloudprovider.kubernetes.io/uninitialized + value: "true" + - effect: NoSchedule + key: node-role.kubernetes.io/master + - effect: NoSchedule + key: node-role.kubernetes.io/control-plane + operator: Exists + - effect: NoSchedule + key: node.kubernetes.io/not-ready + volumes: + - name: etckubernetes + hostPath: + path: /etc/kubernetes + - name: etcssl + hostPath: + path: /etc/ssl + - name: etcpki + hostPath: + path: /etc/pki + - name: gceconf + configMap: + name: gceconf + - name: gcekey + secret: + secretName: gcekey + updateStrategy: {} +{{ end }} diff --git a/cli/internal/helm/charts/edgeless/constellation-services/charts/ccm/templates/gcp-secret.yaml b/cli/internal/helm/charts/edgeless/constellation-services/charts/ccm/templates/gcp-secret.yaml new file mode 100644 index 000000000..4e082d8fa --- /dev/null +++ b/cli/internal/helm/charts/edgeless/constellation-services/charts/ccm/templates/gcp-secret.yaml @@ -0,0 +1,9 @@ +{{ if eq .Values.csp "GCP" }} +apiVersion: v1 +kind: Secret +metadata: + name: gcekey + namespace: {{ .Release.Namespace }} +data: + key.json: {{ .Values.GCP.secretData | b64enc }} +{{ end }} diff --git a/cli/internal/helm/charts/edgeless/constellation-services/charts/ccm/templates/serviceaccount.yaml b/cli/internal/helm/charts/edgeless/constellation-services/charts/ccm/templates/serviceaccount.yaml new file mode 100644 index 000000000..b6f54cc2c --- /dev/null +++ b/cli/internal/helm/charts/edgeless/constellation-services/charts/ccm/templates/serviceaccount.yaml @@ -0,0 +1,5 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: cloud-controller-manager + namespace: {{ .Release.Namespace }} diff --git a/cli/internal/helm/charts/edgeless/constellation-services/charts/ccm/values.schema.json b/cli/internal/helm/charts/edgeless/constellation-services/charts/ccm/values.schema.json new file mode 100644 index 000000000..6ae4b6f3a --- /dev/null +++ b/cli/internal/helm/charts/edgeless/constellation-services/charts/ccm/values.schema.json @@ -0,0 +1,105 @@ +{ + "$schema": "https://json-schema.org/draft-07/schema#", + "properties": { + "csp": { + "description": "CSP to which the chart is deployed.", + "enum": ["Azure", "GCP", "AWS", "QEMU"] + }, + "subnetworkCIDR": { + "description": "CIDR for the subnetwork of the cluster", + "type": "string", + "examples": ["192.0.2.0/24"], + "pattern": "[0-9]{1,3}.[0-9]{1,3}.[0-9]{1,3}.[0-9]{1,3}/[0-9]{1,2}" + }, + "GCP": { + "description": "Config values required for deployment on GCP", + "type": "object", + "properties": { + "image": { + "description": "Container image to use for the spawned pods.", + "type": "string" + }, + "projectID": { + "description": "ID of the GCP project into which the cluster is deployed", + "type": "string", + "examples": ["demoproject-581925"] + }, + "uid": { + "description": "Unique identifier for the cluster", + "type": "string" + }, + "secretData": { + "description": "GCP service account key as a json-string", + "type": "string" + } + }, + "required": [ + "image", + "projectID", + "uid", + "secretData" + ] + }, + "Azure": { + "description": "Config values required for deployment on Azure", + "type": "object", + "properties": { + "image": { + "description": "Container image to use for the spawned pods.", + "type": "string", + "examples": ["mcr.microsoft.com/oss/kubernetes/azure-cloud-controller-manager:latest"] + }, + "azureConfig": { + "description": "Base64 encoded json string that hold required config parameters for Azure CCM.", + "type": "string" + } + }, + "required": [ + "image", + "azureConfig" + ] + }, + "AWS": { + "description": "Config values required for deployment on AWS", + "type": "object", + "properties": { + "image": { + "description": "Container image to use for the spawned pods.", + "type": "string" + } + }, + "required": [ + "image" + ] + } + }, + "required": [ + "csp", + "subnetworkCIDR" + ], + "allOf": [ + { + "if": { + "properties": { "csp": { "const": "GCP" } }, + "required": ["csp"] + }, + "then": { "required": ["GCP"] } + }, + { + "if": { + "properties": { "csp": { "const": "Azure" } }, + "required": ["csp"] + }, + "then": { "required": ["Azure"] } + }, + { + "if": { + "properties": { "csp": { "const": "AWS" } }, + "required": ["csp"] + }, + "then": { "required": ["AWS"] } + } + ], + "title": "Values", + "type": "object" +} diff --git a/cli/internal/helm/charts/edgeless/constellation-services/charts/ccm/values.yaml b/cli/internal/helm/charts/edgeless/constellation-services/charts/ccm/values.yaml new file mode 100644 index 000000000..e69de29bb diff --git a/cli/internal/helm/charts/edgeless/constellation-services/values.yaml b/cli/internal/helm/charts/edgeless/constellation-services/values.yaml index 209db6339..70f52dad7 100644 --- a/cli/internal/helm/charts/edgeless/constellation-services/values.yaml +++ b/cli/internal/helm/charts/edgeless/constellation-services/values.yaml @@ -9,3 +9,10 @@ global: k8sVersionCMName: k8s-version # Name of the ConfigMap that holds configs that should not be modified by the user. internalCMName: internal-config + +# Set one of the tags to true to indicate which CSP you are deploying to. +tags: + Azure: false + GCP: false + AWS: false + QEMU: false diff --git a/cli/internal/helm/loader.go b/cli/internal/helm/loader.go index 67941c152..9806898d4 100644 --- a/cli/internal/helm/loader.go +++ b/cli/internal/helm/loader.go @@ -36,13 +36,13 @@ var HelmFS embed.FS type ChartLoader struct{} -func (i *ChartLoader) Load(csp cloudprovider.Provider, conformanceMode bool, masterSecret []byte, salt []byte, enforcedPCRs []uint32, enforceIDKeyDigest bool) ([]byte, error) { +func (i *ChartLoader) Load(csp cloudprovider.Provider, conformanceMode bool, masterSecret []byte, salt []byte, enforcedPCRs []uint32, enforceIDKeyDigest bool, k8sVersion versions.ValidK8sVersion) ([]byte, error) { ciliumRelease, err := i.loadCilium(csp, conformanceMode) if err != nil { return nil, err } - conServicesRelease, err := i.loadConstellationServices(csp, masterSecret, salt, enforcedPCRs, enforceIDKeyDigest) + conServicesRelease, err := i.loadConstellationServices(csp, masterSecret, salt, enforcedPCRs, enforceIDKeyDigest, k8sVersion) if err != nil { return nil, err } @@ -93,7 +93,7 @@ func (i *ChartLoader) loadCilium(csp cloudprovider.Provider, conformanceMode boo // loadConstellationServices loads the constellation-services chart from the embed.FS, marshals it into a helm-package .tgz and sets the values that can be set in the CLI. func (i *ChartLoader) loadConstellationServices(csp cloudprovider.Provider, masterSecret []byte, salt []byte, enforcedPCRs []uint32, - enforceIDKeyDigest bool, + enforceIDKeyDigest bool, k8sVersion versions.ValidK8sVersion, ) (helm.Release, error) { chart, err := loadChartsDir(HelmFS, "charts/edgeless/constellation-services") if err != nil { @@ -134,14 +134,64 @@ func (i *ChartLoader) loadConstellationServices(csp cloudprovider.Provider, "image": versions.JoinImage, "namespace": constants.ConstellationNamespace, }, + "ccm": map[string]interface{}{ + "csp": csp, + }, } - if csp == cloudprovider.Azure { - joinServiceVals, ok := vals["join-service"].(map[string]any) - if !ok { - return helm.Release{}, errors.New("invalid join-service values") + switch csp { + case cloudprovider.Azure: + { + joinServiceVals, ok := vals["join-service"].(map[string]any) + if !ok { + return helm.Release{}, errors.New("invalid join-service values") + } + joinServiceVals["enforceIdKeyDigest"] = enforceIDKeyDigest + + ccmVals, ok := vals["ccm"].(map[string]any) + if !ok { + return helm.Release{}, errors.New("invalid ccm values") + } + ccmVals["Azure"] = map[string]any{ + "image": versions.VersionConfigs[k8sVersion].CloudControllerManagerImageAzure, + } + + vals["tags"] = map[string]any{ + "Azure": true, + } + } + case cloudprovider.GCP: + { + ccmVals, ok := vals["ccm"].(map[string]any) + if !ok { + return helm.Release{}, errors.New("invalid ccm values") + } + ccmVals["GCP"] = map[string]any{ + "image": versions.VersionConfigs[k8sVersion].CloudControllerManagerImageGCP, + } + + vals["tags"] = map[string]any{ + "GCP": true, + } + } + case cloudprovider.QEMU: + { + vals["tags"] = map[string]interface{}{ + "QEMU": true, + } + } + case cloudprovider.AWS: + ccmVals, ok := vals["ccm"].(map[string]any) + if !ok { + return helm.Release{}, errors.New("invalid ccm values") + } + ccmVals["AWS"] = map[string]any{ + "image": versions.VersionConfigs[k8sVersion].CloudControllerManagerImageAWS, + } + + vals["tags"] = map[string]any{ + "AWS": true, } - joinServiceVals["enforceIdKeyDigest"] = enforceIDKeyDigest } return helm.Release{Chart: chartRaw, Values: vals, ReleaseName: "constellation-services", Wait: true}, nil diff --git a/cli/internal/helm/loader_test.go b/cli/internal/helm/loader_test.go index 3d16414f8..57b55c274 100644 --- a/cli/internal/helm/loader_test.go +++ b/cli/internal/helm/loader_test.go @@ -13,6 +13,7 @@ import ( "github.com/edgelesssys/constellation/v2/internal/cloud/cloudprovider" "github.com/edgelesssys/constellation/v2/internal/deploy/helm" + "github.com/edgelesssys/constellation/v2/internal/versions" "github.com/stretchr/testify/assert" "helm.sh/helm/v3/pkg/chart/loader" ) @@ -21,7 +22,7 @@ func TestLoad(t *testing.T) { assert := assert.New(t) chartLoader := ChartLoader{} - release, err := chartLoader.Load(cloudprovider.GCP, true, []byte("secret"), []byte("salt"), nil, false) + release, err := chartLoader.Load(cloudprovider.GCP, true, []byte("secret"), []byte("salt"), nil, false, versions.Default) assert.NoError(err) var helmReleases helm.Releases