From 26305e8f80c0f875334338d74671a589eb0f9076 Mon Sep 17 00:00:00 2001 From: Adrian Stobbe Date: Mon, 31 Jul 2023 10:53:05 +0200 Subject: [PATCH] cli: install helm charts in cli instead of bootstrapper (#2136) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * init * fixup! init * gcp working? * fixup! fixup! init * azure cfg for microService installation * fixup! azure cfg for microService installation * fixup! azure cfg for microService installation * cleanup bootstrapper code * cleanup helminstall code * fixup! cleanup helminstall code * Update internal/deploy/helm/install.go Co-authored-by: Daniel Weiße <66256922+daniel-weisse@users.noreply.github.com> * daniel feedback * TODO add provider (also to CreateCluster) so we can ensure that provider specific output * fixup! daniel feedback * use debugLog in helm installer * placeholderHelmInstaller * rename to stub --------- Co-authored-by: Daniel Weiße <66256922+daniel-weisse@users.noreply.github.com> --- bootstrapper/cmd/bootstrapper/main.go | 2 +- bootstrapper/cmd/bootstrapper/test.go | 2 +- .../internal/initserver/initserver.go | 4 - .../internal/initserver/initserver_test.go | 2 +- bootstrapper/internal/kubernetes/BUILD.bazel | 2 - .../internal/kubernetes/kubernetes.go | 199 +------- .../internal/kubernetes/kubernetes_test.go | 6 +- cli/internal/cloudcmd/BUILD.bazel | 2 - cli/internal/cloudcmd/clients.go | 22 +- cli/internal/cloudcmd/clients_test.go | 24 +- cli/internal/cloudcmd/create.go | 35 +- cli/internal/cloudcmd/create_test.go | 4 +- cli/internal/cloudcmd/iam.go | 50 +- cli/internal/cloudcmd/iam_test.go | 63 +-- cli/internal/cloudcmd/rollback.go | 4 +- cli/internal/cloudcmd/terminate.go | 6 +- cli/internal/cloudcmd/terminate_test.go | 4 +- cli/internal/cmd/BUILD.bazel | 1 + cli/internal/cmd/init.go | 40 +- cli/internal/cmd/init_test.go | 12 +- cli/internal/cmd/tfmigrationclient.go | 2 +- cli/internal/helm/BUILD.bazel | 7 + cli/internal/helm/helminstaller.go | 100 ++++ cli/internal/helm/loader.go | 20 +- cli/internal/helm/setup.go | 134 ++++++ cli/internal/terraform/terraform.go | 439 +++++++++++------- .../terraform/terraform/azure/main.tf | 3 + .../terraform/terraform/azure/outputs.tf | 21 + .../terraform/terraform/gcp/outputs.tf | 12 + cli/internal/terraform/terraform_test.go | 21 +- cli/internal/upgrade/terraform.go | 19 +- cli/internal/upgrade/terraform_test.go | 2 +- internal/cloud/azure/azure.go | 94 ---- internal/cloud/azure/azure_test.go | 331 ------------- internal/cloud/azureshared/appcredentials.go | 23 + .../cloud/azureshared/appcredentials_test.go | 4 + internal/deploy/helm/BUILD.bazel | 2 - internal/deploy/helm/install.go | 27 +- 38 files changed, 775 insertions(+), 970 deletions(-) create mode 100644 cli/internal/helm/helminstaller.go create mode 100644 cli/internal/helm/setup.go diff --git a/bootstrapper/cmd/bootstrapper/main.go b/bootstrapper/cmd/bootstrapper/main.go index 1bf7da9f9..9c41c9176 100644 --- a/bootstrapper/cmd/bootstrapper/main.go +++ b/bootstrapper/cmd/bootstrapper/main.go @@ -67,7 +67,7 @@ func main() { var openDevice vtpm.TPMOpenFunc var fs afero.Fs - helmClient, err := helm.NewInstaller(log, constants.ControlPlaneAdminConfFilename) + helmClient, err := helm.NewInstaller(constants.ControlPlaneAdminConfFilename, log) if err != nil { log.With(zap.Error(err)).Fatalf("Helm client could not be initialized") } diff --git a/bootstrapper/cmd/bootstrapper/test.go b/bootstrapper/cmd/bootstrapper/test.go index b0ff5ed3c..ee7d7dea0 100644 --- a/bootstrapper/cmd/bootstrapper/test.go +++ b/bootstrapper/cmd/bootstrapper/test.go @@ -21,7 +21,7 @@ type clusterFake struct{} // InitCluster fakes bootstrapping a new cluster with the current node being the master, returning the arguments required to join the cluster. func (c *clusterFake) InitCluster( - context.Context, string, string, string, []byte, + context.Context, string, string, []byte, bool, components.Components, []string, *logger.Logger, ) ([]byte, error) { return []byte{}, nil diff --git a/bootstrapper/internal/initserver/initserver.go b/bootstrapper/internal/initserver/initserver.go index 5de431d5a..cd822890e 100644 --- a/bootstrapper/internal/initserver/initserver.go +++ b/bootstrapper/internal/initserver/initserver.go @@ -213,10 +213,8 @@ func (s *Server) Init(req *initproto.InitRequest, stream initproto.API_InitServe } kubeconfig, err := s.initializer.InitCluster(stream.Context(), - req.CloudServiceAccountUri, req.KubernetesVersion, clusterName, - measurementSalt, req.HelmDeployments, req.ConformanceMode, components.NewComponentsFromInitProto(req.KubernetesComponents), @@ -342,10 +340,8 @@ type ClusterInitializer interface { // InitCluster initializes a new Kubernetes cluster. InitCluster( ctx context.Context, - cloudServiceAccountURI string, k8sVersion string, clusterName string, - measurementSalt []byte, helmDeployments []byte, conformanceMode bool, kubernetesComponents components.Components, diff --git a/bootstrapper/internal/initserver/initserver_test.go b/bootstrapper/internal/initserver/initserver_test.go index 4abb6cea8..a2505becd 100644 --- a/bootstrapper/internal/initserver/initserver_test.go +++ b/bootstrapper/internal/initserver/initserver_test.go @@ -406,7 +406,7 @@ type stubClusterInitializer struct { } func (i *stubClusterInitializer) InitCluster( - context.Context, string, string, string, []byte, + context.Context, string, string, []byte, bool, components.Components, []string, *logger.Logger, ) ([]byte, error) { return i.initClusterKubeconfig, i.initClusterErr diff --git a/bootstrapper/internal/kubernetes/BUILD.bazel b/bootstrapper/internal/kubernetes/BUILD.bazel index 07902029c..dcddfe228 100644 --- a/bootstrapper/internal/kubernetes/BUILD.bazel +++ b/bootstrapper/internal/kubernetes/BUILD.bazel @@ -14,9 +14,7 @@ go_library( "//bootstrapper/internal/kubernetes/k8sapi", "//bootstrapper/internal/kubernetes/kubewaiter", "//internal/cloud/cloudprovider", - "//internal/cloud/gcpshared", "//internal/cloud/metadata", - "//internal/cloud/openstack", "//internal/constants", "//internal/deploy/helm", "//internal/kubernetes", diff --git a/bootstrapper/internal/kubernetes/kubernetes.go b/bootstrapper/internal/kubernetes/kubernetes.go index 76d6d811c..a49b751a3 100644 --- a/bootstrapper/internal/kubernetes/kubernetes.go +++ b/bootstrapper/internal/kubernetes/kubernetes.go @@ -9,7 +9,6 @@ package kubernetes import ( "context" - "encoding/base64" "encoding/json" "errors" "fmt" @@ -22,8 +21,6 @@ import ( "github.com/edgelesssys/constellation/v2/bootstrapper/internal/kubernetes/k8sapi" "github.com/edgelesssys/constellation/v2/bootstrapper/internal/kubernetes/kubewaiter" "github.com/edgelesssys/constellation/v2/internal/cloud/cloudprovider" - "github.com/edgelesssys/constellation/v2/internal/cloud/gcpshared" - "github.com/edgelesssys/constellation/v2/internal/cloud/openstack" "github.com/edgelesssys/constellation/v2/internal/constants" "github.com/edgelesssys/constellation/v2/internal/deploy/helm" "github.com/edgelesssys/constellation/v2/internal/kubernetes" @@ -78,7 +75,7 @@ func New(cloudProvider string, clusterUtil clusterUtil, configProvider configura // InitCluster initializes a new Kubernetes cluster and applies pod network provider. func (k *KubeWrapper) InitCluster( - ctx context.Context, cloudServiceAccountURI, versionString, clusterName string, measurementSalt []byte, + ctx context.Context, versionString, clusterName string, helmReleasesRaw []byte, conformanceMode bool, kubernetesComponents components.Components, apiServerCertSANs []string, log *logger.Logger, ) ([]byte, error) { log.With(zap.String("version", versionString)).Infof("Installing Kubernetes components") @@ -223,75 +220,10 @@ func (k *KubeWrapper) InitCluster( // Continue and don't throw an error here - things might be okay. } - serviceConfig := constellationServicesConfig{ - measurementSalt: measurementSalt, - subnetworkPodCIDR: subnetworkPodCIDR, - cloudServiceAccountURI: cloudServiceAccountURI, - loadBalancerIP: controlPlaneHost, - } - constellationVals, err := k.setupExtraVals(ctx, serviceConfig) - if err != nil { - return nil, fmt.Errorf("setting up extraVals: %w", err) - } - log.Infof("Setting up internal-config ConfigMap") if err := k.setupInternalConfigMap(ctx); err != nil { return nil, fmt.Errorf("failed to setup internal ConfigMap: %w", err) } - - log.Infof("Installing Constellation microservices") - if err = k.helmClient.InstallChartWithValues(ctx, helmReleases.ConstellationServices, constellationVals); err != nil { - return nil, fmt.Errorf("installing constellation-services: %w", err) - } - - // cert-manager provides CRDs used by other deployments, - // so it should be installed as early as possible, but after the services cert-manager depends on. - log.Infof("Installing cert-manager") - if err = k.helmClient.InstallChart(ctx, helmReleases.CertManager); err != nil { - return nil, fmt.Errorf("installing cert-manager: %w", err) - } - - // Install CSI drivers if enabled by the user. - if helmReleases.CSI != nil { - var csiVals map[string]any - if cloudprovider.FromString(k.cloudProvider) == cloudprovider.OpenStack { - creds, err := openstack.AccountKeyFromURI(serviceConfig.cloudServiceAccountURI) - if err != nil { - return nil, err - } - cinderIni := creds.CloudINI().CinderCSIConfiguration() - csiVals = map[string]any{ - "cinder-config": map[string]any{ - "secretData": cinderIni, - }, - } - } - - log.Infof("Installing CSI deployments") - if err := k.helmClient.InstallChartWithValues(ctx, *helmReleases.CSI, csiVals); err != nil { - return nil, fmt.Errorf("installing CSI snapshot CRDs: %w", err) - } - } - - if helmReleases.AWSLoadBalancerController != nil { - log.Infof("Installing AWS Load Balancer Controller") - if err = k.helmClient.InstallChart(ctx, *helmReleases.AWSLoadBalancerController); err != nil { - return nil, fmt.Errorf("installing AWS Load Balancer Controller: %w", err) - } - } - - operatorVals, err := k.setupOperatorVals(ctx) - if err != nil { - return nil, fmt.Errorf("setting up operator vals: %w", err) - } - - // Constellation operators require CRDs from cert-manager. - // They must be installed after it. - log.Infof("Installing operators") - if err = k.helmClient.InstallChartWithValues(ctx, helmReleases.ConstellationOperators, operatorVals); err != nil { - return nil, fmt.Errorf("installing operators: %w", err) - } - return kubeConfig, nil } @@ -447,120 +379,6 @@ func getIPAddr() (string, error) { return localAddr.IP.String(), nil } -// setupExtraVals create a helm values map for consumption by helm-install. -// Will move to a more dedicated place once that place becomes apparent. -func (k *KubeWrapper) setupExtraVals(ctx context.Context, serviceConfig constellationServicesConfig) (map[string]any, error) { - extraVals := map[string]any{ - "join-service": map[string]any{ - "measurementSalt": base64.StdEncoding.EncodeToString(serviceConfig.measurementSalt), - }, - "verification-service": map[string]any{ - "loadBalancerIP": serviceConfig.loadBalancerIP, - }, - "konnectivity": map[string]any{ - "loadBalancerIP": serviceConfig.loadBalancerIP, - }, - } - - instance, err := k.providerMetadata.Self(ctx) - if err != nil { - return nil, fmt.Errorf("retrieving current instance: %w", err) - } - - switch cloudprovider.FromString(k.cloudProvider) { - case cloudprovider.GCP: - uid, err := k.providerMetadata.UID(ctx) - if err != nil { - return nil, fmt.Errorf("getting uid: %w", err) - } - - projectID, _, _, err := gcpshared.SplitProviderID(instance.ProviderID) - if err != nil { - return nil, fmt.Errorf("splitting providerID: %w", err) - } - - serviceAccountKey, err := gcpshared.ServiceAccountKeyFromURI(serviceConfig.cloudServiceAccountURI) - if err != nil { - return nil, fmt.Errorf("getting service account key: %w", err) - } - rawKey, err := json.Marshal(serviceAccountKey) - if err != nil { - return nil, fmt.Errorf("marshaling service account key: %w", err) - } - - extraVals["ccm"] = map[string]any{ - "GCP": map[string]any{ - "projectID": projectID, - "uid": uid, - "secretData": string(rawKey), - "subnetworkPodCIDR": serviceConfig.subnetworkPodCIDR, - }, - } - - case cloudprovider.Azure: - ccmAzure, ok := k.providerMetadata.(ccmConfigGetter) - if !ok { - return nil, errors.New("invalid cloud provider metadata for Azure") - } - - ccmConfig, err := ccmAzure.GetCCMConfig(ctx, instance.ProviderID, serviceConfig.cloudServiceAccountURI) - if err != nil { - return nil, fmt.Errorf("creating ccm secret: %w", err) - } - - extraVals["ccm"] = map[string]any{ - "Azure": map[string]any{ - "azureConfig": string(ccmConfig), - }, - } - - case cloudprovider.OpenStack: - creds, err := openstack.AccountKeyFromURI(serviceConfig.cloudServiceAccountURI) - if err != nil { - return nil, err - } - credsIni := creds.CloudINI().FullConfiguration() - networkIDsGetter, ok := k.providerMetadata.(openstackMetadata) - if !ok { - return nil, errors.New("generating yawol configuration: cloud provider metadata does not implement OpenStack specific methods") - } - networkIDs, err := networkIDsGetter.GetNetworkIDs(ctx) - if err != nil { - return nil, fmt.Errorf("getting network IDs: %w", err) - } - if len(networkIDs) == 0 { - return nil, errors.New("getting network IDs: no network IDs found") - } - extraVals["ccm"] = map[string]any{ - "OpenStack": map[string]any{ - "secretData": credsIni, - }, - } - yawolIni := creds.CloudINI().YawolConfiguration() - extraVals["yawol-config"] = map[string]any{ - "secretData": yawolIni, - } - extraVals["yawol-controller"] = map[string]any{ - "yawolNetworkID": networkIDs[0], - "yawolAPIHost": fmt.Sprintf("https://%s:%d", serviceConfig.loadBalancerIP, constants.KubernetesPort), - } - } - return extraVals, nil -} - -func (k *KubeWrapper) setupOperatorVals(ctx context.Context) (map[string]any, error) { - uid, err := k.providerMetadata.UID(ctx) - if err != nil { - return nil, fmt.Errorf("retrieving constellation UID: %w", err) - } - - return map[string]any{ - "constellation-operator": map[string]any{ - "constellationUID": uid, - }, - }, nil -} - func (k *KubeWrapper) setupCiliumVals(ctx context.Context, in k8sapi.SetupPodNetworkInput) (map[string]any, error) { vals := map[string]any{ "k8sServiceHost": in.LoadBalancerHost, @@ -585,18 +403,3 @@ func (k *KubeWrapper) setupCiliumVals(ctx context.Context, in k8sapi.SetupPodNet return vals, nil } - -type ccmConfigGetter interface { - GetCCMConfig(ctx context.Context, providerID, cloudServiceAccountURI string) ([]byte, error) -} - -type constellationServicesConfig struct { - measurementSalt []byte - subnetworkPodCIDR string - cloudServiceAccountURI string - loadBalancerIP string -} - -type openstackMetadata interface { - GetNetworkIDs(ctx context.Context) ([]string, error) -} diff --git a/bootstrapper/internal/kubernetes/kubernetes_test.go b/bootstrapper/internal/kubernetes/kubernetes_test.go index f48d97248..7bc513907 100644 --- a/bootstrapper/internal/kubernetes/kubernetes_test.go +++ b/bootstrapper/internal/kubernetes/kubernetes_test.go @@ -35,8 +35,6 @@ func TestMain(m *testing.M) { } func TestInitCluster(t *testing.T) { - serviceAccountURI := "some-service-account-uri" - nodeName := "node-name" providerID := "provider-id" privateIP := "192.0.2.1" @@ -193,8 +191,8 @@ func TestInitCluster(t *testing.T) { } _, err := kube.InitCluster( - context.Background(), serviceAccountURI, string(tc.k8sVersion), "kubernetes", - nil, []byte("{}"), false, nil, nil, logger.NewTest(t), + context.Background(), string(tc.k8sVersion), "kubernetes", + []byte("{}"), false, nil, nil, logger.NewTest(t), ) if tc.wantErr { diff --git a/cli/internal/cloudcmd/BUILD.bazel b/cli/internal/cloudcmd/BUILD.bazel index c1316fa54..bbee7c8bf 100644 --- a/cli/internal/cloudcmd/BUILD.bazel +++ b/cli/internal/cloudcmd/BUILD.bazel @@ -34,7 +34,6 @@ go_library( "@com_github_azure_azure_sdk_for_go//profiles/latest/attestation/attestation", "@com_github_azure_azure_sdk_for_go_sdk_azcore//policy", "@com_github_azure_azure_sdk_for_go_sdk_azidentity//:azidentity", - "@com_github_hashicorp_terraform_json//:terraform-json", "@com_github_spf13_cobra//:cobra", ], ) @@ -59,7 +58,6 @@ go_test( "//internal/cloud/cloudprovider", "//internal/cloud/gcpshared", "//internal/config", - "@com_github_hashicorp_terraform_json//:terraform-json", "@com_github_stretchr_testify//assert", "@com_github_stretchr_testify//require", "@org_uber_go_goleak//:goleak", diff --git a/cli/internal/cloudcmd/clients.go b/cli/internal/cloudcmd/clients.go index 4d0a0c5e7..4e6b6cd09 100644 --- a/cli/internal/cloudcmd/clients.go +++ b/cli/internal/cloudcmd/clients.go @@ -13,7 +13,6 @@ import ( "github.com/edgelesssys/constellation/v2/cli/internal/terraform" "github.com/edgelesssys/constellation/v2/internal/attestation/variant" "github.com/edgelesssys/constellation/v2/internal/cloud/cloudprovider" - tfjson "github.com/hashicorp/terraform-json" ) // imageFetcher gets an image reference from the versionsapi. @@ -24,14 +23,23 @@ type imageFetcher interface { ) (string, error) } -type terraformClient interface { - PrepareWorkspace(path string, input terraform.Variables) error - ApplyIAMConfig(ctx context.Context, provider cloudprovider.Provider, logLevel terraform.LogLevel) (terraform.IAMOutput, error) - CreateCluster(ctx context.Context, logLevel terraform.LogLevel) (terraform.ApplyOutput, error) - Destroy(ctx context.Context, logLevel terraform.LogLevel) error +type tfCommonClient interface { CleanUpWorkspace() error + Destroy(ctx context.Context, logLevel terraform.LogLevel) error + PrepareWorkspace(path string, input terraform.Variables) error RemoveInstaller() - Show(ctx context.Context) (*tfjson.State, error) +} + +type tfResourceClient interface { + tfCommonClient + CreateCluster(ctx context.Context, provider cloudprovider.Provider, logLevel terraform.LogLevel) (terraform.ApplyOutput, error) + ShowCluster(ctx context.Context, provider cloudprovider.Provider) (terraform.ApplyOutput, error) +} + +type tfIAMClient interface { + tfCommonClient + ApplyIAMConfig(ctx context.Context, provider cloudprovider.Provider, logLevel terraform.LogLevel) (terraform.IAMOutput, error) + ShowIAM(ctx context.Context, provider cloudprovider.Provider) (terraform.IAMOutput, error) } type libvirtRunner interface { diff --git a/cli/internal/cloudcmd/clients_test.go b/cli/internal/cloudcmd/clients_test.go index 2bd70b846..d38ea3066 100644 --- a/cli/internal/cloudcmd/clients_test.go +++ b/cli/internal/cloudcmd/clients_test.go @@ -14,7 +14,6 @@ import ( "github.com/edgelesssys/constellation/v2/cli/internal/terraform" "github.com/edgelesssys/constellation/v2/internal/attestation/variant" "github.com/edgelesssys/constellation/v2/internal/cloud/cloudprovider" - tfjson "github.com/hashicorp/terraform-json" "go.uber.org/goleak" ) @@ -32,7 +31,7 @@ type stubTerraformClient struct { iamOutput terraform.IAMOutput uid string attestationURL string - tfjsonState *tfjson.State + applyOutput terraform.ApplyOutput cleanUpWorkspaceCalled bool removeInstallerCalled bool destroyCalled bool @@ -45,12 +44,14 @@ type stubTerraformClient struct { showErr error } -func (c *stubTerraformClient) CreateCluster(_ context.Context, _ terraform.LogLevel) (terraform.ApplyOutput, error) { +func (c *stubTerraformClient) CreateCluster(_ context.Context, _ cloudprovider.Provider, _ terraform.LogLevel) (terraform.ApplyOutput, error) { return terraform.ApplyOutput{ - IP: c.ip, - Secret: c.initSecret, - UID: c.uid, - AttestationURL: c.attestationURL, + IP: c.ip, + Secret: c.initSecret, + UID: c.uid, + Azure: &terraform.AzureApplyOutput{ + AttestationURL: c.attestationURL, + }, }, c.createClusterErr } @@ -76,9 +77,14 @@ func (c *stubTerraformClient) RemoveInstaller() { c.removeInstallerCalled = true } -func (c *stubTerraformClient) Show(_ context.Context) (*tfjson.State, error) { +func (c *stubTerraformClient) ShowCluster(_ context.Context, _ cloudprovider.Provider) (terraform.ApplyOutput, error) { c.showCalled = true - return c.tfjsonState, c.showErr + return c.applyOutput, c.showErr +} + +func (c *stubTerraformClient) ShowIAM(_ context.Context, _ cloudprovider.Provider) (terraform.IAMOutput, error) { + c.showCalled = true + return c.iamOutput, c.showErr } type stubLibvirtRunner struct { diff --git a/cli/internal/cloudcmd/create.go b/cli/internal/cloudcmd/create.go index b08e9dbf7..942511917 100644 --- a/cli/internal/cloudcmd/create.go +++ b/cli/internal/cloudcmd/create.go @@ -31,7 +31,7 @@ import ( type Creator struct { out io.Writer image imageFetcher - newTerraformClient func(ctx context.Context) (terraformClient, error) + newTerraformClient func(ctx context.Context) (tfResourceClient, error) newLibvirtRunner func() libvirtRunner newRawDownloader func() rawDownloader policyPatcher policyPatcher @@ -42,7 +42,7 @@ func NewCreator(out io.Writer) *Creator { return &Creator{ out: out, image: imagefetcher.New(), - newTerraformClient: func(ctx context.Context) (terraformClient, error) { + newTerraformClient: func(ctx context.Context) (tfResourceClient, error) { return terraform.New(ctx, constants.TerraformWorkingDir) }, newLibvirtRunner: func() libvirtRunner { @@ -115,18 +115,20 @@ func (c *Creator) Create(ctx context.Context, opts CreateOptions) (clusterid.Fil if err != nil { return clusterid.File{}, fmt.Errorf("creating cluster: %w", err) } - - return clusterid.File{ + res := clusterid.File{ CloudProvider: opts.Provider, IP: tfOutput.IP, APIServerCertSANs: tfOutput.APIServerCertSANs, InitSecret: []byte(tfOutput.Secret), UID: tfOutput.UID, - AttestationURL: tfOutput.AttestationURL, - }, nil + } + if tfOutput.Azure != nil { + res.AttestationURL = tfOutput.Azure.AttestationURL + } + return res, nil } -func (c *Creator) createAWS(ctx context.Context, cl terraformClient, opts CreateOptions) (tfOutput terraform.ApplyOutput, retErr error) { +func (c *Creator) createAWS(ctx context.Context, cl tfResourceClient, opts CreateOptions) (tfOutput terraform.ApplyOutput, retErr error) { vars := awsTerraformVars(opts.Config, opts.image, &opts.ControlPlaneCount, &opts.WorkerCount) tfOutput, err := runTerraformCreate(ctx, cl, cloudprovider.AWS, vars, c.out, opts.TFLogLevel) @@ -137,7 +139,7 @@ func (c *Creator) createAWS(ctx context.Context, cl terraformClient, opts Create return tfOutput, nil } -func (c *Creator) createGCP(ctx context.Context, cl terraformClient, opts CreateOptions) (tfOutput terraform.ApplyOutput, retErr error) { +func (c *Creator) createGCP(ctx context.Context, cl tfResourceClient, opts CreateOptions) (tfOutput terraform.ApplyOutput, retErr error) { vars := gcpTerraformVars(opts.Config, opts.image, &opts.ControlPlaneCount, &opts.WorkerCount) tfOutput, err := runTerraformCreate(ctx, cl, cloudprovider.GCP, vars, c.out, opts.TFLogLevel) @@ -148,7 +150,7 @@ func (c *Creator) createGCP(ctx context.Context, cl terraformClient, opts Create return tfOutput, nil } -func (c *Creator) createAzure(ctx context.Context, cl terraformClient, opts CreateOptions) (tfOutput terraform.ApplyOutput, retErr error) { +func (c *Creator) createAzure(ctx context.Context, cl tfResourceClient, opts CreateOptions) (tfOutput terraform.ApplyOutput, retErr error) { vars := azureTerraformVars(opts.Config, opts.image, &opts.ControlPlaneCount, &opts.WorkerCount) tfOutput, err := runTerraformCreate(ctx, cl, cloudprovider.Azure, vars, c.out, opts.TFLogLevel) @@ -158,7 +160,10 @@ func (c *Creator) createAzure(ctx context.Context, cl terraformClient, opts Crea if vars.GetCreateMAA() { // Patch the attestation policy to allow the cluster to boot while having secure boot disabled. - if err := c.policyPatcher.Patch(ctx, tfOutput.AttestationURL); err != nil { + if tfOutput.Azure == nil { + return terraform.ApplyOutput{}, errors.New("no Terraform Azure output found") + } + if err := c.policyPatcher.Patch(ctx, tfOutput.Azure.AttestationURL); err != nil { return terraform.ApplyOutput{}, err } } @@ -197,7 +202,7 @@ func normalizeAzureURIs(vars *terraform.AzureClusterVariables) *terraform.AzureC return vars } -func (c *Creator) createOpenStack(ctx context.Context, cl terraformClient, opts CreateOptions) (tfOutput terraform.ApplyOutput, retErr error) { +func (c *Creator) createOpenStack(ctx context.Context, cl tfResourceClient, opts CreateOptions) (tfOutput terraform.ApplyOutput, retErr error) { // TODO(malt3): Remove this once OpenStack is supported. if os.Getenv("CONSTELLATION_OPENSTACK_DEV") != "1" { return terraform.ApplyOutput{}, errors.New("OpenStack isn't supported yet") @@ -221,13 +226,13 @@ func (c *Creator) createOpenStack(ctx context.Context, cl terraformClient, opts return tfOutput, nil } -func runTerraformCreate(ctx context.Context, cl terraformClient, provider cloudprovider.Provider, vars terraform.Variables, outWriter io.Writer, loglevel terraform.LogLevel) (output terraform.ApplyOutput, retErr error) { +func runTerraformCreate(ctx context.Context, cl tfResourceClient, provider cloudprovider.Provider, vars terraform.Variables, outWriter io.Writer, loglevel terraform.LogLevel) (output terraform.ApplyOutput, retErr error) { if err := cl.PrepareWorkspace(path.Join("terraform", strings.ToLower(provider.String())), vars); err != nil { return terraform.ApplyOutput{}, err } defer rollbackOnError(outWriter, &retErr, &rollbackerTerraform{client: cl}, loglevel) - tfOutput, err := cl.CreateCluster(ctx, loglevel) + tfOutput, err := cl.CreateCluster(ctx, provider, loglevel) if err != nil { return terraform.ApplyOutput{}, err } @@ -240,7 +245,7 @@ type qemuCreateOptions struct { CreateOptions } -func (c *Creator) createQEMU(ctx context.Context, cl terraformClient, lv libvirtRunner, opts qemuCreateOptions) (tfOutput terraform.ApplyOutput, retErr error) { +func (c *Creator) createQEMU(ctx context.Context, cl tfResourceClient, lv libvirtRunner, opts qemuCreateOptions) (tfOutput terraform.ApplyOutput, retErr error) { qemuRollbacker := &rollbackerQEMU{client: cl, libvirt: lv, createdWorkspace: false} defer rollbackOnError(c.out, &retErr, qemuRollbacker, opts.TFLogLevel) @@ -300,7 +305,7 @@ func (c *Creator) createQEMU(ctx context.Context, cl terraformClient, lv libvirt // Allow rollback of QEMU Terraform workspace from this point on qemuRollbacker.createdWorkspace = true - tfOutput, err = cl.CreateCluster(ctx, opts.TFLogLevel) + tfOutput, err = cl.CreateCluster(ctx, opts.Provider, opts.TFLogLevel) if err != nil { return terraform.ApplyOutput{}, fmt.Errorf("create cluster: %w", err) } diff --git a/cli/internal/cloudcmd/create_test.go b/cli/internal/cloudcmd/create_test.go index b8b5fb6f9..5b2434b60 100644 --- a/cli/internal/cloudcmd/create_test.go +++ b/cli/internal/cloudcmd/create_test.go @@ -28,7 +28,7 @@ func TestCreator(t *testing.T) { someErr := errors.New("failed") testCases := map[string]struct { - tfClient terraformClient + tfClient tfResourceClient newTfClientErr error libvirt *stubLibvirtRunner provider cloudprovider.Provider @@ -203,7 +203,7 @@ func TestCreator(t *testing.T) { image: &stubImageFetcher{ reference: "some-image", }, - newTerraformClient: func(ctx context.Context) (terraformClient, error) { + newTerraformClient: func(ctx context.Context) (tfResourceClient, error) { return tc.tfClient, tc.newTfClientErr }, newLibvirtRunner: func() libvirtRunner { diff --git a/cli/internal/cloudcmd/iam.go b/cli/internal/cloudcmd/iam.go index 8bf1337d4..c5c164ebf 100644 --- a/cli/internal/cloudcmd/iam.go +++ b/cli/internal/cloudcmd/iam.go @@ -24,7 +24,7 @@ import ( // IAMDestroyer destroys an IAM configuration. type IAMDestroyer struct { - client terraformClient + client tfIAMClient } // NewIAMDestroyer creates a new IAM Destroyer. @@ -38,35 +38,23 @@ func NewIAMDestroyer(ctx context.Context) (*IAMDestroyer, error) { // GetTfstateServiceAccountKey returns the sa_key output from the terraform state. func (d *IAMDestroyer) GetTfstateServiceAccountKey(ctx context.Context) (gcpshared.ServiceAccountKey, error) { - tfState, err := d.client.Show(ctx) + tfState, err := d.client.ShowIAM(ctx, cloudprovider.GCP) if err != nil { - return gcpshared.ServiceAccountKey{}, err + return gcpshared.ServiceAccountKey{}, fmt.Errorf("getting terraform state: %w", err) } + if saKeyString := tfState.GCP.SaKey; saKeyString != "" { + saKey, err := base64.StdEncoding.DecodeString(saKeyString) + if err != nil { + return gcpshared.ServiceAccountKey{}, err + } + var tfSaKey gcpshared.ServiceAccountKey + if err := json.Unmarshal(saKey, &tfSaKey); err != nil { + return gcpshared.ServiceAccountKey{}, err + } - if tfState.Values == nil { - return gcpshared.ServiceAccountKey{}, errors.New("no Values field in terraform state") + return tfSaKey, nil } - - saKeyJSON := tfState.Values.Outputs["sa_key"] - if saKeyJSON == nil { - return gcpshared.ServiceAccountKey{}, errors.New("no sa_key in outputs of the terraform state") - } - - saKeyString, ok := saKeyJSON.Value.(string) - if !ok { - return gcpshared.ServiceAccountKey{}, errors.New("sa_key field in terraform state is not a string") - } - saKey, err := base64.StdEncoding.DecodeString(saKeyString) - if err != nil { - return gcpshared.ServiceAccountKey{}, err - } - - var tfSaKey gcpshared.ServiceAccountKey - if err := json.Unmarshal(saKey, &tfSaKey); err != nil { - return gcpshared.ServiceAccountKey{}, err - } - - return tfSaKey, nil + return gcpshared.ServiceAccountKey{}, errors.New("no saKey in terraform state") } // DestroyIAMConfiguration destroys the previously created IAM configuration and deletes the local IAM terraform files. @@ -80,7 +68,7 @@ func (d *IAMDestroyer) DestroyIAMConfiguration(ctx context.Context, logLevel ter // IAMCreator creates the IAM configuration on the cloud provider. type IAMCreator struct { out io.Writer - newTerraformClient func(ctx context.Context) (terraformClient, error) + newTerraformClient func(ctx context.Context) (tfIAMClient, error) } // IAMConfigOptions holds the necessary values for IAM configuration. @@ -116,7 +104,7 @@ type AWSIAMConfig struct { func NewIAMCreator(out io.Writer) *IAMCreator { return &IAMCreator{ out: out, - newTerraformClient: func(ctx context.Context) (terraformClient, error) { + newTerraformClient: func(ctx context.Context) (tfIAMClient, error) { return terraform.New(ctx, constants.TerraformIAMWorkingDir) }, } @@ -152,7 +140,7 @@ func (c *IAMCreator) Create(ctx context.Context, provider cloudprovider.Provider } // createGCP creates the IAM configuration on GCP. -func (c *IAMCreator) createGCP(ctx context.Context, cl terraformClient, opts *IAMConfigOptions) (retFile iamid.File, retErr error) { +func (c *IAMCreator) createGCP(ctx context.Context, cl tfIAMClient, opts *IAMConfigOptions) (retFile iamid.File, retErr error) { defer rollbackOnError(c.out, &retErr, &rollbackerTerraform{client: cl}, opts.TFLogLevel) vars := terraform.GCPIAMVariables{ @@ -180,7 +168,7 @@ func (c *IAMCreator) createGCP(ctx context.Context, cl terraformClient, opts *IA } // createAzure creates the IAM configuration on Azure. -func (c *IAMCreator) createAzure(ctx context.Context, cl terraformClient, opts *IAMConfigOptions) (retFile iamid.File, retErr error) { +func (c *IAMCreator) createAzure(ctx context.Context, cl tfIAMClient, opts *IAMConfigOptions) (retFile iamid.File, retErr error) { defer rollbackOnError(c.out, &retErr, &rollbackerTerraform{client: cl}, opts.TFLogLevel) vars := terraform.AzureIAMVariables{ @@ -209,7 +197,7 @@ func (c *IAMCreator) createAzure(ctx context.Context, cl terraformClient, opts * } // createAWS creates the IAM configuration on AWS. -func (c *IAMCreator) createAWS(ctx context.Context, cl terraformClient, opts *IAMConfigOptions) (retFile iamid.File, retErr error) { +func (c *IAMCreator) createAWS(ctx context.Context, cl tfIAMClient, opts *IAMConfigOptions) (retFile iamid.File, retErr error) { defer rollbackOnError(c.out, &retErr, &rollbackerTerraform{client: cl}, opts.TFLogLevel) vars := terraform.AWSIAMVariables{ diff --git a/cli/internal/cloudcmd/iam_test.go b/cli/internal/cloudcmd/iam_test.go index 1ff6ea9a5..498837010 100644 --- a/cli/internal/cloudcmd/iam_test.go +++ b/cli/internal/cloudcmd/iam_test.go @@ -17,7 +17,6 @@ import ( "github.com/edgelesssys/constellation/v2/cli/internal/terraform" "github.com/edgelesssys/constellation/v2/internal/cloud/cloudprovider" "github.com/edgelesssys/constellation/v2/internal/cloud/gcpshared" - tfjson "github.com/hashicorp/terraform-json" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -83,7 +82,7 @@ func TestIAMCreator(t *testing.T) { } testCases := map[string]struct { - tfClient terraformClient + tfClient tfIAMClient newTfClientErr error config *IAMConfigOptions provider cloudprovider.Provider @@ -125,7 +124,7 @@ func TestIAMCreator(t *testing.T) { creator := &IAMCreator{ out: &bytes.Buffer{}, - newTerraformClient: func(ctx context.Context) (terraformClient, error) { + newTerraformClient: func(ctx context.Context) (tfIAMClient, error) { return tc.tfClient, tc.newTfClientErr }, } @@ -225,13 +224,9 @@ func TestGetTfstateServiceAccountKey(t *testing.T) { }{ "valid": { cl: &stubTerraformClient{ - tfjsonState: &tfjson.State{ - Values: &tfjson.StateValues{ - Outputs: map[string]*tfjson.StateOutput{ - "sa_key": { - Value: gcpFileB64, - }, - }, + iamOutput: terraform.IAMOutput{ + GCP: terraform.GCPIAMOutput{ + SaKey: gcpFileB64, }, }, }, @@ -247,31 +242,16 @@ func TestGetTfstateServiceAccountKey(t *testing.T) { }, "nil tfstate values": { cl: &stubTerraformClient{ - tfjsonState: &tfjson.State{ - Values: nil, - }, - }, - wantErr: true, - wantShowCalled: true, - }, - "no key": { - cl: &stubTerraformClient{ - tfjsonState: &tfjson.State{ - Values: &tfjson.StateValues{}, - }, + iamOutput: terraform.IAMOutput{}, }, wantErr: true, wantShowCalled: true, }, "invalid base64": { cl: &stubTerraformClient{ - tfjsonState: &tfjson.State{ - Values: &tfjson.StateValues{ - Outputs: map[string]*tfjson.StateOutput{ - "sa_key": { - Value: "iamnotvalid", - }, - }, + iamOutput: terraform.IAMOutput{ + GCP: terraform.GCPIAMOutput{ + SaKey: "iamnotvalid", }, }, }, @@ -280,28 +260,9 @@ func TestGetTfstateServiceAccountKey(t *testing.T) { }, "valid base64 invalid json": { cl: &stubTerraformClient{ - tfjsonState: &tfjson.State{ - Values: &tfjson.StateValues{ - Outputs: map[string]*tfjson.StateOutput{ - "sa_key": { - Value: base64.StdEncoding.EncodeToString([]byte("asdf")), - }, - }, - }, - }, - }, - wantErr: true, - wantShowCalled: true, - }, - "not string": { - cl: &stubTerraformClient{ - tfjsonState: &tfjson.State{ - Values: &tfjson.StateValues{ - Outputs: map[string]*tfjson.StateOutput{ - "sa_key": { - Value: 1, - }, - }, + iamOutput: terraform.IAMOutput{ + GCP: terraform.GCPIAMOutput{ + SaKey: base64.StdEncoding.EncodeToString([]byte("asdf")), }, }, }, diff --git a/cli/internal/cloudcmd/rollback.go b/cli/internal/cloudcmd/rollback.go index 4b82b97ea..42e30071c 100644 --- a/cli/internal/cloudcmd/rollback.go +++ b/cli/internal/cloudcmd/rollback.go @@ -36,7 +36,7 @@ func rollbackOnError(w io.Writer, onErr *error, roll rollbacker, logLevel terraf } type rollbackerTerraform struct { - client terraformClient + client tfCommonClient } func (r *rollbackerTerraform) rollback(ctx context.Context, logLevel terraform.LogLevel) error { @@ -47,7 +47,7 @@ func (r *rollbackerTerraform) rollback(ctx context.Context, logLevel terraform.L } type rollbackerQEMU struct { - client terraformClient + client tfResourceClient libvirt libvirtRunner createdWorkspace bool } diff --git a/cli/internal/cloudcmd/terminate.go b/cli/internal/cloudcmd/terminate.go index f9fb3e3af..2d2efdd5e 100644 --- a/cli/internal/cloudcmd/terminate.go +++ b/cli/internal/cloudcmd/terminate.go @@ -16,14 +16,14 @@ import ( // Terminator deletes cloud provider resources. type Terminator struct { - newTerraformClient func(ctx context.Context) (terraformClient, error) + newTerraformClient func(ctx context.Context) (tfResourceClient, error) newLibvirtRunner func() libvirtRunner } // NewTerminator create a new cloud terminator. func NewTerminator() *Terminator { return &Terminator{ - newTerraformClient: func(ctx context.Context) (terraformClient, error) { + newTerraformClient: func(ctx context.Context) (tfResourceClient, error) { return terraform.New(ctx, constants.TerraformWorkingDir) }, newLibvirtRunner: func() libvirtRunner { @@ -49,7 +49,7 @@ func (t *Terminator) Terminate(ctx context.Context, logLevel terraform.LogLevel) return t.terminateTerraform(ctx, cl, logLevel) } -func (t *Terminator) terminateTerraform(ctx context.Context, cl terraformClient, logLevel terraform.LogLevel) error { +func (t *Terminator) terminateTerraform(ctx context.Context, cl tfResourceClient, logLevel terraform.LogLevel) error { if err := cl.Destroy(ctx, logLevel); err != nil { return err } diff --git a/cli/internal/cloudcmd/terminate_test.go b/cli/internal/cloudcmd/terminate_test.go index 4548d614d..b12795bf6 100644 --- a/cli/internal/cloudcmd/terminate_test.go +++ b/cli/internal/cloudcmd/terminate_test.go @@ -19,7 +19,7 @@ func TestTerminator(t *testing.T) { someErr := errors.New("failed") testCases := map[string]struct { - tfClient terraformClient + tfClient tfResourceClient newTfClientErr error libvirt *stubLibvirtRunner wantErr bool @@ -55,7 +55,7 @@ func TestTerminator(t *testing.T) { assert := assert.New(t) terminator := &Terminator{ - newTerraformClient: func(ctx context.Context) (terraformClient, error) { + newTerraformClient: func(ctx context.Context) (tfResourceClient, error) { return tc.tfClient, tc.newTfClientErr }, newLibvirtRunner: func() libvirtRunner { diff --git a/cli/internal/cmd/BUILD.bazel b/cli/internal/cmd/BUILD.bazel index 47157035e..953c19782 100644 --- a/cli/internal/cmd/BUILD.bazel +++ b/cli/internal/cmd/BUILD.bazel @@ -153,6 +153,7 @@ go_test( "//internal/config", "//internal/constants", "//internal/crypto/testvector", + "//internal/deploy/helm", "//internal/file", "//internal/grpc/atlscredentials", "//internal/grpc/dialer", diff --git a/cli/internal/cmd/init.go b/cli/internal/cmd/init.go index b5cf3af63..2f1386521 100644 --- a/cli/internal/cmd/init.go +++ b/cli/internal/cmd/init.go @@ -7,8 +7,10 @@ SPDX-License-Identifier: AGPL-3.0-only package cmd import ( + "bytes" "context" "encoding/hex" + "encoding/json" "errors" "fmt" "io" @@ -72,11 +74,12 @@ func NewInitCmd() *cobra.Command { } type initCmd struct { - log debugLog - merger configMerger - spinner spinnerInterf - masterSecret uri.MasterSecret - fh *file.Handler + log debugLog + merger configMerger + spinner spinnerInterf + masterSecret uri.MasterSecret + fh *file.Handler + helmInstaller helm.SuiteInstaller } // runInitialize runs the initialize command. @@ -100,7 +103,11 @@ func runInitialize(cmd *cobra.Command, _ []string) error { ctx, cancel := context.WithTimeout(cmd.Context(), time.Hour) defer cancel() cmd.SetContext(ctx) - i := &initCmd{log: log, spinner: spinner, merger: &kubeconfigMerger{log: log}, fh: &fileHandler} + helmInstaller, err := helm.NewInstallationClient(log) + if err != nil { + return fmt.Errorf("creating Helm installer: %w", err) + } + i := &initCmd{log: log, spinner: spinner, merger: &kubeconfigMerger{log: log}, fh: &fileHandler, helmInstaller: helmInstaller} fetcher := attestationconfigapi.NewFetcher() return i.initialize(cmd, newDialer, fileHandler, license.NewClient(), fetcher) } @@ -180,7 +187,14 @@ func (i *initCmd) initialize(cmd *cobra.Command, newDialer func(validator atls.V helmLoader := helm.NewLoader(provider, k8sVersion, clusterName) i.log.Debugf("Created new Helm loader") - helmDeployments, err := helmLoader.Load(conf, flags.conformance, flags.helmWaitMode, masterSecret.Key, masterSecret.Salt) + releases, err := helmLoader.LoadReleases(conf, flags.conformance, flags.helmWaitMode, masterSecret.Key, masterSecret.Salt) + if err != nil { + return fmt.Errorf("loading Helm charts: %w", err) + } + helmDeployments, err := json.Marshal(releases) + if err != nil { + return err + } i.log.Debugf("Loaded Helm deployments") if err != nil { return fmt.Errorf("loading Helm charts: %w", err) @@ -214,10 +228,20 @@ func (i *initCmd) initialize(cmd *cobra.Command, newDialer func(validator atls.V return err } i.log.Debugf("Initialization request succeeded") + i.log.Debugf("Writing Constellation ID file") idFile.CloudProvider = provider - return i.writeOutput(idFile, resp, flags.mergeConfigs, cmd.OutOrStdout(), fileHandler) + bufferedOutput := &bytes.Buffer{} + err = i.writeOutput(idFile, resp, flags.mergeConfigs, bufferedOutput, fileHandler) + if err != nil { + return err + } + if err := i.helmInstaller.Install(cmd.Context(), provider, masterSecret, idFile, serviceAccURI, releases); err != nil { + return fmt.Errorf("installing Helm charts: %w", err) + } + cmd.Println(bufferedOutput.String()) + return nil } func (i *initCmd) initCall(ctx context.Context, dialer grpcDialer, ip string, req *initproto.InitRequest) (*initproto.InitSuccessResponse, error) { diff --git a/cli/internal/cmd/init_test.go b/cli/internal/cmd/init_test.go index ab3869282..bafe7e4a7 100644 --- a/cli/internal/cmd/init_test.go +++ b/cli/internal/cmd/init_test.go @@ -28,6 +28,7 @@ import ( "github.com/edgelesssys/constellation/v2/internal/cloud/gcpshared" "github.com/edgelesssys/constellation/v2/internal/config" "github.com/edgelesssys/constellation/v2/internal/constants" + helminstaller "github.com/edgelesssys/constellation/v2/internal/deploy/helm" "github.com/edgelesssys/constellation/v2/internal/file" "github.com/edgelesssys/constellation/v2/internal/grpc/atlscredentials" "github.com/edgelesssys/constellation/v2/internal/grpc/dialer" @@ -174,7 +175,7 @@ func TestInitialize(t *testing.T) { ctx, cancel := context.WithTimeout(ctx, 4*time.Second) defer cancel() cmd.SetContext(ctx) - i := &initCmd{log: logger.NewTest(t), spinner: &nopSpinner{}} + i := &initCmd{log: logger.NewTest(t), spinner: &nopSpinner{}, helmInstaller: &stubHelmInstaller{}} err := i.initialize(cmd, newDialer, fileHandler, &stubLicenseClient{}, stubAttestationFetcher{}) if tc.wantErr { @@ -666,3 +667,12 @@ func (c stubInitClient) Recv() (*initproto.InitResponse, error) { return res, err } + +type stubHelmInstaller struct{} + +func (i *stubHelmInstaller) Install(_ context.Context, _ cloudprovider.Provider, _ uri.MasterSecret, + _ clusterid.File, + _ string, _ *helminstaller.Releases, +) error { + return nil +} diff --git a/cli/internal/cmd/tfmigrationclient.go b/cli/internal/cmd/tfmigrationclient.go index 8de5f8939..425552490 100644 --- a/cli/internal/cmd/tfmigrationclient.go +++ b/cli/internal/cmd/tfmigrationclient.go @@ -38,7 +38,7 @@ func (u *tfMigrationClient) planMigration(cmd *cobra.Command, file file.Handler, func (u *tfMigrationClient) applyMigration(cmd *cobra.Command, file file.Handler, migrateCmd upgrade.TfMigrationCmd, yesFlag bool) error { hasDiff, err := u.planMigration(cmd, file, migrateCmd) if err != nil { - return fmt.Errorf("planning terraform migrations: %w", err) + return err } if hasDiff { // If there are any Terraform migrations to apply, ask for confirmation diff --git a/cli/internal/helm/BUILD.bazel b/cli/internal/helm/BUILD.bazel index 611402932..3d4d74ce8 100644 --- a/cli/internal/helm/BUILD.bazel +++ b/cli/internal/helm/BUILD.bazel @@ -7,8 +7,10 @@ go_library( "backup.go", "client.go", "helm.go", + "helminstaller.go", "loader.go", "serviceversion.go", + "setup.go", "values.go", ], embedsrcs = [ @@ -417,12 +419,17 @@ go_library( deps = [ "//cli/internal/clusterid", "//cli/internal/helm/imageversion", + "//cli/internal/terraform", + "//internal/cloud/azureshared", "//internal/cloud/cloudprovider", + "//internal/cloud/gcpshared", + "//internal/cloud/openstack", "//internal/compatibility", "//internal/config", "//internal/constants", "//internal/deploy/helm", "//internal/file", + "//internal/kms/uri", "//internal/semver", "//internal/versions", "@com_github_pkg_errors//:errors", diff --git a/cli/internal/helm/helminstaller.go b/cli/internal/helm/helminstaller.go new file mode 100644 index 000000000..f8c7489d9 --- /dev/null +++ b/cli/internal/helm/helminstaller.go @@ -0,0 +1,100 @@ +/* +Copyright (c) Edgeless Systems GmbH +SPDX-License-Identifier: AGPL-3.0-only +*/ + +package helm + +import ( + "context" + "fmt" + + "github.com/edgelesssys/constellation/v2/cli/internal/clusterid" + "github.com/edgelesssys/constellation/v2/internal/cloud/cloudprovider" + "github.com/edgelesssys/constellation/v2/internal/cloud/openstack" + "github.com/edgelesssys/constellation/v2/internal/constants" + helminstaller "github.com/edgelesssys/constellation/v2/internal/deploy/helm" + "github.com/edgelesssys/constellation/v2/internal/kms/uri" +) + +// SuiteInstaller installs all Helm charts required for a constellation cluster. +type SuiteInstaller interface { + Install(ctx context.Context, provider cloudprovider.Provider, masterSecret uri.MasterSecret, + idFile clusterid.File, + serviceAccURI string, releases *helminstaller.Releases, + ) error +} + +type helmInstallationClient struct { + log debugLog + installer helmInstaller +} + +// NewInstallationClient creates a new Helm installation client to install all Helm charts required for a constellation cluster. +func NewInstallationClient(log debugLog) (SuiteInstaller, error) { + installer, err := helminstaller.NewInstaller(constants.AdminConfFilename, log) + if err != nil { + return nil, fmt.Errorf("creating Helm installer: %w", err) + } + return &helmInstallationClient{log: log, installer: installer}, nil +} + +func (h helmInstallationClient) Install(ctx context.Context, provider cloudprovider.Provider, masterSecret uri.MasterSecret, + idFile clusterid.File, + serviceAccURI string, releases *helminstaller.Releases, +) error { + serviceVals, err := setupMicroserviceVals(ctx, provider, masterSecret.Salt, idFile.UID, serviceAccURI) + if err != nil { + return fmt.Errorf("setting up microservice values: %w", err) + } + if err := h.installer.InstallChartWithValues(ctx, releases.ConstellationServices, serviceVals); err != nil { + return fmt.Errorf("installing microservices: %w", err) + } + + h.log.Debugf("Installing cert-manager") + if err := h.installer.InstallChart(ctx, releases.CertManager); err != nil { + return fmt.Errorf("installing cert-manager: %w", err) + } + + if releases.CSI != nil { + var csiVals map[string]any + if provider == cloudprovider.OpenStack { + creds, err := openstack.AccountKeyFromURI(serviceAccURI) + if err != nil { + return err + } + cinderIni := creds.CloudINI().CinderCSIConfiguration() + csiVals = map[string]any{ + "cinder-config": map[string]any{ + "secretData": cinderIni, + }, + } + } + + h.log.Debugf("Installing CSI deployments") + if err := h.installer.InstallChartWithValues(ctx, *releases.CSI, csiVals); err != nil { + return fmt.Errorf("installing CSI snapshot CRDs: %w", err) + } + } + + if releases.AWSLoadBalancerController != nil { + h.log.Debugf("Installing AWS Load Balancer Controller") + if err := h.installer.InstallChart(ctx, *releases.AWSLoadBalancerController); err != nil { + return fmt.Errorf("installing AWS Load Balancer Controller: %w", err) + } + } + + h.log.Debugf("Installing constellation operators") + operatorVals := setupOperatorVals(ctx, idFile.UID) + if err := h.installer.InstallChartWithValues(ctx, releases.ConstellationOperators, operatorVals); err != nil { + return fmt.Errorf("installing constellation operators: %w", err) + } + + // TODO(elchead): AB#3301 do cilium after version upgrade + return nil +} + +type helmInstaller interface { + InstallChart(context.Context, helminstaller.Release) error + InstallChartWithValues(ctx context.Context, release helminstaller.Release, extraValues map[string]any) error +} diff --git a/cli/internal/helm/loader.go b/cli/internal/helm/loader.go index d8593e241..622c1de59 100644 --- a/cli/internal/helm/loader.go +++ b/cli/internal/helm/loader.go @@ -110,6 +110,19 @@ func NewLoader(csp cloudprovider.Provider, k8sVersion versions.ValidK8sVersion, // Load the embedded helm charts. func (i *ChartLoader) Load(config *config.Config, conformanceMode bool, helmWaitMode helm.WaitMode, masterSecret, salt []byte) ([]byte, error) { + releases, err := i.LoadReleases(config, conformanceMode, helmWaitMode, masterSecret, salt) + if err != nil { + return nil, fmt.Errorf("loading releases: %w", err) + } + rel, err := json.Marshal(releases) + if err != nil { + return nil, err + } + return rel, nil +} + +// LoadReleases loads the embedded helm charts and returns them as a HelmReleases object. +func (i *ChartLoader) LoadReleases(config *config.Config, conformanceMode bool, helmWaitMode helm.WaitMode, masterSecret, salt []byte) (*helm.Releases, error) { ciliumRelease, err := i.loadRelease(ciliumInfo, helmWaitMode) if err != nil { return nil, fmt.Errorf("loading cilium: %w", err) @@ -150,12 +163,7 @@ func (i *ChartLoader) Load(config *config.Config, conformanceMode bool, helmWait } releases.CSI = &csi } - - rel, err := json.Marshal(releases) - if err != nil { - return nil, err - } - return rel, nil + return &releases, nil } // loadRelease loads the embedded chart and values depending on the given info argument. diff --git a/cli/internal/helm/setup.go b/cli/internal/helm/setup.go new file mode 100644 index 000000000..209230a7e --- /dev/null +++ b/cli/internal/helm/setup.go @@ -0,0 +1,134 @@ +/* +Copyright (c) Edgeless Systems GmbH + +SPDX-License-Identifier: AGPL-3.0-only +*/ + +package helm + +import ( + "context" + "encoding/base64" + "encoding/json" + "fmt" + + "github.com/edgelesssys/constellation/v2/cli/internal/terraform" + "github.com/edgelesssys/constellation/v2/internal/cloud/azureshared" + "github.com/edgelesssys/constellation/v2/internal/cloud/cloudprovider" + "github.com/edgelesssys/constellation/v2/internal/cloud/gcpshared" + "github.com/edgelesssys/constellation/v2/internal/constants" +) + +// setupMicroserviceVals returns the values for the microservice chart. +func setupMicroserviceVals(ctx context.Context, provider cloudprovider.Provider, measurementSalt []byte, uid, serviceAccURI string) (map[string]any, error) { + tfClient, err := terraform.New(ctx, constants.TerraformWorkingDir) + if err != nil { + return nil, fmt.Errorf("creating Terraform client: %w", err) + } + output, err := tfClient.ShowCluster(ctx, provider) + if err != nil { + return nil, fmt.Errorf("getting Terraform output: %w", err) + } + extraVals := map[string]any{ + "join-service": map[string]any{ + "measurementSalt": base64.StdEncoding.EncodeToString(measurementSalt), + }, + "verification-service": map[string]any{ + "loadBalancerIP": output.IP, + }, + "konnectivity": map[string]any{ + "loadBalancerIP": output.IP, + }, + } + switch provider { + case cloudprovider.GCP: + serviceAccountKey, err := gcpshared.ServiceAccountKeyFromURI(serviceAccURI) + if err != nil { + return nil, fmt.Errorf("getting service account key: %w", err) + } + rawKey, err := json.Marshal(serviceAccountKey) + if err != nil { + return nil, fmt.Errorf("marshaling service account key: %w", err) + } + if output.GCP == nil { + return nil, fmt.Errorf("no GCP output from Terraform") + } + extraVals["ccm"] = map[string]any{ + "GCP": map[string]any{ + "projectID": output.GCP.ProjectID, + "uid": uid, + "secretData": string(rawKey), + "subnetworkPodCIDR": output.GCP.IPCidrPod, + }, + } + case cloudprovider.Azure: + if output.Azure == nil { + return nil, fmt.Errorf("no Azure output from Terraform") + } + ccmConfig, err := getCCMConfig(*output.Azure, serviceAccURI) + if err != nil { + return nil, fmt.Errorf("getting Azure CCM config: %w", err) + } + extraVals["ccm"] = map[string]any{ + "Azure": map[string]any{ + "azureConfig": string(ccmConfig), + }, + } + } + + return extraVals, nil +} + +// setupOperatorVals returns the values for the constellation-operator chart. +func setupOperatorVals(_ context.Context, uid string) map[string]any { + return map[string]any{ + "constellation-operator": map[string]any{ + "constellationUID": uid, + }, + } +} + +type cloudConfig struct { + Cloud string `json:"cloud,omitempty"` + TenantID string `json:"tenantId,omitempty"` + SubscriptionID string `json:"subscriptionId,omitempty"` + ResourceGroup string `json:"resourceGroup,omitempty"` + Location string `json:"location,omitempty"` + SubnetName string `json:"subnetName,omitempty"` + SecurityGroupName string `json:"securityGroupName,omitempty"` + SecurityGroupResourceGroup string `json:"securityGroupResourceGroup,omitempty"` + LoadBalancerName string `json:"loadBalancerName,omitempty"` + LoadBalancerSku string `json:"loadBalancerSku,omitempty"` + VNetName string `json:"vnetName,omitempty"` + VNetResourceGroup string `json:"vnetResourceGroup,omitempty"` + CloudProviderBackoff bool `json:"cloudProviderBackoff,omitempty"` + UseInstanceMetadata bool `json:"useInstanceMetadata,omitempty"` + VMType string `json:"vmType,omitempty"` + UseManagedIdentityExtension bool `json:"useManagedIdentityExtension,omitempty"` + UserAssignedIdentityID string `json:"userAssignedIdentityID,omitempty"` +} + +// GetCCMConfig returns the configuration needed for the Kubernetes Cloud Controller Manager on Azure. +func getCCMConfig(tfOutput terraform.AzureApplyOutput, serviceAccURI string) ([]byte, error) { + creds, err := azureshared.ApplicationCredentialsFromURI(serviceAccURI) + if err != nil { + return nil, fmt.Errorf("getting service account key: %w", err) + } + useManagedIdentityExtension := creds.PreferredAuthMethod == azureshared.AuthMethodUserAssignedIdentity + config := cloudConfig{ + Cloud: "AzurePublicCloud", + TenantID: creds.TenantID, + SubscriptionID: tfOutput.SubscriptionID, + ResourceGroup: tfOutput.ResourceGroup, + LoadBalancerSku: "standard", + SecurityGroupName: tfOutput.NetworkSecurityGroupName, + LoadBalancerName: tfOutput.LoadBalancerName, + UseInstanceMetadata: true, + VMType: "vmss", + Location: creds.Location, + UseManagedIdentityExtension: useManagedIdentityExtension, + UserAssignedIdentityID: tfOutput.UserAssignedIdentity, + } + + return json.Marshal(config) +} diff --git a/cli/internal/terraform/terraform.go b/cli/internal/terraform/terraform.go index 3241d555b..c4dd52aff 100644 --- a/cli/internal/terraform/terraform.go +++ b/cli/internal/terraform/terraform.go @@ -78,172 +78,8 @@ func (c *Client) WithManualStateMigration(migration StateMigration) *Client { return c } -// Show reads the default state path and outputs the state. -func (c *Client) Show(ctx context.Context) (*tfjson.State, error) { - return c.tf.Show(ctx) -} - -// PrepareWorkspace prepares a Terraform workspace for a Constellation cluster. -func (c *Client) PrepareWorkspace(path string, vars Variables) error { - if err := prepareWorkspace(path, c.file, c.workingDir); err != nil { - return fmt.Errorf("prepare workspace: %w", err) - } - - return c.writeVars(vars) -} - -// PrepareUpgradeWorkspace prepares a Terraform workspace for a Constellation version upgrade. -// It copies the Terraform state from the old working dir and the embedded Terraform files into the new working dir. -func (c *Client) PrepareUpgradeWorkspace(path, oldWorkingDir, newWorkingDir, backupDir string, vars Variables) error { - if err := prepareUpgradeWorkspace(path, c.file, oldWorkingDir, newWorkingDir, backupDir); err != nil { - return fmt.Errorf("prepare upgrade workspace: %w", err) - } - - return c.writeVars(vars) -} - -// PrepareIAMUpgradeWorkspace prepares a Terraform workspace for a Constellation IAM upgrade. -func PrepareIAMUpgradeWorkspace(file file.Handler, path, oldWorkingDir, newWorkingDir, backupDir string) error { - if err := prepareUpgradeWorkspace(path, file, oldWorkingDir, newWorkingDir, backupDir); err != nil { - return fmt.Errorf("prepare upgrade workspace: %w", err) - } - // copy the vars file from the old working dir to the new working dir - if err := file.CopyFile(filepath.Join(oldWorkingDir, terraformVarsFile), filepath.Join(newWorkingDir, terraformVarsFile)); err != nil { - return fmt.Errorf("copying vars file: %w", err) - } - return nil -} - -// CreateCluster creates a Constellation cluster using Terraform. -func (c *Client) CreateCluster(ctx context.Context, logLevel LogLevel) (ApplyOutput, error) { - if err := c.setLogLevel(logLevel); err != nil { - return ApplyOutput{}, fmt.Errorf("set terraform log level %s: %w", logLevel.String(), err) - } - - if err := c.tf.Init(ctx); err != nil { - return ApplyOutput{}, fmt.Errorf("terraform init: %w", err) - } - - if err := c.applyManualStateMigrations(ctx); err != nil { - return ApplyOutput{}, fmt.Errorf("apply manual state migrations: %w", err) - } - - if err := c.tf.Apply(ctx); err != nil { - return ApplyOutput{}, fmt.Errorf("terraform apply: %w", err) - } - - tfState, err := c.tf.Show(ctx) - if err != nil { - return ApplyOutput{}, fmt.Errorf("terraform show: %w", err) - } - - ipOutput, ok := tfState.Values.Outputs["ip"] - if !ok { - return ApplyOutput{}, errors.New("no IP output found") - } - ip, ok := ipOutput.Value.(string) - if !ok { - return ApplyOutput{}, errors.New("invalid type in IP output: not a string") - } - - apiServerCertSANsOutput, ok := tfState.Values.Outputs["api_server_cert_sans"] - if !ok { - return ApplyOutput{}, errors.New("no api_server_cert_sans output found") - } - apiServerCertSANsUntyped, ok := apiServerCertSANsOutput.Value.([]any) - if !ok { - return ApplyOutput{}, fmt.Errorf("invalid type in api_server_cert_sans output: %s is not a list of elements", apiServerCertSANsOutput.Type.FriendlyName()) - } - apiServerCertSANs, err := toStringSlice(apiServerCertSANsUntyped) - if err != nil { - return ApplyOutput{}, fmt.Errorf("convert api_server_cert_sans output: %w", err) - } - - secretOutput, ok := tfState.Values.Outputs["initSecret"] - if !ok { - return ApplyOutput{}, errors.New("no initSecret output found") - } - secret, ok := secretOutput.Value.(string) - if !ok { - return ApplyOutput{}, errors.New("invalid type in initSecret output: not a string") - } - - uidOutput, ok := tfState.Values.Outputs["uid"] - if !ok { - return ApplyOutput{}, errors.New("no uid output found") - } - uid, ok := uidOutput.Value.(string) - if !ok { - return ApplyOutput{}, errors.New("invalid type in uid output: not a string") - } - - var attestationURL string - if attestationURLOutput, ok := tfState.Values.Outputs["attestationURL"]; ok { - if attestationURLString, ok := attestationURLOutput.Value.(string); ok { - attestationURL = attestationURLString - } - } - - return ApplyOutput{ - IP: ip, - APIServerCertSANs: apiServerCertSANs, - Secret: secret, - UID: uid, - AttestationURL: attestationURL, - }, nil -} - -// ApplyOutput contains the Terraform output values of a cluster creation -// or apply operation. -type ApplyOutput struct { - IP string - APIServerCertSANs []string - Secret string - UID string - // AttestationURL is the URL of the attestation provider. - // It is only set if the cluster is created on Azure. - AttestationURL string -} - -// IAMOutput contains the output information of the Terraform IAM operations. -type IAMOutput struct { - GCP GCPIAMOutput - Azure AzureIAMOutput - AWS AWSIAMOutput -} - -// GCPIAMOutput contains the output information of the Terraform IAM operation on GCP. -type GCPIAMOutput struct { - SaKey string -} - -// AzureIAMOutput contains the output information of the Terraform IAM operation on Microsoft Azure. -type AzureIAMOutput struct { - SubscriptionID string - TenantID string - UAMIID string -} - -// AWSIAMOutput contains the output information of the Terraform IAM operation on GCP. -type AWSIAMOutput struct { - ControlPlaneInstanceProfile string - WorkerNodeInstanceProfile string -} - -// ApplyIAMConfig creates an IAM configuration using Terraform. -func (c *Client) ApplyIAMConfig(ctx context.Context, provider cloudprovider.Provider, logLevel LogLevel) (IAMOutput, error) { - if err := c.setLogLevel(logLevel); err != nil { - return IAMOutput{}, fmt.Errorf("set terraform log level %s: %w", logLevel.String(), err) - } - - if err := c.tf.Init(ctx); err != nil { - return IAMOutput{}, err - } - - if err := c.tf.Apply(ctx); err != nil { - return IAMOutput{}, err - } - +// ShowIAM reads the state of Constellation IAM resources from Terraform. +func (c *Client) ShowIAM(ctx context.Context, provider cloudprovider.Provider) (IAMOutput, error) { tfState, err := c.tf.Show(ctx) if err != nil { return IAMOutput{}, err @@ -324,6 +160,277 @@ func (c *Client) ApplyIAMConfig(ctx context.Context, provider cloudprovider.Prov } } +// ShowCluster reads the state of Constellation cluster resources from Terraform. +func (c *Client) ShowCluster(ctx context.Context, provider cloudprovider.Provider) (ApplyOutput, error) { + tfState, err := c.tf.Show(ctx) + if err != nil { + return ApplyOutput{}, fmt.Errorf("terraform show: %w", err) + } + + ipOutput, ok := tfState.Values.Outputs["ip"] + if !ok { + return ApplyOutput{}, errors.New("no IP output found") + } + ip, ok := ipOutput.Value.(string) + if !ok { + return ApplyOutput{}, errors.New("invalid type in IP output: not a string") + } + + apiServerCertSANsOutput, ok := tfState.Values.Outputs["api_server_cert_sans"] + if !ok { + return ApplyOutput{}, errors.New("no api_server_cert_sans output found") + } + apiServerCertSANsUntyped, ok := apiServerCertSANsOutput.Value.([]any) + if !ok { + return ApplyOutput{}, fmt.Errorf("invalid type in api_server_cert_sans output: %s is not a list of elements", apiServerCertSANsOutput.Type.FriendlyName()) + } + apiServerCertSANs, err := toStringSlice(apiServerCertSANsUntyped) + if err != nil { + return ApplyOutput{}, fmt.Errorf("convert api_server_cert_sans output: %w", err) + } + + secretOutput, ok := tfState.Values.Outputs["initSecret"] + if !ok { + return ApplyOutput{}, errors.New("no initSecret output found") + } + secret, ok := secretOutput.Value.(string) + if !ok { + return ApplyOutput{}, errors.New("invalid type in initSecret output: not a string") + } + + uidOutput, ok := tfState.Values.Outputs["uid"] + if !ok { + return ApplyOutput{}, errors.New("no uid output found") + } + uid, ok := uidOutput.Value.(string) + if !ok { + return ApplyOutput{}, errors.New("invalid type in uid output: not a string") + } + + res := ApplyOutput{ + IP: ip, + APIServerCertSANs: apiServerCertSANs, + Secret: secret, + UID: uid, + } + // TODO add provider + switch provider { + case cloudprovider.GCP: + gcpProjectOutput, ok := tfState.Values.Outputs["project"] + if ok { + gcpProject, ok := gcpProjectOutput.Value.(string) + if !ok { + return ApplyOutput{}, errors.New("invalid type in project output: not a string") + } + cidrNodesOutput, ok := tfState.Values.Outputs["ip_cidr_nodes"] + if !ok { + return ApplyOutput{}, errors.New("no ip_cidr_nodes output found") + } + cidrNodes, ok := cidrNodesOutput.Value.(string) + if !ok { + return ApplyOutput{}, errors.New("invalid type in ip_cidr_nodes output: not a string") + } + cidrPodsOutput, ok := tfState.Values.Outputs["ip_cidr_pods"] + if !ok { + return ApplyOutput{}, errors.New("no ip_cidr_pods output found") + } + cidrPods, ok := cidrPodsOutput.Value.(string) + if !ok { + return ApplyOutput{}, errors.New("invalid type in ip_cidr_pods output: not a string") + } + res.GCP = &GCPApplyOutput{ + ProjectID: gcpProject, + IPCidrNode: cidrNodes, + IPCidrPod: cidrPods, + } + } + case cloudprovider.Azure: + var attestationURL string + if ok { + if attestationURLOutput, ok := tfState.Values.Outputs["attestationURL"]; ok { + if attestationURLString, ok := attestationURLOutput.Value.(string); ok { + attestationURL = attestationURLString + } + } + } + + azureUAMIOutput, ok := tfState.Values.Outputs["user_assigned_identity"] + if !ok { + return ApplyOutput{}, errors.New("no user_assigned_identity output found") + } + azureUAMI, ok := azureUAMIOutput.Value.(string) + if !ok { + return ApplyOutput{}, errors.New("invalid type in user_assigned_identity output: not a string") + } + + rgOutput, ok := tfState.Values.Outputs["resource_group"] + if !ok { + return ApplyOutput{}, errors.New("no resource_group output found") + } + rg, ok := rgOutput.Value.(string) + if !ok { + return ApplyOutput{}, errors.New("invalid type in resource_group output: not a string") + } + + subscriptionOutput, ok := tfState.Values.Outputs["subscription_id"] + if !ok { + return ApplyOutput{}, errors.New("no subscription_id output found") + } + subscriptionID, ok := subscriptionOutput.Value.(string) + if !ok { + return ApplyOutput{}, errors.New("invalid type in subscription_id output: not a string") + } + + networkSGNameOutput, ok := tfState.Values.Outputs["network_security_group_name"] + if !ok { + return ApplyOutput{}, errors.New("no network_security_group_name output found") + } + networkSGName, ok := networkSGNameOutput.Value.(string) + if !ok { + return ApplyOutput{}, errors.New("invalid type in network_security_group_name output: not a string") + } + loadBalancerNameOutput, ok := tfState.Values.Outputs["loadbalancer_name"] + if !ok { + return ApplyOutput{}, errors.New("no loadbalancer_name output found") + } + loadBalancerName, ok := loadBalancerNameOutput.Value.(string) + if !ok { + return ApplyOutput{}, errors.New("invalid type in loadbalancer_name output: not a string") + } + res.Azure = &AzureApplyOutput{ + ResourceGroup: rg, + SubscriptionID: subscriptionID, + UserAssignedIdentity: azureUAMI, + NetworkSecurityGroupName: networkSGName, + LoadBalancerName: loadBalancerName, + AttestationURL: attestationURL, + } + } + return res, nil +} + +// PrepareWorkspace prepares a Terraform workspace for a Constellation cluster. +func (c *Client) PrepareWorkspace(path string, vars Variables) error { + if err := prepareWorkspace(path, c.file, c.workingDir); err != nil { + return fmt.Errorf("prepare workspace: %w", err) + } + + return c.writeVars(vars) +} + +// PrepareUpgradeWorkspace prepares a Terraform workspace for a Constellation version upgrade. +// It copies the Terraform state from the old working dir and the embedded Terraform files into the new working dir. +func (c *Client) PrepareUpgradeWorkspace(path, oldWorkingDir, newWorkingDir, backupDir string, vars Variables) error { + if err := prepareUpgradeWorkspace(path, c.file, oldWorkingDir, newWorkingDir, backupDir); err != nil { + return fmt.Errorf("prepare upgrade workspace: %w", err) + } + + return c.writeVars(vars) +} + +// PrepareIAMUpgradeWorkspace prepares a Terraform workspace for a Constellation IAM upgrade. +func PrepareIAMUpgradeWorkspace(file file.Handler, path, oldWorkingDir, newWorkingDir, backupDir string) error { + if err := prepareUpgradeWorkspace(path, file, oldWorkingDir, newWorkingDir, backupDir); err != nil { + return fmt.Errorf("prepare upgrade workspace: %w", err) + } + // copy the vars file from the old working dir to the new working dir + if err := file.CopyFile(filepath.Join(oldWorkingDir, terraformVarsFile), filepath.Join(newWorkingDir, terraformVarsFile)); err != nil { + return fmt.Errorf("copying vars file: %w", err) + } + return nil +} + +// CreateCluster creates a Constellation cluster using Terraform. +func (c *Client) CreateCluster(ctx context.Context, provider cloudprovider.Provider, logLevel LogLevel) (ApplyOutput, error) { + if err := c.setLogLevel(logLevel); err != nil { + return ApplyOutput{}, fmt.Errorf("set terraform log level %s: %w", logLevel.String(), err) + } + + if err := c.tf.Init(ctx); err != nil { + return ApplyOutput{}, fmt.Errorf("terraform init: %w", err) + } + + if err := c.applyManualStateMigrations(ctx); err != nil { + return ApplyOutput{}, fmt.Errorf("apply manual state migrations: %w", err) + } + + if err := c.tf.Apply(ctx); err != nil { + return ApplyOutput{}, fmt.Errorf("terraform apply: %w", err) + } + + return c.ShowCluster(ctx, provider) +} + +// ApplyOutput contains the Terraform output values of a cluster creation +// or apply operation. +type ApplyOutput struct { + IP string + APIServerCertSANs []string + Secret string + UID string + GCP *GCPApplyOutput + Azure *AzureApplyOutput +} + +// AzureApplyOutput contains the Terraform output values of a terraform apply operation on Microsoft Azure. +type AzureApplyOutput struct { + ResourceGroup string + SubscriptionID string + NetworkSecurityGroupName string + LoadBalancerName string + UserAssignedIdentity string + // AttestationURL is the URL of the attestation provider. + AttestationURL string +} + +// GCPApplyOutput contains the Terraform output values of a terraform apply operation on GCP. +type GCPApplyOutput struct { + ProjectID string + IPCidrNode string + IPCidrPod string +} + +// IAMOutput contains the output information of the Terraform IAM operations. +type IAMOutput struct { + GCP GCPIAMOutput + Azure AzureIAMOutput + AWS AWSIAMOutput +} + +// GCPIAMOutput contains the output information of the Terraform IAM operation on GCP. +type GCPIAMOutput struct { + SaKey string +} + +// AzureIAMOutput contains the output information of the Terraform IAM operation on Microsoft Azure. +type AzureIAMOutput struct { + SubscriptionID string + TenantID string + UAMIID string +} + +// AWSIAMOutput contains the output information of the Terraform IAM operation on GCP. +type AWSIAMOutput struct { + ControlPlaneInstanceProfile string + WorkerNodeInstanceProfile string +} + +// ApplyIAMConfig creates an IAM configuration using Terraform. +func (c *Client) ApplyIAMConfig(ctx context.Context, provider cloudprovider.Provider, logLevel LogLevel) (IAMOutput, error) { + if err := c.setLogLevel(logLevel); err != nil { + return IAMOutput{}, fmt.Errorf("set terraform log level %s: %w", logLevel.String(), err) + } + + if err := c.tf.Init(ctx); err != nil { + return IAMOutput{}, err + } + + if err := c.tf.Apply(ctx); err != nil { + return IAMOutput{}, err + } + return c.ShowIAM(ctx, provider) +} + // Plan determines the diff that will be applied by Terraform. The plan output is written to the planFile. // If there is a diff, the returned bool is true. Otherwise, it is false. func (c *Client) Plan(ctx context.Context, logLevel LogLevel, planFile string) (bool, error) { diff --git a/cli/internal/terraform/terraform/azure/main.tf b/cli/internal/terraform/terraform/azure/main.tf index 60879bd6c..18d64e3c6 100644 --- a/cli/internal/terraform/terraform/azure/main.tf +++ b/cli/internal/terraform/terraform/azure/main.tf @@ -277,6 +277,9 @@ module "scale_set_group" { ] } +data "azurerm_subscription" "current" { +} + moved { from = module.scale_set_control_plane to = module.scale_set_group["control_plane_default"] diff --git a/cli/internal/terraform/terraform/azure/outputs.tf b/cli/internal/terraform/terraform/azure/outputs.tf index 875ac7d96..c857f3113 100644 --- a/cli/internal/terraform/terraform/azure/outputs.tf +++ b/cli/internal/terraform/terraform/azure/outputs.tf @@ -18,3 +18,24 @@ output "initSecret" { output "attestationURL" { value = var.create_maa ? azurerm_attestation_provider.attestation_provider[0].attestation_uri : "" } + +output "network_security_group_name" { + value = azurerm_network_security_group.security_group.name +} + +output "loadbalancer_name" { + value = azurerm_lb.loadbalancer.name +} + + +output "user_assigned_identity" { + value = var.user_assigned_identity +} + +output "resource_group" { + value = var.resource_group +} + +output "subscription_id" { + value = data.azurerm_subscription.current.subscription_id +} diff --git a/cli/internal/terraform/terraform/gcp/outputs.tf b/cli/internal/terraform/terraform/gcp/outputs.tf index 1059c8883..91fb606bd 100644 --- a/cli/internal/terraform/terraform/gcp/outputs.tf +++ b/cli/internal/terraform/terraform/gcp/outputs.tf @@ -18,3 +18,15 @@ output "initSecret" { value = random_password.initSecret.result sensitive = true } + +output "project" { + value = var.project +} + +output "ip_cidr_nodes" { + value = local.cidr_vpc_subnet_nodes +} + +output "ip_cidr_pods" { + value = local.cidr_vpc_subnet_pods +} diff --git a/cli/internal/terraform/terraform_test.go b/cli/internal/terraform/terraform_test.go index 5b7f9e0db..435cc5533 100644 --- a/cli/internal/terraform/terraform_test.go +++ b/cli/internal/terraform/terraform_test.go @@ -248,6 +248,21 @@ func TestCreateCluster(t *testing.T) { "api_server_cert_sans": { Value: []any{"192.0.2.100"}, }, + "user_assigned_identity": { + Value: "test_uami_id", + }, + "resource_group": { + Value: "test_rg", + }, + "subscription_id": { + Value: "test_subscription_id", + }, + "network_security_group_name": { + Value: "test_nsg_name", + }, + "loadbalancer_name": { + Value: "test_lb_name", + }, }, }, } @@ -435,7 +450,7 @@ func TestCreateCluster(t *testing.T) { path := path.Join(tc.pathBase, strings.ToLower(tc.provider.String())) require.NoError(c.PrepareWorkspace(path, tc.vars)) - tfOutput, err := c.CreateCluster(context.Background(), LogLevelDebug) + tfOutput, err := c.CreateCluster(context.Background(), tc.provider, LogLevelDebug) if tc.wantErr { assert.Error(err) @@ -445,7 +460,9 @@ func TestCreateCluster(t *testing.T) { assert.Equal("192.0.2.100", tfOutput.IP) assert.Equal("initSecret", tfOutput.Secret) assert.Equal("12345abc", tfOutput.UID) - assert.Equal(tc.expectedAttestationURL, tfOutput.AttestationURL) + if tc.provider == cloudprovider.Azure { + assert.Equal(tc.expectedAttestationURL, tfOutput.Azure.AttestationURL) + } }) } } diff --git a/cli/internal/upgrade/terraform.go b/cli/internal/upgrade/terraform.go index ca4dec002..18da89440 100644 --- a/cli/internal/upgrade/terraform.go +++ b/cli/internal/upgrade/terraform.go @@ -153,25 +153,24 @@ func CleanUpTerraformMigrations(upgradeID string, fileHandler file.Handler) erro // In case of a successful upgrade, the output will be written to the specified file and the old Terraform directory is replaced // By the new one. func (u *TerraformUpgrader) ApplyTerraformMigrations(ctx context.Context, opts TerraformUpgradeOptions, upgradeID string) error { - tfOutput, err := u.tf.CreateCluster(ctx, opts.LogLevel) + tfOutput, err := u.tf.CreateCluster(ctx, opts.CSP, opts.LogLevel) if err != nil { return fmt.Errorf("terraform apply: %w", err) } - // AttestationURL is only set for Azure. - if tfOutput.AttestationURL != "" { - if err := u.policyPatcher.Patch(ctx, tfOutput.AttestationURL); err != nil { - return fmt.Errorf("patching policies: %w", err) - } - } - outputFileContents := clusterid.File{ CloudProvider: opts.CSP, InitSecret: []byte(tfOutput.Secret), IP: tfOutput.IP, APIServerCertSANs: tfOutput.APIServerCertSANs, UID: tfOutput.UID, - AttestationURL: tfOutput.AttestationURL, + } + // AttestationURL is only set for Azure. + if tfOutput.Azure != nil { + if err := u.policyPatcher.Patch(ctx, tfOutput.Azure.AttestationURL); err != nil { + return fmt.Errorf("patching policies: %w", err) + } + outputFileContents.AttestationURL = tfOutput.Azure.AttestationURL } if err := u.fileHandler.RemoveAll(constants.TerraformWorkingDir); err != nil { @@ -215,7 +214,7 @@ type tfClientCommon interface { // tfResourceClient is a Terraform client for managing cluster resources. type tfResourceClient interface { PrepareUpgradeWorkspace(path, oldWorkingDir, newWorkingDir, backupDir string, vars terraform.Variables) error - CreateCluster(ctx context.Context, logLevel terraform.LogLevel) (terraform.ApplyOutput, error) + CreateCluster(ctx context.Context, provider cloudprovider.Provider, logLevel terraform.LogLevel) (terraform.ApplyOutput, error) tfClientCommon } diff --git a/cli/internal/upgrade/terraform_test.go b/cli/internal/upgrade/terraform_test.go index e6336bf95..d15a99811 100644 --- a/cli/internal/upgrade/terraform_test.go +++ b/cli/internal/upgrade/terraform_test.go @@ -371,7 +371,7 @@ func (u *stubTerraformClient) Plan(context.Context, terraform.LogLevel, string) return u.hasDiff, u.planErr } -func (u *stubTerraformClient) CreateCluster(context.Context, terraform.LogLevel) (terraform.ApplyOutput, error) { +func (u *stubTerraformClient) CreateCluster(context.Context, cloudprovider.Provider, terraform.LogLevel) (terraform.ApplyOutput, error) { return terraform.ApplyOutput{}, u.CreateClusterErr } diff --git a/internal/cloud/azure/azure.go b/internal/cloud/azure/azure.go index 4f724ab16..f552ab8a5 100644 --- a/internal/cloud/azure/azure.go +++ b/internal/cloud/azure/azure.go @@ -17,7 +17,6 @@ package azure import ( "context" - "encoding/json" "errors" "fmt" "path" @@ -99,61 +98,6 @@ func New(ctx context.Context) (*Cloud, error) { }, nil } -// GetCCMConfig returns the configuration needed for the Kubernetes Cloud Controller Manager on Azure. -func (c *Cloud) GetCCMConfig(ctx context.Context, providerID string, cloudServiceAccountURI string) ([]byte, error) { - subscriptionID, resourceGroup, err := azureshared.BasicsFromProviderID(providerID) - if err != nil { - return nil, fmt.Errorf("parsing provider ID: %w", err) - } - creds, err := azureshared.ApplicationCredentialsFromURI(cloudServiceAccountURI) - if err != nil { - return nil, fmt.Errorf("parsing service account URI: %w", err) - } - uid, err := c.imds.uid(ctx) - if err != nil { - return nil, fmt.Errorf("retrieving instance UID: %w", err) - } - - securityGroupName, err := c.getNetworkSecurityGroupName(ctx, resourceGroup, uid) - if err != nil { - return nil, fmt.Errorf("retrieving network security group name: %w", err) - } - - loadBalancer, err := c.getLoadBalancer(ctx, resourceGroup, uid) - if err != nil { - return nil, fmt.Errorf("retrieving load balancer: %w", err) - } - if loadBalancer == nil || loadBalancer.Name == nil { - return nil, fmt.Errorf("could not dereference load balancer name") - } - - var uamiClientID string - useManagedIdentityExtension := creds.PreferredAuthMethod == azureshared.AuthMethodUserAssignedIdentity - if useManagedIdentityExtension { - uamiClientID, err = c.getUAMIClientIDFromURI(ctx, providerID, creds.UamiResourceID) - if err != nil { - return nil, fmt.Errorf("retrieving user-assigned managed identity client ID: %w", err) - } - } - - config := cloudConfig{ - Cloud: "AzurePublicCloud", - TenantID: creds.TenantID, - SubscriptionID: subscriptionID, - ResourceGroup: resourceGroup, - LoadBalancerSku: "standard", - SecurityGroupName: securityGroupName, - LoadBalancerName: *loadBalancer.Name, - UseInstanceMetadata: true, - VMType: "vmss", - Location: creds.Location, - UseManagedIdentityExtension: useManagedIdentityExtension, - UserAssignedIdentityID: uamiClientID, - } - - return json.Marshal(config) -} - // GetLoadBalancerEndpoint retrieves the first load balancer IP from cloud provider metadata. // // The returned string is an IP address without a port, but the method name needs to satisfy the @@ -286,24 +230,6 @@ func (c *Cloud) getInstance(ctx context.Context, providerID string) (metadata.In return instance, nil } -func (c *Cloud) getUAMIClientIDFromURI(ctx context.Context, providerID, resourceID string) (string, error) { - // userAssignedIdentityURI := "/subscriptions/{subscription-id}/resourcegroups/{resource-group}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/{identity-name}" - _, resourceGroup, scaleSet, instanceID, err := azureshared.ScaleSetInformationFromProviderID(providerID) - if err != nil { - return "", fmt.Errorf("invalid provider ID: %w", err) - } - vmResp, err := c.scaleSetsVMAPI.Get(ctx, resourceGroup, scaleSet, instanceID, nil) - if err != nil { - return "", fmt.Errorf("retrieving instance: %w", err) - } - for rID, v := range vmResp.Identity.UserAssignedIdentities { - if rID == resourceID { - return *v.ClientID, nil - } - } - return "", fmt.Errorf("no user assinged identity found for resource ID %s", resourceID) -} - // getNetworkSecurityGroupName returns the security group name of the resource group. func (c *Cloud) getNetworkSecurityGroupName(ctx context.Context, resourceGroup, uid string) (string, error) { pager := c.secGroupAPI.NewListPager(resourceGroup, nil) @@ -462,26 +388,6 @@ func (c *Cloud) getLoadBalancerDNSName(ctx context.Context) (string, error) { } */ -type cloudConfig struct { - Cloud string `json:"cloud,omitempty"` - TenantID string `json:"tenantId,omitempty"` - SubscriptionID string `json:"subscriptionId,omitempty"` - ResourceGroup string `json:"resourceGroup,omitempty"` - Location string `json:"location,omitempty"` - SubnetName string `json:"subnetName,omitempty"` - SecurityGroupName string `json:"securityGroupName,omitempty"` - SecurityGroupResourceGroup string `json:"securityGroupResourceGroup,omitempty"` - LoadBalancerName string `json:"loadBalancerName,omitempty"` - LoadBalancerSku string `json:"loadBalancerSku,omitempty"` - VNetName string `json:"vnetName,omitempty"` - VNetResourceGroup string `json:"vnetResourceGroup,omitempty"` - CloudProviderBackoff bool `json:"cloudProviderBackoff,omitempty"` - UseInstanceMetadata bool `json:"useInstanceMetadata,omitempty"` - VMType string `json:"vmType,omitempty"` - UseManagedIdentityExtension bool `json:"useManagedIdentityExtension,omitempty"` - UserAssignedIdentityID string `json:"userAssignedIdentityID,omitempty"` -} - // convertToInstanceMetadata converts a armcomputev2.VirtualMachineScaleSetVM to a metadata.InstanceMetadata. func convertToInstanceMetadata(vm armcompute.VirtualMachineScaleSetVM, networkInterfaces []armnetwork.Interface, ) (metadata.InstanceMetadata, error) { diff --git a/internal/cloud/azure/azure_test.go b/internal/cloud/azure/azure_test.go index 7b1b44b9e..15d753276 100644 --- a/internal/cloud/azure/azure_test.go +++ b/internal/cloud/azure/azure_test.go @@ -8,7 +8,6 @@ package azure import ( "context" - "encoding/json" "errors" "testing" @@ -28,336 +27,6 @@ func TestMain(m *testing.M) { goleak.VerifyTestMain(m) } -func TestGetCCMConfig(t *testing.T) { - someErr := errors.New("failed") - goodLB := armnetwork.LoadBalancer{ - Name: to.Ptr("load-balancer"), - Tags: map[string]*string{ - cloud.TagUID: to.Ptr("uid"), - }, - Properties: &armnetwork.LoadBalancerPropertiesFormat{ - FrontendIPConfigurations: []*armnetwork.FrontendIPConfiguration{ - { - Properties: &armnetwork.FrontendIPConfigurationPropertiesFormat{ - PublicIPAddress: &armnetwork.PublicIPAddress{ID: to.Ptr("/subscriptions/subscription/resourceGroups/resourceGroup/providers/Microsoft.Network/publicIPAddresses/pubIPName")}, - }, - }, - }, - }, - } - goodSecurityGroup := armnetwork.SecurityGroup{ - Tags: map[string]*string{ - cloud.TagUID: to.Ptr("uid"), - }, - Name: to.Ptr("security-group"), - } - - uamiClientID := "uami-client-id" - - testCases := map[string]struct { - imdsAPI imdsAPI - loadBalancerAPI loadBalancerAPI - secGroupAPI securityGroupsAPI - scaleSetsVMAPI virtualMachineScaleSetVMsAPI - providerID string - cloudServiceAccountURI string - wantErr bool - wantConfig cloudConfig - }{ - "success": { - imdsAPI: &stubIMDSAPI{ - uidVal: "uid", - }, - loadBalancerAPI: &stubLoadBalancersAPI{ - pager: &stubLoadBalancersClientListPager{ - list: []armnetwork.LoadBalancer{goodLB}, - }, - }, - secGroupAPI: &stubSecurityGroupsAPI{ - pager: &stubSecurityGroupsClientListPager{ - list: []armnetwork.SecurityGroup{goodSecurityGroup}, - }, - }, - scaleSetsVMAPI: &stubVirtualMachineScaleSetVMsAPI{ - getVM: armcompute.VirtualMachineScaleSetVM{ - Identity: &armcompute.VirtualMachineIdentity{ - UserAssignedIdentities: map[string]*armcompute.UserAssignedIdentitiesValue{ - "subscriptions/9b352db0-82af-408c-a02c-36fbffbf7015/resourceGroups/resourceGroupName/providers/Microsoft.ManagedIdentity/userAssignedIdentities/UAMIName": {ClientID: &uamiClientID}, - }, - }, - }, - }, - providerID: "azure:///subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/virtualMachineScaleSets/scale-set/virtualMachines/0", - cloudServiceAccountURI: "serviceaccount://azure?tenant_id=tenant-id&client_id=client-id&client_secret=client-secret&location=westeurope&preferred_auth_method=userassignedidentity&uami_resource_id=subscriptions%2F9b352db0-82af-408c-a02c-36fbffbf7015%2FresourceGroups%2FresourceGroupName%2Fproviders%2FMicrosoft.ManagedIdentity%2FuserAssignedIdentities%2FUAMIName", - wantConfig: cloudConfig{ - Cloud: "AzurePublicCloud", - TenantID: "tenant-id", - SubscriptionID: "subscription-id", - ResourceGroup: "resource-group", - LoadBalancerSku: "standard", - SecurityGroupName: "security-group", - LoadBalancerName: "load-balancer", - UseInstanceMetadata: true, - UseManagedIdentityExtension: true, - UserAssignedIdentityID: uamiClientID, - VMType: "vmss", - Location: "westeurope", - }, - }, - "no app registration": { - imdsAPI: &stubIMDSAPI{ - uidVal: "uid", - }, - loadBalancerAPI: &stubLoadBalancersAPI{ - pager: &stubLoadBalancersClientListPager{ - list: []armnetwork.LoadBalancer{goodLB}, - }, - }, - secGroupAPI: &stubSecurityGroupsAPI{ - pager: &stubSecurityGroupsClientListPager{ - list: []armnetwork.SecurityGroup{goodSecurityGroup}, - }, - }, - providerID: "azure:///subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/virtualMachineScaleSets/scale-set/virtualMachines/0", - cloudServiceAccountURI: "serviceaccount://azure?tenant_id=tenant-id&location=westeurope", - wantConfig: cloudConfig{ - Cloud: "AzurePublicCloud", - TenantID: "tenant-id", - SubscriptionID: "subscription-id", - ResourceGroup: "resource-group", - LoadBalancerSku: "standard", - SecurityGroupName: "security-group", - LoadBalancerName: "load-balancer", - UseInstanceMetadata: true, - VMType: "vmss", - Location: "westeurope", - }, - }, - "missing UID tag": { - imdsAPI: &stubIMDSAPI{ - uidVal: "uid", - }, - loadBalancerAPI: &stubLoadBalancersAPI{ - pager: &stubLoadBalancersClientListPager{ - list: []armnetwork.LoadBalancer{ - { - Name: to.Ptr("load-balancer"), - Properties: &armnetwork.LoadBalancerPropertiesFormat{ - FrontendIPConfigurations: []*armnetwork.FrontendIPConfiguration{ - { - Properties: &armnetwork.FrontendIPConfigurationPropertiesFormat{ - PublicIPAddress: &armnetwork.PublicIPAddress{ID: to.Ptr("/subscriptions/subscription/resourceGroups/resourceGroup/providers/Microsoft.Network/publicIPAddresses/pubIPName")}, - }, - }, - }, - }, - }, - }, - }, - }, - secGroupAPI: &stubSecurityGroupsAPI{ - pager: &stubSecurityGroupsClientListPager{ - list: []armnetwork.SecurityGroup{goodSecurityGroup}, - }, - }, - providerID: "azure:///subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/virtualMachineScaleSets/scale-set/virtualMachines/0", - cloudServiceAccountURI: "serviceaccount://azure?tenant_id=tenant-id&client_id=client-id&client_secret=client-secret&location=westeurope", - wantErr: true, - }, - "only correct UID is chosen": { - imdsAPI: &stubIMDSAPI{ - uidVal: "uid", - }, - loadBalancerAPI: &stubLoadBalancersAPI{ - pager: &stubLoadBalancersClientListPager{ - list: []armnetwork.LoadBalancer{ - { - Name: to.Ptr("load-balancer"), - Tags: map[string]*string{ - cloud.TagUID: to.Ptr("different-uid"), - }, - Properties: &armnetwork.LoadBalancerPropertiesFormat{ - FrontendIPConfigurations: []*armnetwork.FrontendIPConfiguration{ - { - Properties: &armnetwork.FrontendIPConfigurationPropertiesFormat{ - PublicIPAddress: &armnetwork.PublicIPAddress{ID: to.Ptr("/subscriptions/subscription/resourceGroups/resourceGroup/providers/Microsoft.Network/publicIPAddresses/pubIPName")}, - }, - }, - }, - }, - }, - goodLB, - }, - }, - }, - secGroupAPI: &stubSecurityGroupsAPI{ - pager: &stubSecurityGroupsClientListPager{ - list: []armnetwork.SecurityGroup{goodSecurityGroup}, - }, - }, - providerID: "azure:///subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/virtualMachineScaleSets/scale-set/virtualMachines/0", - cloudServiceAccountURI: "serviceaccount://azure?tenant_id=tenant-id&client_id=client-id&client_secret=client-secret&location=westeurope", - wantConfig: cloudConfig{ - Cloud: "AzurePublicCloud", - TenantID: "tenant-id", - SubscriptionID: "subscription-id", - ResourceGroup: "resource-group", - LoadBalancerSku: "standard", - SecurityGroupName: "security-group", - LoadBalancerName: "load-balancer", - UseInstanceMetadata: true, - VMType: "vmss", - Location: "westeurope", - }, - }, - "load balancer list error": { - imdsAPI: &stubIMDSAPI{ - uidVal: "uid", - }, - loadBalancerAPI: &stubLoadBalancersAPI{ - pager: &stubLoadBalancersClientListPager{ - fetchErr: someErr, - }, - }, - secGroupAPI: &stubSecurityGroupsAPI{ - pager: &stubSecurityGroupsClientListPager{ - list: []armnetwork.SecurityGroup{goodSecurityGroup}, - }, - }, - providerID: "azure:///subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/virtualMachineScaleSets/scale-set/virtualMachines/0", - cloudServiceAccountURI: "serviceaccount://azure?tenant_id=tenant-id&client_id=client-id&client_secret=client-secret&location=westeurope", - wantErr: true, - }, - "missing load balancer name": { - imdsAPI: &stubIMDSAPI{ - uidVal: "uid", - }, - loadBalancerAPI: &stubLoadBalancersAPI{ - pager: &stubLoadBalancersClientListPager{ - list: []armnetwork.LoadBalancer{{ - Tags: map[string]*string{ - cloud.TagUID: to.Ptr("uid"), - }, - Properties: &armnetwork.LoadBalancerPropertiesFormat{ - FrontendIPConfigurations: []*armnetwork.FrontendIPConfiguration{ - { - Properties: &armnetwork.FrontendIPConfigurationPropertiesFormat{ - PublicIPAddress: &armnetwork.PublicIPAddress{ID: to.Ptr("/subscriptions/subscription/resourceGroups/resourceGroup/providers/Microsoft.Network/publicIPAddresses/pubIPName")}, - }, - }, - }, - }, - }}, - }, - }, - secGroupAPI: &stubSecurityGroupsAPI{ - pager: &stubSecurityGroupsClientListPager{ - list: []armnetwork.SecurityGroup{goodSecurityGroup}, - }, - }, - providerID: "azure:///subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/virtualMachineScaleSets/scale-set/virtualMachines/0", - cloudServiceAccountURI: "serviceaccount://azure?tenant_id=tenant-id&client_id=client-id&client_secret=client-secret&location=westeurope", - wantErr: true, - }, - "security group list error": { - imdsAPI: &stubIMDSAPI{ - uidVal: "uid", - }, - loadBalancerAPI: &stubLoadBalancersAPI{ - pager: &stubLoadBalancersClientListPager{ - list: []armnetwork.LoadBalancer{goodLB}, - }, - }, - secGroupAPI: &stubSecurityGroupsAPI{ - pager: &stubSecurityGroupsClientListPager{ - fetchErr: someErr, - }, - }, - providerID: "azure:///subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/virtualMachineScaleSets/scale-set/virtualMachines/0", - cloudServiceAccountURI: "serviceaccount://azure?tenant_id=tenant-id&client_id=client-id&client_secret=client-secret&location=westeurope", - wantErr: true, - }, - "invalid provider ID": { - imdsAPI: &stubIMDSAPI{ - uidVal: "uid", - }, - loadBalancerAPI: &stubLoadBalancersAPI{ - pager: &stubLoadBalancersClientListPager{ - list: []armnetwork.LoadBalancer{goodLB}, - }, - }, - secGroupAPI: &stubSecurityGroupsAPI{ - pager: &stubSecurityGroupsClientListPager{ - list: []armnetwork.SecurityGroup{goodSecurityGroup}, - }, - }, - providerID: "invalid:///subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/virtualMachineScaleSets/scale-set/virtualMachines/0", - cloudServiceAccountURI: "serviceaccount://azure?tenant_id=tenant-id&client_id=client-id&client_secret=client-secret&location=westeurope", - wantErr: true, - }, - "invalid cloud service account URI": { - imdsAPI: &stubIMDSAPI{ - uidVal: "uid", - }, - loadBalancerAPI: &stubLoadBalancersAPI{ - pager: &stubLoadBalancersClientListPager{ - list: []armnetwork.LoadBalancer{goodLB}, - }, - }, - secGroupAPI: &stubSecurityGroupsAPI{ - pager: &stubSecurityGroupsClientListPager{ - list: []armnetwork.SecurityGroup{goodSecurityGroup}, - }, - }, - providerID: "azure:///subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/virtualMachineScaleSets/scale-set/virtualMachines/0", - cloudServiceAccountURI: "invalid://azure?tenant_id=tenant-id&client_id=client-id&client_secret=client-secret&location=westeurope", - wantErr: true, - }, - "imds error": { - imdsAPI: &stubIMDSAPI{ - uidErr: someErr, - }, - loadBalancerAPI: &stubLoadBalancersAPI{ - pager: &stubLoadBalancersClientListPager{ - list: []armnetwork.LoadBalancer{goodLB}, - }, - }, - secGroupAPI: &stubSecurityGroupsAPI{ - pager: &stubSecurityGroupsClientListPager{ - list: []armnetwork.SecurityGroup{goodSecurityGroup}, - }, - }, - providerID: "azure:///subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/virtualMachineScaleSets/scale-set/virtualMachines/0", - cloudServiceAccountURI: "serviceaccount://azure?tenant_id=tenant-id&client_id=client-id&client_secret=client-secret&location=westeurope", - wantErr: true, - }, - } - - for name, tc := range testCases { - t.Run(name, func(t *testing.T) { - assert := assert.New(t) - require := require.New(t) - - cloud := &Cloud{ - imds: tc.imdsAPI, - loadBalancerAPI: tc.loadBalancerAPI, - secGroupAPI: tc.secGroupAPI, - scaleSetsVMAPI: tc.scaleSetsVMAPI, - } - config, err := cloud.GetCCMConfig(context.Background(), tc.providerID, tc.cloudServiceAccountURI) - if tc.wantErr { - assert.Error(err) - return - } - assert.NoError(err) - - wantConfig, err := json.Marshal(tc.wantConfig) - require.NoError(err) - assert.JSONEq(string(wantConfig), string(config)) - }) - } -} - func TestGetInstance(t *testing.T) { someErr := errors.New("failed") sampleProviderID := "azure:///subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/virtualMachineScaleSets/scale-set-name/virtualMachines/instance-id" diff --git a/internal/cloud/azureshared/appcredentials.go b/internal/cloud/azureshared/appcredentials.go index c1435f5aa..7c6c7ec65 100644 --- a/internal/cloud/azureshared/appcredentials.go +++ b/internal/cloud/azureshared/appcredentials.go @@ -9,13 +9,21 @@ package azureshared import ( "fmt" "net/url" + "regexp" "strings" ) +var ( + subscriptionPattern = regexp.MustCompile(`subscriptions/([^/]+)/`) + rgPattern = regexp.MustCompile(`resourceGroups/([^/]+)/`) +) + // ApplicationCredentials is a set of Azure API credentials. // It can contain a client secret and carries the preferred authentication method. // It is the equivalent of a service account key in other cloud providers. type ApplicationCredentials struct { + SubscriptionID string + ResourceGroup string TenantID string AppClientID string ClientSecretValue string @@ -37,8 +45,14 @@ func ApplicationCredentialsFromURI(cloudServiceAccountURI string) (ApplicationCr return ApplicationCredentials{}, fmt.Errorf("invalid service account URI: invalid host: %s", uri.Host) } query := uri.Query() + + subscriptionID := getFirstMatchOrEmpty(subscriptionPattern, query.Get("uami_resource_id")) + resourceGroup := getFirstMatchOrEmpty(rgPattern, query.Get("uami_resource_id")) + preferredAuthMethod := FromString(query.Get("preferred_auth_method")) return ApplicationCredentials{ + SubscriptionID: subscriptionID, + ResourceGroup: resourceGroup, TenantID: query.Get("tenant_id"), AppClientID: query.Get("client_id"), ClientSecretValue: query.Get("client_secret"), @@ -48,6 +62,15 @@ func ApplicationCredentialsFromURI(cloudServiceAccountURI string) (ApplicationCr }, nil } +func getFirstMatchOrEmpty(pattern *regexp.Regexp, str string) string { + subscriptionMatches := pattern.FindStringSubmatch(str) + var subscriptionID string + if len(subscriptionMatches) > 1 { + subscriptionID = subscriptionMatches[1] + } + return subscriptionID +} + // ToCloudServiceAccountURI converts the ApplicationCredentials into a cloud service account URI. func (c ApplicationCredentials) ToCloudServiceAccountURI() string { query := url.Values{} diff --git a/internal/cloud/azureshared/appcredentials_test.go b/internal/cloud/azureshared/appcredentials_test.go index c12be2ec0..086a16d8f 100644 --- a/internal/cloud/azureshared/appcredentials_test.go +++ b/internal/cloud/azureshared/appcredentials_test.go @@ -27,12 +27,16 @@ func TestApplicationCredentialsFromURI(t *testing.T) { Location: "location", UamiResourceID: "subscriptions/9b352db0-82af-408c-a02c-36fbffbf7015/resourceGroups/resourceGroupName/providers/Microsoft.ManagedIdentity/userAssignedIdentities/UAMIName", PreferredAuthMethod: AuthMethodServicePrincipal, + SubscriptionID: "9b352db0-82af-408c-a02c-36fbffbf7015", + ResourceGroup: "resourceGroupName", } credsWithoutSecret := ApplicationCredentials{ TenantID: "tenant-id", Location: "location", UamiResourceID: "subscriptions/9b352db0-82af-408c-a02c-36fbffbf7015/resourceGroups/resourceGroupName/providers/Microsoft.ManagedIdentity/userAssignedIdentities/UAMIName", PreferredAuthMethod: AuthMethodUserAssignedIdentity, + SubscriptionID: "9b352db0-82af-408c-a02c-36fbffbf7015", + ResourceGroup: "resourceGroupName", } credsWithoutPreferrredAuthMethod := ApplicationCredentials{ TenantID: "tenant-id", diff --git a/internal/deploy/helm/BUILD.bazel b/internal/deploy/helm/BUILD.bazel index 9fb581932..5cb17b315 100644 --- a/internal/deploy/helm/BUILD.bazel +++ b/internal/deploy/helm/BUILD.bazel @@ -11,10 +11,8 @@ go_library( visibility = ["//:__subpackages__"], deps = [ "//internal/constants", - "//internal/logger", "//internal/retry", "@io_k8s_apimachinery//pkg/util/wait", - "@org_uber_go_zap//:zap", "@sh_helm_helm_v3//pkg/action", "@sh_helm_helm_v3//pkg/chart", "@sh_helm_helm_v3//pkg/chart/loader", diff --git a/internal/deploy/helm/install.go b/internal/deploy/helm/install.go index 66511b1eb..6e0558a26 100644 --- a/internal/deploy/helm/install.go +++ b/internal/deploy/helm/install.go @@ -14,9 +14,7 @@ import ( "time" "github.com/edgelesssys/constellation/v2/internal/constants" - "github.com/edgelesssys/constellation/v2/internal/logger" "github.com/edgelesssys/constellation/v2/internal/retry" - "go.uber.org/zap" "helm.sh/helm/v3/pkg/action" "helm.sh/helm/v3/pkg/chart" "helm.sh/helm/v3/pkg/chart/loader" @@ -31,20 +29,25 @@ const ( maximumRetryAttempts = 3 ) +type debugLog interface { + Debugf(format string, args ...any) + Sync() +} + // Installer is a wrapper for a helm install action. type Installer struct { *action.Install - log *logger.Logger + log debugLog } // NewInstaller creates a new Installer with the given logger. -func NewInstaller(log *logger.Logger, kubeconfig string) (*Installer, error) { +func NewInstaller(kubeconfig string, logger debugLog) (*Installer, error) { settings := cli.New() settings.KubeConfig = kubeconfig actionConfig := &action.Configuration{} if err := actionConfig.Init(settings.RESTClientGetter(), constants.HelmNamespace, - "secret", log.Infof); err != nil { + "secret", logger.Debugf); err != nil { return nil, err } @@ -53,13 +56,12 @@ func NewInstaller(log *logger.Logger, kubeconfig string) (*Installer, error) { action.Timeout = timeout return &Installer{ - action, - log, + Install: action, + log: logger, }, nil } // InstallChart is the generic install function for helm charts. -// When timeout is nil, the default timeout is used. func (h *Installer) InstallChart(ctx context.Context, release Release) error { return h.InstallChartWithValues(ctx, release, nil) } @@ -115,7 +117,7 @@ func (h *Installer) install(ctx context.Context, chartRaw []byte, values map[str return fmt.Errorf("helm install: %w", err) } retryLoopFinishDuration := time.Since(retryLoopStartTime) - h.log.With(zap.String("chart", chart.Name()), zap.Duration("duration", retryLoopFinishDuration)).Infof("Helm chart installation finished") + h.log.Debugf("Helm chart %q installation finished after %s", chart.Name(), retryLoopFinishDuration) return nil } @@ -143,15 +145,14 @@ type installDoer struct { Installer *Installer chart *chart.Chart values map[string]any - log *logger.Logger + log debugLog } // Do logs which chart is installed and tries to install it. func (i installDoer) Do(ctx context.Context) error { - i.log.With(zap.String("chart", i.chart.Name())).Infof("Trying to install Helm chart") - + i.log.Debugf("Trying to install Helm chart %s", i.chart.Name()) if _, err := i.Installer.RunWithContext(ctx, i.chart, i.values); err != nil { - i.log.With(zap.Error(err), zap.String("chart", i.chart.Name())).Errorf("Helm chart installation failed") + i.log.Debugf("Helm chart installation % failed: %v", i.chart.Name(), err) return err }