cli: install helm charts in cli instead of bootstrapper (#2136)

* init

* fixup! init

* gcp working?

* fixup! fixup! init

* azure cfg for microService installation

* fixup! azure cfg for microService installation

* fixup! azure cfg for microService installation

* cleanup bootstrapper code

* cleanup helminstall code

* fixup! cleanup helminstall code

* Update internal/deploy/helm/install.go

Co-authored-by: Daniel Weiße <66256922+daniel-weisse@users.noreply.github.com>

* daniel feedback

* TODO add provider (also to CreateCluster) so we can ensure that provider specific output

* fixup! daniel feedback

* use debugLog in helm installer

* placeholderHelmInstaller

* rename to stub

---------

Co-authored-by: Daniel Weiße <66256922+daniel-weisse@users.noreply.github.com>
This commit is contained in:
Adrian Stobbe 2023-07-31 10:53:05 +02:00 committed by GitHub
parent ef60d00a60
commit 26305e8f80
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
38 changed files with 775 additions and 970 deletions

View File

@ -67,7 +67,7 @@ func main() {
var openDevice vtpm.TPMOpenFunc var openDevice vtpm.TPMOpenFunc
var fs afero.Fs var fs afero.Fs
helmClient, err := helm.NewInstaller(log, constants.ControlPlaneAdminConfFilename) helmClient, err := helm.NewInstaller(constants.ControlPlaneAdminConfFilename, log)
if err != nil { if err != nil {
log.With(zap.Error(err)).Fatalf("Helm client could not be initialized") log.With(zap.Error(err)).Fatalf("Helm client could not be initialized")
} }

View File

@ -21,7 +21,7 @@ type clusterFake struct{}
// InitCluster fakes bootstrapping a new cluster with the current node being the master, returning the arguments required to join the cluster. // InitCluster fakes bootstrapping a new cluster with the current node being the master, returning the arguments required to join the cluster.
func (c *clusterFake) InitCluster( func (c *clusterFake) InitCluster(
context.Context, string, string, string, []byte, context.Context, string, string,
[]byte, bool, components.Components, []string, *logger.Logger, []byte, bool, components.Components, []string, *logger.Logger,
) ([]byte, error) { ) ([]byte, error) {
return []byte{}, nil return []byte{}, nil

View File

@ -213,10 +213,8 @@ func (s *Server) Init(req *initproto.InitRequest, stream initproto.API_InitServe
} }
kubeconfig, err := s.initializer.InitCluster(stream.Context(), kubeconfig, err := s.initializer.InitCluster(stream.Context(),
req.CloudServiceAccountUri,
req.KubernetesVersion, req.KubernetesVersion,
clusterName, clusterName,
measurementSalt,
req.HelmDeployments, req.HelmDeployments,
req.ConformanceMode, req.ConformanceMode,
components.NewComponentsFromInitProto(req.KubernetesComponents), components.NewComponentsFromInitProto(req.KubernetesComponents),
@ -342,10 +340,8 @@ type ClusterInitializer interface {
// InitCluster initializes a new Kubernetes cluster. // InitCluster initializes a new Kubernetes cluster.
InitCluster( InitCluster(
ctx context.Context, ctx context.Context,
cloudServiceAccountURI string,
k8sVersion string, k8sVersion string,
clusterName string, clusterName string,
measurementSalt []byte,
helmDeployments []byte, helmDeployments []byte,
conformanceMode bool, conformanceMode bool,
kubernetesComponents components.Components, kubernetesComponents components.Components,

View File

@ -406,7 +406,7 @@ type stubClusterInitializer struct {
} }
func (i *stubClusterInitializer) InitCluster( func (i *stubClusterInitializer) InitCluster(
context.Context, string, string, string, []byte, context.Context, string, string,
[]byte, bool, components.Components, []string, *logger.Logger, []byte, bool, components.Components, []string, *logger.Logger,
) ([]byte, error) { ) ([]byte, error) {
return i.initClusterKubeconfig, i.initClusterErr return i.initClusterKubeconfig, i.initClusterErr

View File

@ -14,9 +14,7 @@ go_library(
"//bootstrapper/internal/kubernetes/k8sapi", "//bootstrapper/internal/kubernetes/k8sapi",
"//bootstrapper/internal/kubernetes/kubewaiter", "//bootstrapper/internal/kubernetes/kubewaiter",
"//internal/cloud/cloudprovider", "//internal/cloud/cloudprovider",
"//internal/cloud/gcpshared",
"//internal/cloud/metadata", "//internal/cloud/metadata",
"//internal/cloud/openstack",
"//internal/constants", "//internal/constants",
"//internal/deploy/helm", "//internal/deploy/helm",
"//internal/kubernetes", "//internal/kubernetes",

View File

@ -9,7 +9,6 @@ package kubernetes
import ( import (
"context" "context"
"encoding/base64"
"encoding/json" "encoding/json"
"errors" "errors"
"fmt" "fmt"
@ -22,8 +21,6 @@ import (
"github.com/edgelesssys/constellation/v2/bootstrapper/internal/kubernetes/k8sapi" "github.com/edgelesssys/constellation/v2/bootstrapper/internal/kubernetes/k8sapi"
"github.com/edgelesssys/constellation/v2/bootstrapper/internal/kubernetes/kubewaiter" "github.com/edgelesssys/constellation/v2/bootstrapper/internal/kubernetes/kubewaiter"
"github.com/edgelesssys/constellation/v2/internal/cloud/cloudprovider" "github.com/edgelesssys/constellation/v2/internal/cloud/cloudprovider"
"github.com/edgelesssys/constellation/v2/internal/cloud/gcpshared"
"github.com/edgelesssys/constellation/v2/internal/cloud/openstack"
"github.com/edgelesssys/constellation/v2/internal/constants" "github.com/edgelesssys/constellation/v2/internal/constants"
"github.com/edgelesssys/constellation/v2/internal/deploy/helm" "github.com/edgelesssys/constellation/v2/internal/deploy/helm"
"github.com/edgelesssys/constellation/v2/internal/kubernetes" "github.com/edgelesssys/constellation/v2/internal/kubernetes"
@ -78,7 +75,7 @@ func New(cloudProvider string, clusterUtil clusterUtil, configProvider configura
// InitCluster initializes a new Kubernetes cluster and applies pod network provider. // InitCluster initializes a new Kubernetes cluster and applies pod network provider.
func (k *KubeWrapper) InitCluster( func (k *KubeWrapper) InitCluster(
ctx context.Context, cloudServiceAccountURI, versionString, clusterName string, measurementSalt []byte, ctx context.Context, versionString, clusterName string,
helmReleasesRaw []byte, conformanceMode bool, kubernetesComponents components.Components, apiServerCertSANs []string, log *logger.Logger, helmReleasesRaw []byte, conformanceMode bool, kubernetesComponents components.Components, apiServerCertSANs []string, log *logger.Logger,
) ([]byte, error) { ) ([]byte, error) {
log.With(zap.String("version", versionString)).Infof("Installing Kubernetes components") log.With(zap.String("version", versionString)).Infof("Installing Kubernetes components")
@ -223,75 +220,10 @@ func (k *KubeWrapper) InitCluster(
// Continue and don't throw an error here - things might be okay. // Continue and don't throw an error here - things might be okay.
} }
serviceConfig := constellationServicesConfig{
measurementSalt: measurementSalt,
subnetworkPodCIDR: subnetworkPodCIDR,
cloudServiceAccountURI: cloudServiceAccountURI,
loadBalancerIP: controlPlaneHost,
}
constellationVals, err := k.setupExtraVals(ctx, serviceConfig)
if err != nil {
return nil, fmt.Errorf("setting up extraVals: %w", err)
}
log.Infof("Setting up internal-config ConfigMap") log.Infof("Setting up internal-config ConfigMap")
if err := k.setupInternalConfigMap(ctx); err != nil { if err := k.setupInternalConfigMap(ctx); err != nil {
return nil, fmt.Errorf("failed to setup internal ConfigMap: %w", err) return nil, fmt.Errorf("failed to setup internal ConfigMap: %w", err)
} }
log.Infof("Installing Constellation microservices")
if err = k.helmClient.InstallChartWithValues(ctx, helmReleases.ConstellationServices, constellationVals); err != nil {
return nil, fmt.Errorf("installing constellation-services: %w", err)
}
// cert-manager provides CRDs used by other deployments,
// so it should be installed as early as possible, but after the services cert-manager depends on.
log.Infof("Installing cert-manager")
if err = k.helmClient.InstallChart(ctx, helmReleases.CertManager); err != nil {
return nil, fmt.Errorf("installing cert-manager: %w", err)
}
// Install CSI drivers if enabled by the user.
if helmReleases.CSI != nil {
var csiVals map[string]any
if cloudprovider.FromString(k.cloudProvider) == cloudprovider.OpenStack {
creds, err := openstack.AccountKeyFromURI(serviceConfig.cloudServiceAccountURI)
if err != nil {
return nil, err
}
cinderIni := creds.CloudINI().CinderCSIConfiguration()
csiVals = map[string]any{
"cinder-config": map[string]any{
"secretData": cinderIni,
},
}
}
log.Infof("Installing CSI deployments")
if err := k.helmClient.InstallChartWithValues(ctx, *helmReleases.CSI, csiVals); err != nil {
return nil, fmt.Errorf("installing CSI snapshot CRDs: %w", err)
}
}
if helmReleases.AWSLoadBalancerController != nil {
log.Infof("Installing AWS Load Balancer Controller")
if err = k.helmClient.InstallChart(ctx, *helmReleases.AWSLoadBalancerController); err != nil {
return nil, fmt.Errorf("installing AWS Load Balancer Controller: %w", err)
}
}
operatorVals, err := k.setupOperatorVals(ctx)
if err != nil {
return nil, fmt.Errorf("setting up operator vals: %w", err)
}
// Constellation operators require CRDs from cert-manager.
// They must be installed after it.
log.Infof("Installing operators")
if err = k.helmClient.InstallChartWithValues(ctx, helmReleases.ConstellationOperators, operatorVals); err != nil {
return nil, fmt.Errorf("installing operators: %w", err)
}
return kubeConfig, nil return kubeConfig, nil
} }
@ -447,120 +379,6 @@ func getIPAddr() (string, error) {
return localAddr.IP.String(), nil return localAddr.IP.String(), nil
} }
// setupExtraVals create a helm values map for consumption by helm-install.
// Will move to a more dedicated place once that place becomes apparent.
func (k *KubeWrapper) setupExtraVals(ctx context.Context, serviceConfig constellationServicesConfig) (map[string]any, error) {
extraVals := map[string]any{
"join-service": map[string]any{
"measurementSalt": base64.StdEncoding.EncodeToString(serviceConfig.measurementSalt),
},
"verification-service": map[string]any{
"loadBalancerIP": serviceConfig.loadBalancerIP,
},
"konnectivity": map[string]any{
"loadBalancerIP": serviceConfig.loadBalancerIP,
},
}
instance, err := k.providerMetadata.Self(ctx)
if err != nil {
return nil, fmt.Errorf("retrieving current instance: %w", err)
}
switch cloudprovider.FromString(k.cloudProvider) {
case cloudprovider.GCP:
uid, err := k.providerMetadata.UID(ctx)
if err != nil {
return nil, fmt.Errorf("getting uid: %w", err)
}
projectID, _, _, err := gcpshared.SplitProviderID(instance.ProviderID)
if err != nil {
return nil, fmt.Errorf("splitting providerID: %w", err)
}
serviceAccountKey, err := gcpshared.ServiceAccountKeyFromURI(serviceConfig.cloudServiceAccountURI)
if err != nil {
return nil, fmt.Errorf("getting service account key: %w", err)
}
rawKey, err := json.Marshal(serviceAccountKey)
if err != nil {
return nil, fmt.Errorf("marshaling service account key: %w", err)
}
extraVals["ccm"] = map[string]any{
"GCP": map[string]any{
"projectID": projectID,
"uid": uid,
"secretData": string(rawKey),
"subnetworkPodCIDR": serviceConfig.subnetworkPodCIDR,
},
}
case cloudprovider.Azure:
ccmAzure, ok := k.providerMetadata.(ccmConfigGetter)
if !ok {
return nil, errors.New("invalid cloud provider metadata for Azure")
}
ccmConfig, err := ccmAzure.GetCCMConfig(ctx, instance.ProviderID, serviceConfig.cloudServiceAccountURI)
if err != nil {
return nil, fmt.Errorf("creating ccm secret: %w", err)
}
extraVals["ccm"] = map[string]any{
"Azure": map[string]any{
"azureConfig": string(ccmConfig),
},
}
case cloudprovider.OpenStack:
creds, err := openstack.AccountKeyFromURI(serviceConfig.cloudServiceAccountURI)
if err != nil {
return nil, err
}
credsIni := creds.CloudINI().FullConfiguration()
networkIDsGetter, ok := k.providerMetadata.(openstackMetadata)
if !ok {
return nil, errors.New("generating yawol configuration: cloud provider metadata does not implement OpenStack specific methods")
}
networkIDs, err := networkIDsGetter.GetNetworkIDs(ctx)
if err != nil {
return nil, fmt.Errorf("getting network IDs: %w", err)
}
if len(networkIDs) == 0 {
return nil, errors.New("getting network IDs: no network IDs found")
}
extraVals["ccm"] = map[string]any{
"OpenStack": map[string]any{
"secretData": credsIni,
},
}
yawolIni := creds.CloudINI().YawolConfiguration()
extraVals["yawol-config"] = map[string]any{
"secretData": yawolIni,
}
extraVals["yawol-controller"] = map[string]any{
"yawolNetworkID": networkIDs[0],
"yawolAPIHost": fmt.Sprintf("https://%s:%d", serviceConfig.loadBalancerIP, constants.KubernetesPort),
}
}
return extraVals, nil
}
func (k *KubeWrapper) setupOperatorVals(ctx context.Context) (map[string]any, error) {
uid, err := k.providerMetadata.UID(ctx)
if err != nil {
return nil, fmt.Errorf("retrieving constellation UID: %w", err)
}
return map[string]any{
"constellation-operator": map[string]any{
"constellationUID": uid,
},
}, nil
}
func (k *KubeWrapper) setupCiliumVals(ctx context.Context, in k8sapi.SetupPodNetworkInput) (map[string]any, error) { func (k *KubeWrapper) setupCiliumVals(ctx context.Context, in k8sapi.SetupPodNetworkInput) (map[string]any, error) {
vals := map[string]any{ vals := map[string]any{
"k8sServiceHost": in.LoadBalancerHost, "k8sServiceHost": in.LoadBalancerHost,
@ -585,18 +403,3 @@ func (k *KubeWrapper) setupCiliumVals(ctx context.Context, in k8sapi.SetupPodNet
return vals, nil return vals, nil
} }
type ccmConfigGetter interface {
GetCCMConfig(ctx context.Context, providerID, cloudServiceAccountURI string) ([]byte, error)
}
type constellationServicesConfig struct {
measurementSalt []byte
subnetworkPodCIDR string
cloudServiceAccountURI string
loadBalancerIP string
}
type openstackMetadata interface {
GetNetworkIDs(ctx context.Context) ([]string, error)
}

View File

@ -35,8 +35,6 @@ func TestMain(m *testing.M) {
} }
func TestInitCluster(t *testing.T) { func TestInitCluster(t *testing.T) {
serviceAccountURI := "some-service-account-uri"
nodeName := "node-name" nodeName := "node-name"
providerID := "provider-id" providerID := "provider-id"
privateIP := "192.0.2.1" privateIP := "192.0.2.1"
@ -193,8 +191,8 @@ func TestInitCluster(t *testing.T) {
} }
_, err := kube.InitCluster( _, err := kube.InitCluster(
context.Background(), serviceAccountURI, string(tc.k8sVersion), "kubernetes", context.Background(), string(tc.k8sVersion), "kubernetes",
nil, []byte("{}"), false, nil, nil, logger.NewTest(t), []byte("{}"), false, nil, nil, logger.NewTest(t),
) )
if tc.wantErr { if tc.wantErr {

View File

@ -34,7 +34,6 @@ go_library(
"@com_github_azure_azure_sdk_for_go//profiles/latest/attestation/attestation", "@com_github_azure_azure_sdk_for_go//profiles/latest/attestation/attestation",
"@com_github_azure_azure_sdk_for_go_sdk_azcore//policy", "@com_github_azure_azure_sdk_for_go_sdk_azcore//policy",
"@com_github_azure_azure_sdk_for_go_sdk_azidentity//:azidentity", "@com_github_azure_azure_sdk_for_go_sdk_azidentity//:azidentity",
"@com_github_hashicorp_terraform_json//:terraform-json",
"@com_github_spf13_cobra//:cobra", "@com_github_spf13_cobra//:cobra",
], ],
) )
@ -59,7 +58,6 @@ go_test(
"//internal/cloud/cloudprovider", "//internal/cloud/cloudprovider",
"//internal/cloud/gcpshared", "//internal/cloud/gcpshared",
"//internal/config", "//internal/config",
"@com_github_hashicorp_terraform_json//:terraform-json",
"@com_github_stretchr_testify//assert", "@com_github_stretchr_testify//assert",
"@com_github_stretchr_testify//require", "@com_github_stretchr_testify//require",
"@org_uber_go_goleak//:goleak", "@org_uber_go_goleak//:goleak",

View File

@ -13,7 +13,6 @@ import (
"github.com/edgelesssys/constellation/v2/cli/internal/terraform" "github.com/edgelesssys/constellation/v2/cli/internal/terraform"
"github.com/edgelesssys/constellation/v2/internal/attestation/variant" "github.com/edgelesssys/constellation/v2/internal/attestation/variant"
"github.com/edgelesssys/constellation/v2/internal/cloud/cloudprovider" "github.com/edgelesssys/constellation/v2/internal/cloud/cloudprovider"
tfjson "github.com/hashicorp/terraform-json"
) )
// imageFetcher gets an image reference from the versionsapi. // imageFetcher gets an image reference from the versionsapi.
@ -24,14 +23,23 @@ type imageFetcher interface {
) (string, error) ) (string, error)
} }
type terraformClient interface { type tfCommonClient interface {
PrepareWorkspace(path string, input terraform.Variables) error
ApplyIAMConfig(ctx context.Context, provider cloudprovider.Provider, logLevel terraform.LogLevel) (terraform.IAMOutput, error)
CreateCluster(ctx context.Context, logLevel terraform.LogLevel) (terraform.ApplyOutput, error)
Destroy(ctx context.Context, logLevel terraform.LogLevel) error
CleanUpWorkspace() error CleanUpWorkspace() error
Destroy(ctx context.Context, logLevel terraform.LogLevel) error
PrepareWorkspace(path string, input terraform.Variables) error
RemoveInstaller() RemoveInstaller()
Show(ctx context.Context) (*tfjson.State, error) }
type tfResourceClient interface {
tfCommonClient
CreateCluster(ctx context.Context, provider cloudprovider.Provider, logLevel terraform.LogLevel) (terraform.ApplyOutput, error)
ShowCluster(ctx context.Context, provider cloudprovider.Provider) (terraform.ApplyOutput, error)
}
type tfIAMClient interface {
tfCommonClient
ApplyIAMConfig(ctx context.Context, provider cloudprovider.Provider, logLevel terraform.LogLevel) (terraform.IAMOutput, error)
ShowIAM(ctx context.Context, provider cloudprovider.Provider) (terraform.IAMOutput, error)
} }
type libvirtRunner interface { type libvirtRunner interface {

View File

@ -14,7 +14,6 @@ import (
"github.com/edgelesssys/constellation/v2/cli/internal/terraform" "github.com/edgelesssys/constellation/v2/cli/internal/terraform"
"github.com/edgelesssys/constellation/v2/internal/attestation/variant" "github.com/edgelesssys/constellation/v2/internal/attestation/variant"
"github.com/edgelesssys/constellation/v2/internal/cloud/cloudprovider" "github.com/edgelesssys/constellation/v2/internal/cloud/cloudprovider"
tfjson "github.com/hashicorp/terraform-json"
"go.uber.org/goleak" "go.uber.org/goleak"
) )
@ -32,7 +31,7 @@ type stubTerraformClient struct {
iamOutput terraform.IAMOutput iamOutput terraform.IAMOutput
uid string uid string
attestationURL string attestationURL string
tfjsonState *tfjson.State applyOutput terraform.ApplyOutput
cleanUpWorkspaceCalled bool cleanUpWorkspaceCalled bool
removeInstallerCalled bool removeInstallerCalled bool
destroyCalled bool destroyCalled bool
@ -45,12 +44,14 @@ type stubTerraformClient struct {
showErr error showErr error
} }
func (c *stubTerraformClient) CreateCluster(_ context.Context, _ terraform.LogLevel) (terraform.ApplyOutput, error) { func (c *stubTerraformClient) CreateCluster(_ context.Context, _ cloudprovider.Provider, _ terraform.LogLevel) (terraform.ApplyOutput, error) {
return terraform.ApplyOutput{ return terraform.ApplyOutput{
IP: c.ip, IP: c.ip,
Secret: c.initSecret, Secret: c.initSecret,
UID: c.uid, UID: c.uid,
Azure: &terraform.AzureApplyOutput{
AttestationURL: c.attestationURL, AttestationURL: c.attestationURL,
},
}, c.createClusterErr }, c.createClusterErr
} }
@ -76,9 +77,14 @@ func (c *stubTerraformClient) RemoveInstaller() {
c.removeInstallerCalled = true c.removeInstallerCalled = true
} }
func (c *stubTerraformClient) Show(_ context.Context) (*tfjson.State, error) { func (c *stubTerraformClient) ShowCluster(_ context.Context, _ cloudprovider.Provider) (terraform.ApplyOutput, error) {
c.showCalled = true c.showCalled = true
return c.tfjsonState, c.showErr return c.applyOutput, c.showErr
}
func (c *stubTerraformClient) ShowIAM(_ context.Context, _ cloudprovider.Provider) (terraform.IAMOutput, error) {
c.showCalled = true
return c.iamOutput, c.showErr
} }
type stubLibvirtRunner struct { type stubLibvirtRunner struct {

View File

@ -31,7 +31,7 @@ import (
type Creator struct { type Creator struct {
out io.Writer out io.Writer
image imageFetcher image imageFetcher
newTerraformClient func(ctx context.Context) (terraformClient, error) newTerraformClient func(ctx context.Context) (tfResourceClient, error)
newLibvirtRunner func() libvirtRunner newLibvirtRunner func() libvirtRunner
newRawDownloader func() rawDownloader newRawDownloader func() rawDownloader
policyPatcher policyPatcher policyPatcher policyPatcher
@ -42,7 +42,7 @@ func NewCreator(out io.Writer) *Creator {
return &Creator{ return &Creator{
out: out, out: out,
image: imagefetcher.New(), image: imagefetcher.New(),
newTerraformClient: func(ctx context.Context) (terraformClient, error) { newTerraformClient: func(ctx context.Context) (tfResourceClient, error) {
return terraform.New(ctx, constants.TerraformWorkingDir) return terraform.New(ctx, constants.TerraformWorkingDir)
}, },
newLibvirtRunner: func() libvirtRunner { newLibvirtRunner: func() libvirtRunner {
@ -115,18 +115,20 @@ func (c *Creator) Create(ctx context.Context, opts CreateOptions) (clusterid.Fil
if err != nil { if err != nil {
return clusterid.File{}, fmt.Errorf("creating cluster: %w", err) return clusterid.File{}, fmt.Errorf("creating cluster: %w", err)
} }
res := clusterid.File{
return clusterid.File{
CloudProvider: opts.Provider, CloudProvider: opts.Provider,
IP: tfOutput.IP, IP: tfOutput.IP,
APIServerCertSANs: tfOutput.APIServerCertSANs, APIServerCertSANs: tfOutput.APIServerCertSANs,
InitSecret: []byte(tfOutput.Secret), InitSecret: []byte(tfOutput.Secret),
UID: tfOutput.UID, UID: tfOutput.UID,
AttestationURL: tfOutput.AttestationURL, }
}, nil if tfOutput.Azure != nil {
res.AttestationURL = tfOutput.Azure.AttestationURL
}
return res, nil
} }
func (c *Creator) createAWS(ctx context.Context, cl terraformClient, opts CreateOptions) (tfOutput terraform.ApplyOutput, retErr error) { func (c *Creator) createAWS(ctx context.Context, cl tfResourceClient, opts CreateOptions) (tfOutput terraform.ApplyOutput, retErr error) {
vars := awsTerraformVars(opts.Config, opts.image, &opts.ControlPlaneCount, &opts.WorkerCount) vars := awsTerraformVars(opts.Config, opts.image, &opts.ControlPlaneCount, &opts.WorkerCount)
tfOutput, err := runTerraformCreate(ctx, cl, cloudprovider.AWS, vars, c.out, opts.TFLogLevel) tfOutput, err := runTerraformCreate(ctx, cl, cloudprovider.AWS, vars, c.out, opts.TFLogLevel)
@ -137,7 +139,7 @@ func (c *Creator) createAWS(ctx context.Context, cl terraformClient, opts Create
return tfOutput, nil return tfOutput, nil
} }
func (c *Creator) createGCP(ctx context.Context, cl terraformClient, opts CreateOptions) (tfOutput terraform.ApplyOutput, retErr error) { func (c *Creator) createGCP(ctx context.Context, cl tfResourceClient, opts CreateOptions) (tfOutput terraform.ApplyOutput, retErr error) {
vars := gcpTerraformVars(opts.Config, opts.image, &opts.ControlPlaneCount, &opts.WorkerCount) vars := gcpTerraformVars(opts.Config, opts.image, &opts.ControlPlaneCount, &opts.WorkerCount)
tfOutput, err := runTerraformCreate(ctx, cl, cloudprovider.GCP, vars, c.out, opts.TFLogLevel) tfOutput, err := runTerraformCreate(ctx, cl, cloudprovider.GCP, vars, c.out, opts.TFLogLevel)
@ -148,7 +150,7 @@ func (c *Creator) createGCP(ctx context.Context, cl terraformClient, opts Create
return tfOutput, nil return tfOutput, nil
} }
func (c *Creator) createAzure(ctx context.Context, cl terraformClient, opts CreateOptions) (tfOutput terraform.ApplyOutput, retErr error) { func (c *Creator) createAzure(ctx context.Context, cl tfResourceClient, opts CreateOptions) (tfOutput terraform.ApplyOutput, retErr error) {
vars := azureTerraformVars(opts.Config, opts.image, &opts.ControlPlaneCount, &opts.WorkerCount) vars := azureTerraformVars(opts.Config, opts.image, &opts.ControlPlaneCount, &opts.WorkerCount)
tfOutput, err := runTerraformCreate(ctx, cl, cloudprovider.Azure, vars, c.out, opts.TFLogLevel) tfOutput, err := runTerraformCreate(ctx, cl, cloudprovider.Azure, vars, c.out, opts.TFLogLevel)
@ -158,7 +160,10 @@ func (c *Creator) createAzure(ctx context.Context, cl terraformClient, opts Crea
if vars.GetCreateMAA() { if vars.GetCreateMAA() {
// Patch the attestation policy to allow the cluster to boot while having secure boot disabled. // Patch the attestation policy to allow the cluster to boot while having secure boot disabled.
if err := c.policyPatcher.Patch(ctx, tfOutput.AttestationURL); err != nil { if tfOutput.Azure == nil {
return terraform.ApplyOutput{}, errors.New("no Terraform Azure output found")
}
if err := c.policyPatcher.Patch(ctx, tfOutput.Azure.AttestationURL); err != nil {
return terraform.ApplyOutput{}, err return terraform.ApplyOutput{}, err
} }
} }
@ -197,7 +202,7 @@ func normalizeAzureURIs(vars *terraform.AzureClusterVariables) *terraform.AzureC
return vars return vars
} }
func (c *Creator) createOpenStack(ctx context.Context, cl terraformClient, opts CreateOptions) (tfOutput terraform.ApplyOutput, retErr error) { func (c *Creator) createOpenStack(ctx context.Context, cl tfResourceClient, opts CreateOptions) (tfOutput terraform.ApplyOutput, retErr error) {
// TODO(malt3): Remove this once OpenStack is supported. // TODO(malt3): Remove this once OpenStack is supported.
if os.Getenv("CONSTELLATION_OPENSTACK_DEV") != "1" { if os.Getenv("CONSTELLATION_OPENSTACK_DEV") != "1" {
return terraform.ApplyOutput{}, errors.New("OpenStack isn't supported yet") return terraform.ApplyOutput{}, errors.New("OpenStack isn't supported yet")
@ -221,13 +226,13 @@ func (c *Creator) createOpenStack(ctx context.Context, cl terraformClient, opts
return tfOutput, nil return tfOutput, nil
} }
func runTerraformCreate(ctx context.Context, cl terraformClient, provider cloudprovider.Provider, vars terraform.Variables, outWriter io.Writer, loglevel terraform.LogLevel) (output terraform.ApplyOutput, retErr error) { func runTerraformCreate(ctx context.Context, cl tfResourceClient, provider cloudprovider.Provider, vars terraform.Variables, outWriter io.Writer, loglevel terraform.LogLevel) (output terraform.ApplyOutput, retErr error) {
if err := cl.PrepareWorkspace(path.Join("terraform", strings.ToLower(provider.String())), vars); err != nil { if err := cl.PrepareWorkspace(path.Join("terraform", strings.ToLower(provider.String())), vars); err != nil {
return terraform.ApplyOutput{}, err return terraform.ApplyOutput{}, err
} }
defer rollbackOnError(outWriter, &retErr, &rollbackerTerraform{client: cl}, loglevel) defer rollbackOnError(outWriter, &retErr, &rollbackerTerraform{client: cl}, loglevel)
tfOutput, err := cl.CreateCluster(ctx, loglevel) tfOutput, err := cl.CreateCluster(ctx, provider, loglevel)
if err != nil { if err != nil {
return terraform.ApplyOutput{}, err return terraform.ApplyOutput{}, err
} }
@ -240,7 +245,7 @@ type qemuCreateOptions struct {
CreateOptions CreateOptions
} }
func (c *Creator) createQEMU(ctx context.Context, cl terraformClient, lv libvirtRunner, opts qemuCreateOptions) (tfOutput terraform.ApplyOutput, retErr error) { func (c *Creator) createQEMU(ctx context.Context, cl tfResourceClient, lv libvirtRunner, opts qemuCreateOptions) (tfOutput terraform.ApplyOutput, retErr error) {
qemuRollbacker := &rollbackerQEMU{client: cl, libvirt: lv, createdWorkspace: false} qemuRollbacker := &rollbackerQEMU{client: cl, libvirt: lv, createdWorkspace: false}
defer rollbackOnError(c.out, &retErr, qemuRollbacker, opts.TFLogLevel) defer rollbackOnError(c.out, &retErr, qemuRollbacker, opts.TFLogLevel)
@ -300,7 +305,7 @@ func (c *Creator) createQEMU(ctx context.Context, cl terraformClient, lv libvirt
// Allow rollback of QEMU Terraform workspace from this point on // Allow rollback of QEMU Terraform workspace from this point on
qemuRollbacker.createdWorkspace = true qemuRollbacker.createdWorkspace = true
tfOutput, err = cl.CreateCluster(ctx, opts.TFLogLevel) tfOutput, err = cl.CreateCluster(ctx, opts.Provider, opts.TFLogLevel)
if err != nil { if err != nil {
return terraform.ApplyOutput{}, fmt.Errorf("create cluster: %w", err) return terraform.ApplyOutput{}, fmt.Errorf("create cluster: %w", err)
} }

View File

@ -28,7 +28,7 @@ func TestCreator(t *testing.T) {
someErr := errors.New("failed") someErr := errors.New("failed")
testCases := map[string]struct { testCases := map[string]struct {
tfClient terraformClient tfClient tfResourceClient
newTfClientErr error newTfClientErr error
libvirt *stubLibvirtRunner libvirt *stubLibvirtRunner
provider cloudprovider.Provider provider cloudprovider.Provider
@ -203,7 +203,7 @@ func TestCreator(t *testing.T) {
image: &stubImageFetcher{ image: &stubImageFetcher{
reference: "some-image", reference: "some-image",
}, },
newTerraformClient: func(ctx context.Context) (terraformClient, error) { newTerraformClient: func(ctx context.Context) (tfResourceClient, error) {
return tc.tfClient, tc.newTfClientErr return tc.tfClient, tc.newTfClientErr
}, },
newLibvirtRunner: func() libvirtRunner { newLibvirtRunner: func() libvirtRunner {

View File

@ -24,7 +24,7 @@ import (
// IAMDestroyer destroys an IAM configuration. // IAMDestroyer destroys an IAM configuration.
type IAMDestroyer struct { type IAMDestroyer struct {
client terraformClient client tfIAMClient
} }
// NewIAMDestroyer creates a new IAM Destroyer. // NewIAMDestroyer creates a new IAM Destroyer.
@ -38,29 +38,15 @@ func NewIAMDestroyer(ctx context.Context) (*IAMDestroyer, error) {
// GetTfstateServiceAccountKey returns the sa_key output from the terraform state. // GetTfstateServiceAccountKey returns the sa_key output from the terraform state.
func (d *IAMDestroyer) GetTfstateServiceAccountKey(ctx context.Context) (gcpshared.ServiceAccountKey, error) { func (d *IAMDestroyer) GetTfstateServiceAccountKey(ctx context.Context) (gcpshared.ServiceAccountKey, error) {
tfState, err := d.client.Show(ctx) tfState, err := d.client.ShowIAM(ctx, cloudprovider.GCP)
if err != nil { if err != nil {
return gcpshared.ServiceAccountKey{}, err return gcpshared.ServiceAccountKey{}, fmt.Errorf("getting terraform state: %w", err)
}
if tfState.Values == nil {
return gcpshared.ServiceAccountKey{}, errors.New("no Values field in terraform state")
}
saKeyJSON := tfState.Values.Outputs["sa_key"]
if saKeyJSON == nil {
return gcpshared.ServiceAccountKey{}, errors.New("no sa_key in outputs of the terraform state")
}
saKeyString, ok := saKeyJSON.Value.(string)
if !ok {
return gcpshared.ServiceAccountKey{}, errors.New("sa_key field in terraform state is not a string")
} }
if saKeyString := tfState.GCP.SaKey; saKeyString != "" {
saKey, err := base64.StdEncoding.DecodeString(saKeyString) saKey, err := base64.StdEncoding.DecodeString(saKeyString)
if err != nil { if err != nil {
return gcpshared.ServiceAccountKey{}, err return gcpshared.ServiceAccountKey{}, err
} }
var tfSaKey gcpshared.ServiceAccountKey var tfSaKey gcpshared.ServiceAccountKey
if err := json.Unmarshal(saKey, &tfSaKey); err != nil { if err := json.Unmarshal(saKey, &tfSaKey); err != nil {
return gcpshared.ServiceAccountKey{}, err return gcpshared.ServiceAccountKey{}, err
@ -68,6 +54,8 @@ func (d *IAMDestroyer) GetTfstateServiceAccountKey(ctx context.Context) (gcpshar
return tfSaKey, nil return tfSaKey, nil
} }
return gcpshared.ServiceAccountKey{}, errors.New("no saKey in terraform state")
}
// DestroyIAMConfiguration destroys the previously created IAM configuration and deletes the local IAM terraform files. // DestroyIAMConfiguration destroys the previously created IAM configuration and deletes the local IAM terraform files.
func (d *IAMDestroyer) DestroyIAMConfiguration(ctx context.Context, logLevel terraform.LogLevel) error { func (d *IAMDestroyer) DestroyIAMConfiguration(ctx context.Context, logLevel terraform.LogLevel) error {
@ -80,7 +68,7 @@ func (d *IAMDestroyer) DestroyIAMConfiguration(ctx context.Context, logLevel ter
// IAMCreator creates the IAM configuration on the cloud provider. // IAMCreator creates the IAM configuration on the cloud provider.
type IAMCreator struct { type IAMCreator struct {
out io.Writer out io.Writer
newTerraformClient func(ctx context.Context) (terraformClient, error) newTerraformClient func(ctx context.Context) (tfIAMClient, error)
} }
// IAMConfigOptions holds the necessary values for IAM configuration. // IAMConfigOptions holds the necessary values for IAM configuration.
@ -116,7 +104,7 @@ type AWSIAMConfig struct {
func NewIAMCreator(out io.Writer) *IAMCreator { func NewIAMCreator(out io.Writer) *IAMCreator {
return &IAMCreator{ return &IAMCreator{
out: out, out: out,
newTerraformClient: func(ctx context.Context) (terraformClient, error) { newTerraformClient: func(ctx context.Context) (tfIAMClient, error) {
return terraform.New(ctx, constants.TerraformIAMWorkingDir) return terraform.New(ctx, constants.TerraformIAMWorkingDir)
}, },
} }
@ -152,7 +140,7 @@ func (c *IAMCreator) Create(ctx context.Context, provider cloudprovider.Provider
} }
// createGCP creates the IAM configuration on GCP. // createGCP creates the IAM configuration on GCP.
func (c *IAMCreator) createGCP(ctx context.Context, cl terraformClient, opts *IAMConfigOptions) (retFile iamid.File, retErr error) { func (c *IAMCreator) createGCP(ctx context.Context, cl tfIAMClient, opts *IAMConfigOptions) (retFile iamid.File, retErr error) {
defer rollbackOnError(c.out, &retErr, &rollbackerTerraform{client: cl}, opts.TFLogLevel) defer rollbackOnError(c.out, &retErr, &rollbackerTerraform{client: cl}, opts.TFLogLevel)
vars := terraform.GCPIAMVariables{ vars := terraform.GCPIAMVariables{
@ -180,7 +168,7 @@ func (c *IAMCreator) createGCP(ctx context.Context, cl terraformClient, opts *IA
} }
// createAzure creates the IAM configuration on Azure. // createAzure creates the IAM configuration on Azure.
func (c *IAMCreator) createAzure(ctx context.Context, cl terraformClient, opts *IAMConfigOptions) (retFile iamid.File, retErr error) { func (c *IAMCreator) createAzure(ctx context.Context, cl tfIAMClient, opts *IAMConfigOptions) (retFile iamid.File, retErr error) {
defer rollbackOnError(c.out, &retErr, &rollbackerTerraform{client: cl}, opts.TFLogLevel) defer rollbackOnError(c.out, &retErr, &rollbackerTerraform{client: cl}, opts.TFLogLevel)
vars := terraform.AzureIAMVariables{ vars := terraform.AzureIAMVariables{
@ -209,7 +197,7 @@ func (c *IAMCreator) createAzure(ctx context.Context, cl terraformClient, opts *
} }
// createAWS creates the IAM configuration on AWS. // createAWS creates the IAM configuration on AWS.
func (c *IAMCreator) createAWS(ctx context.Context, cl terraformClient, opts *IAMConfigOptions) (retFile iamid.File, retErr error) { func (c *IAMCreator) createAWS(ctx context.Context, cl tfIAMClient, opts *IAMConfigOptions) (retFile iamid.File, retErr error) {
defer rollbackOnError(c.out, &retErr, &rollbackerTerraform{client: cl}, opts.TFLogLevel) defer rollbackOnError(c.out, &retErr, &rollbackerTerraform{client: cl}, opts.TFLogLevel)
vars := terraform.AWSIAMVariables{ vars := terraform.AWSIAMVariables{

View File

@ -17,7 +17,6 @@ import (
"github.com/edgelesssys/constellation/v2/cli/internal/terraform" "github.com/edgelesssys/constellation/v2/cli/internal/terraform"
"github.com/edgelesssys/constellation/v2/internal/cloud/cloudprovider" "github.com/edgelesssys/constellation/v2/internal/cloud/cloudprovider"
"github.com/edgelesssys/constellation/v2/internal/cloud/gcpshared" "github.com/edgelesssys/constellation/v2/internal/cloud/gcpshared"
tfjson "github.com/hashicorp/terraform-json"
"github.com/stretchr/testify/assert" "github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require" "github.com/stretchr/testify/require"
) )
@ -83,7 +82,7 @@ func TestIAMCreator(t *testing.T) {
} }
testCases := map[string]struct { testCases := map[string]struct {
tfClient terraformClient tfClient tfIAMClient
newTfClientErr error newTfClientErr error
config *IAMConfigOptions config *IAMConfigOptions
provider cloudprovider.Provider provider cloudprovider.Provider
@ -125,7 +124,7 @@ func TestIAMCreator(t *testing.T) {
creator := &IAMCreator{ creator := &IAMCreator{
out: &bytes.Buffer{}, out: &bytes.Buffer{},
newTerraformClient: func(ctx context.Context) (terraformClient, error) { newTerraformClient: func(ctx context.Context) (tfIAMClient, error) {
return tc.tfClient, tc.newTfClientErr return tc.tfClient, tc.newTfClientErr
}, },
} }
@ -225,13 +224,9 @@ func TestGetTfstateServiceAccountKey(t *testing.T) {
}{ }{
"valid": { "valid": {
cl: &stubTerraformClient{ cl: &stubTerraformClient{
tfjsonState: &tfjson.State{ iamOutput: terraform.IAMOutput{
Values: &tfjson.StateValues{ GCP: terraform.GCPIAMOutput{
Outputs: map[string]*tfjson.StateOutput{ SaKey: gcpFileB64,
"sa_key": {
Value: gcpFileB64,
},
},
}, },
}, },
}, },
@ -247,31 +242,16 @@ func TestGetTfstateServiceAccountKey(t *testing.T) {
}, },
"nil tfstate values": { "nil tfstate values": {
cl: &stubTerraformClient{ cl: &stubTerraformClient{
tfjsonState: &tfjson.State{ iamOutput: terraform.IAMOutput{},
Values: nil,
},
},
wantErr: true,
wantShowCalled: true,
},
"no key": {
cl: &stubTerraformClient{
tfjsonState: &tfjson.State{
Values: &tfjson.StateValues{},
},
}, },
wantErr: true, wantErr: true,
wantShowCalled: true, wantShowCalled: true,
}, },
"invalid base64": { "invalid base64": {
cl: &stubTerraformClient{ cl: &stubTerraformClient{
tfjsonState: &tfjson.State{ iamOutput: terraform.IAMOutput{
Values: &tfjson.StateValues{ GCP: terraform.GCPIAMOutput{
Outputs: map[string]*tfjson.StateOutput{ SaKey: "iamnotvalid",
"sa_key": {
Value: "iamnotvalid",
},
},
}, },
}, },
}, },
@ -280,28 +260,9 @@ func TestGetTfstateServiceAccountKey(t *testing.T) {
}, },
"valid base64 invalid json": { "valid base64 invalid json": {
cl: &stubTerraformClient{ cl: &stubTerraformClient{
tfjsonState: &tfjson.State{ iamOutput: terraform.IAMOutput{
Values: &tfjson.StateValues{ GCP: terraform.GCPIAMOutput{
Outputs: map[string]*tfjson.StateOutput{ SaKey: base64.StdEncoding.EncodeToString([]byte("asdf")),
"sa_key": {
Value: base64.StdEncoding.EncodeToString([]byte("asdf")),
},
},
},
},
},
wantErr: true,
wantShowCalled: true,
},
"not string": {
cl: &stubTerraformClient{
tfjsonState: &tfjson.State{
Values: &tfjson.StateValues{
Outputs: map[string]*tfjson.StateOutput{
"sa_key": {
Value: 1,
},
},
}, },
}, },
}, },

View File

@ -36,7 +36,7 @@ func rollbackOnError(w io.Writer, onErr *error, roll rollbacker, logLevel terraf
} }
type rollbackerTerraform struct { type rollbackerTerraform struct {
client terraformClient client tfCommonClient
} }
func (r *rollbackerTerraform) rollback(ctx context.Context, logLevel terraform.LogLevel) error { func (r *rollbackerTerraform) rollback(ctx context.Context, logLevel terraform.LogLevel) error {
@ -47,7 +47,7 @@ func (r *rollbackerTerraform) rollback(ctx context.Context, logLevel terraform.L
} }
type rollbackerQEMU struct { type rollbackerQEMU struct {
client terraformClient client tfResourceClient
libvirt libvirtRunner libvirt libvirtRunner
createdWorkspace bool createdWorkspace bool
} }

View File

@ -16,14 +16,14 @@ import (
// Terminator deletes cloud provider resources. // Terminator deletes cloud provider resources.
type Terminator struct { type Terminator struct {
newTerraformClient func(ctx context.Context) (terraformClient, error) newTerraformClient func(ctx context.Context) (tfResourceClient, error)
newLibvirtRunner func() libvirtRunner newLibvirtRunner func() libvirtRunner
} }
// NewTerminator create a new cloud terminator. // NewTerminator create a new cloud terminator.
func NewTerminator() *Terminator { func NewTerminator() *Terminator {
return &Terminator{ return &Terminator{
newTerraformClient: func(ctx context.Context) (terraformClient, error) { newTerraformClient: func(ctx context.Context) (tfResourceClient, error) {
return terraform.New(ctx, constants.TerraformWorkingDir) return terraform.New(ctx, constants.TerraformWorkingDir)
}, },
newLibvirtRunner: func() libvirtRunner { newLibvirtRunner: func() libvirtRunner {
@ -49,7 +49,7 @@ func (t *Terminator) Terminate(ctx context.Context, logLevel terraform.LogLevel)
return t.terminateTerraform(ctx, cl, logLevel) return t.terminateTerraform(ctx, cl, logLevel)
} }
func (t *Terminator) terminateTerraform(ctx context.Context, cl terraformClient, logLevel terraform.LogLevel) error { func (t *Terminator) terminateTerraform(ctx context.Context, cl tfResourceClient, logLevel terraform.LogLevel) error {
if err := cl.Destroy(ctx, logLevel); err != nil { if err := cl.Destroy(ctx, logLevel); err != nil {
return err return err
} }

View File

@ -19,7 +19,7 @@ func TestTerminator(t *testing.T) {
someErr := errors.New("failed") someErr := errors.New("failed")
testCases := map[string]struct { testCases := map[string]struct {
tfClient terraformClient tfClient tfResourceClient
newTfClientErr error newTfClientErr error
libvirt *stubLibvirtRunner libvirt *stubLibvirtRunner
wantErr bool wantErr bool
@ -55,7 +55,7 @@ func TestTerminator(t *testing.T) {
assert := assert.New(t) assert := assert.New(t)
terminator := &Terminator{ terminator := &Terminator{
newTerraformClient: func(ctx context.Context) (terraformClient, error) { newTerraformClient: func(ctx context.Context) (tfResourceClient, error) {
return tc.tfClient, tc.newTfClientErr return tc.tfClient, tc.newTfClientErr
}, },
newLibvirtRunner: func() libvirtRunner { newLibvirtRunner: func() libvirtRunner {

View File

@ -153,6 +153,7 @@ go_test(
"//internal/config", "//internal/config",
"//internal/constants", "//internal/constants",
"//internal/crypto/testvector", "//internal/crypto/testvector",
"//internal/deploy/helm",
"//internal/file", "//internal/file",
"//internal/grpc/atlscredentials", "//internal/grpc/atlscredentials",
"//internal/grpc/dialer", "//internal/grpc/dialer",

View File

@ -7,8 +7,10 @@ SPDX-License-Identifier: AGPL-3.0-only
package cmd package cmd
import ( import (
"bytes"
"context" "context"
"encoding/hex" "encoding/hex"
"encoding/json"
"errors" "errors"
"fmt" "fmt"
"io" "io"
@ -77,6 +79,7 @@ type initCmd struct {
spinner spinnerInterf spinner spinnerInterf
masterSecret uri.MasterSecret masterSecret uri.MasterSecret
fh *file.Handler fh *file.Handler
helmInstaller helm.SuiteInstaller
} }
// runInitialize runs the initialize command. // runInitialize runs the initialize command.
@ -100,7 +103,11 @@ func runInitialize(cmd *cobra.Command, _ []string) error {
ctx, cancel := context.WithTimeout(cmd.Context(), time.Hour) ctx, cancel := context.WithTimeout(cmd.Context(), time.Hour)
defer cancel() defer cancel()
cmd.SetContext(ctx) cmd.SetContext(ctx)
i := &initCmd{log: log, spinner: spinner, merger: &kubeconfigMerger{log: log}, fh: &fileHandler} helmInstaller, err := helm.NewInstallationClient(log)
if err != nil {
return fmt.Errorf("creating Helm installer: %w", err)
}
i := &initCmd{log: log, spinner: spinner, merger: &kubeconfigMerger{log: log}, fh: &fileHandler, helmInstaller: helmInstaller}
fetcher := attestationconfigapi.NewFetcher() fetcher := attestationconfigapi.NewFetcher()
return i.initialize(cmd, newDialer, fileHandler, license.NewClient(), fetcher) return i.initialize(cmd, newDialer, fileHandler, license.NewClient(), fetcher)
} }
@ -180,7 +187,14 @@ func (i *initCmd) initialize(cmd *cobra.Command, newDialer func(validator atls.V
helmLoader := helm.NewLoader(provider, k8sVersion, clusterName) helmLoader := helm.NewLoader(provider, k8sVersion, clusterName)
i.log.Debugf("Created new Helm loader") i.log.Debugf("Created new Helm loader")
helmDeployments, err := helmLoader.Load(conf, flags.conformance, flags.helmWaitMode, masterSecret.Key, masterSecret.Salt) releases, err := helmLoader.LoadReleases(conf, flags.conformance, flags.helmWaitMode, masterSecret.Key, masterSecret.Salt)
if err != nil {
return fmt.Errorf("loading Helm charts: %w", err)
}
helmDeployments, err := json.Marshal(releases)
if err != nil {
return err
}
i.log.Debugf("Loaded Helm deployments") i.log.Debugf("Loaded Helm deployments")
if err != nil { if err != nil {
return fmt.Errorf("loading Helm charts: %w", err) return fmt.Errorf("loading Helm charts: %w", err)
@ -214,10 +228,20 @@ func (i *initCmd) initialize(cmd *cobra.Command, newDialer func(validator atls.V
return err return err
} }
i.log.Debugf("Initialization request succeeded") i.log.Debugf("Initialization request succeeded")
i.log.Debugf("Writing Constellation ID file") i.log.Debugf("Writing Constellation ID file")
idFile.CloudProvider = provider idFile.CloudProvider = provider
return i.writeOutput(idFile, resp, flags.mergeConfigs, cmd.OutOrStdout(), fileHandler) bufferedOutput := &bytes.Buffer{}
err = i.writeOutput(idFile, resp, flags.mergeConfigs, bufferedOutput, fileHandler)
if err != nil {
return err
}
if err := i.helmInstaller.Install(cmd.Context(), provider, masterSecret, idFile, serviceAccURI, releases); err != nil {
return fmt.Errorf("installing Helm charts: %w", err)
}
cmd.Println(bufferedOutput.String())
return nil
} }
func (i *initCmd) initCall(ctx context.Context, dialer grpcDialer, ip string, req *initproto.InitRequest) (*initproto.InitSuccessResponse, error) { func (i *initCmd) initCall(ctx context.Context, dialer grpcDialer, ip string, req *initproto.InitRequest) (*initproto.InitSuccessResponse, error) {

View File

@ -28,6 +28,7 @@ import (
"github.com/edgelesssys/constellation/v2/internal/cloud/gcpshared" "github.com/edgelesssys/constellation/v2/internal/cloud/gcpshared"
"github.com/edgelesssys/constellation/v2/internal/config" "github.com/edgelesssys/constellation/v2/internal/config"
"github.com/edgelesssys/constellation/v2/internal/constants" "github.com/edgelesssys/constellation/v2/internal/constants"
helminstaller "github.com/edgelesssys/constellation/v2/internal/deploy/helm"
"github.com/edgelesssys/constellation/v2/internal/file" "github.com/edgelesssys/constellation/v2/internal/file"
"github.com/edgelesssys/constellation/v2/internal/grpc/atlscredentials" "github.com/edgelesssys/constellation/v2/internal/grpc/atlscredentials"
"github.com/edgelesssys/constellation/v2/internal/grpc/dialer" "github.com/edgelesssys/constellation/v2/internal/grpc/dialer"
@ -174,7 +175,7 @@ func TestInitialize(t *testing.T) {
ctx, cancel := context.WithTimeout(ctx, 4*time.Second) ctx, cancel := context.WithTimeout(ctx, 4*time.Second)
defer cancel() defer cancel()
cmd.SetContext(ctx) cmd.SetContext(ctx)
i := &initCmd{log: logger.NewTest(t), spinner: &nopSpinner{}} i := &initCmd{log: logger.NewTest(t), spinner: &nopSpinner{}, helmInstaller: &stubHelmInstaller{}}
err := i.initialize(cmd, newDialer, fileHandler, &stubLicenseClient{}, stubAttestationFetcher{}) err := i.initialize(cmd, newDialer, fileHandler, &stubLicenseClient{}, stubAttestationFetcher{})
if tc.wantErr { if tc.wantErr {
@ -666,3 +667,12 @@ func (c stubInitClient) Recv() (*initproto.InitResponse, error) {
return res, err return res, err
} }
type stubHelmInstaller struct{}
func (i *stubHelmInstaller) Install(_ context.Context, _ cloudprovider.Provider, _ uri.MasterSecret,
_ clusterid.File,
_ string, _ *helminstaller.Releases,
) error {
return nil
}

View File

@ -38,7 +38,7 @@ func (u *tfMigrationClient) planMigration(cmd *cobra.Command, file file.Handler,
func (u *tfMigrationClient) applyMigration(cmd *cobra.Command, file file.Handler, migrateCmd upgrade.TfMigrationCmd, yesFlag bool) error { func (u *tfMigrationClient) applyMigration(cmd *cobra.Command, file file.Handler, migrateCmd upgrade.TfMigrationCmd, yesFlag bool) error {
hasDiff, err := u.planMigration(cmd, file, migrateCmd) hasDiff, err := u.planMigration(cmd, file, migrateCmd)
if err != nil { if err != nil {
return fmt.Errorf("planning terraform migrations: %w", err) return err
} }
if hasDiff { if hasDiff {
// If there are any Terraform migrations to apply, ask for confirmation // If there are any Terraform migrations to apply, ask for confirmation

View File

@ -7,8 +7,10 @@ go_library(
"backup.go", "backup.go",
"client.go", "client.go",
"helm.go", "helm.go",
"helminstaller.go",
"loader.go", "loader.go",
"serviceversion.go", "serviceversion.go",
"setup.go",
"values.go", "values.go",
], ],
embedsrcs = [ embedsrcs = [
@ -417,12 +419,17 @@ go_library(
deps = [ deps = [
"//cli/internal/clusterid", "//cli/internal/clusterid",
"//cli/internal/helm/imageversion", "//cli/internal/helm/imageversion",
"//cli/internal/terraform",
"//internal/cloud/azureshared",
"//internal/cloud/cloudprovider", "//internal/cloud/cloudprovider",
"//internal/cloud/gcpshared",
"//internal/cloud/openstack",
"//internal/compatibility", "//internal/compatibility",
"//internal/config", "//internal/config",
"//internal/constants", "//internal/constants",
"//internal/deploy/helm", "//internal/deploy/helm",
"//internal/file", "//internal/file",
"//internal/kms/uri",
"//internal/semver", "//internal/semver",
"//internal/versions", "//internal/versions",
"@com_github_pkg_errors//:errors", "@com_github_pkg_errors//:errors",

View File

@ -0,0 +1,100 @@
/*
Copyright (c) Edgeless Systems GmbH
SPDX-License-Identifier: AGPL-3.0-only
*/
package helm
import (
"context"
"fmt"
"github.com/edgelesssys/constellation/v2/cli/internal/clusterid"
"github.com/edgelesssys/constellation/v2/internal/cloud/cloudprovider"
"github.com/edgelesssys/constellation/v2/internal/cloud/openstack"
"github.com/edgelesssys/constellation/v2/internal/constants"
helminstaller "github.com/edgelesssys/constellation/v2/internal/deploy/helm"
"github.com/edgelesssys/constellation/v2/internal/kms/uri"
)
// SuiteInstaller installs all Helm charts required for a constellation cluster.
type SuiteInstaller interface {
Install(ctx context.Context, provider cloudprovider.Provider, masterSecret uri.MasterSecret,
idFile clusterid.File,
serviceAccURI string, releases *helminstaller.Releases,
) error
}
type helmInstallationClient struct {
log debugLog
installer helmInstaller
}
// NewInstallationClient creates a new Helm installation client to install all Helm charts required for a constellation cluster.
func NewInstallationClient(log debugLog) (SuiteInstaller, error) {
installer, err := helminstaller.NewInstaller(constants.AdminConfFilename, log)
if err != nil {
return nil, fmt.Errorf("creating Helm installer: %w", err)
}
return &helmInstallationClient{log: log, installer: installer}, nil
}
func (h helmInstallationClient) Install(ctx context.Context, provider cloudprovider.Provider, masterSecret uri.MasterSecret,
idFile clusterid.File,
serviceAccURI string, releases *helminstaller.Releases,
) error {
serviceVals, err := setupMicroserviceVals(ctx, provider, masterSecret.Salt, idFile.UID, serviceAccURI)
if err != nil {
return fmt.Errorf("setting up microservice values: %w", err)
}
if err := h.installer.InstallChartWithValues(ctx, releases.ConstellationServices, serviceVals); err != nil {
return fmt.Errorf("installing microservices: %w", err)
}
h.log.Debugf("Installing cert-manager")
if err := h.installer.InstallChart(ctx, releases.CertManager); err != nil {
return fmt.Errorf("installing cert-manager: %w", err)
}
if releases.CSI != nil {
var csiVals map[string]any
if provider == cloudprovider.OpenStack {
creds, err := openstack.AccountKeyFromURI(serviceAccURI)
if err != nil {
return err
}
cinderIni := creds.CloudINI().CinderCSIConfiguration()
csiVals = map[string]any{
"cinder-config": map[string]any{
"secretData": cinderIni,
},
}
}
h.log.Debugf("Installing CSI deployments")
if err := h.installer.InstallChartWithValues(ctx, *releases.CSI, csiVals); err != nil {
return fmt.Errorf("installing CSI snapshot CRDs: %w", err)
}
}
if releases.AWSLoadBalancerController != nil {
h.log.Debugf("Installing AWS Load Balancer Controller")
if err := h.installer.InstallChart(ctx, *releases.AWSLoadBalancerController); err != nil {
return fmt.Errorf("installing AWS Load Balancer Controller: %w", err)
}
}
h.log.Debugf("Installing constellation operators")
operatorVals := setupOperatorVals(ctx, idFile.UID)
if err := h.installer.InstallChartWithValues(ctx, releases.ConstellationOperators, operatorVals); err != nil {
return fmt.Errorf("installing constellation operators: %w", err)
}
// TODO(elchead): AB#3301 do cilium after version upgrade
return nil
}
type helmInstaller interface {
InstallChart(context.Context, helminstaller.Release) error
InstallChartWithValues(ctx context.Context, release helminstaller.Release, extraValues map[string]any) error
}

View File

@ -110,6 +110,19 @@ func NewLoader(csp cloudprovider.Provider, k8sVersion versions.ValidK8sVersion,
// Load the embedded helm charts. // Load the embedded helm charts.
func (i *ChartLoader) Load(config *config.Config, conformanceMode bool, helmWaitMode helm.WaitMode, masterSecret, salt []byte) ([]byte, error) { func (i *ChartLoader) Load(config *config.Config, conformanceMode bool, helmWaitMode helm.WaitMode, masterSecret, salt []byte) ([]byte, error) {
releases, err := i.LoadReleases(config, conformanceMode, helmWaitMode, masterSecret, salt)
if err != nil {
return nil, fmt.Errorf("loading releases: %w", err)
}
rel, err := json.Marshal(releases)
if err != nil {
return nil, err
}
return rel, nil
}
// LoadReleases loads the embedded helm charts and returns them as a HelmReleases object.
func (i *ChartLoader) LoadReleases(config *config.Config, conformanceMode bool, helmWaitMode helm.WaitMode, masterSecret, salt []byte) (*helm.Releases, error) {
ciliumRelease, err := i.loadRelease(ciliumInfo, helmWaitMode) ciliumRelease, err := i.loadRelease(ciliumInfo, helmWaitMode)
if err != nil { if err != nil {
return nil, fmt.Errorf("loading cilium: %w", err) return nil, fmt.Errorf("loading cilium: %w", err)
@ -150,12 +163,7 @@ func (i *ChartLoader) Load(config *config.Config, conformanceMode bool, helmWait
} }
releases.CSI = &csi releases.CSI = &csi
} }
return &releases, nil
rel, err := json.Marshal(releases)
if err != nil {
return nil, err
}
return rel, nil
} }
// loadRelease loads the embedded chart and values depending on the given info argument. // loadRelease loads the embedded chart and values depending on the given info argument.

134
cli/internal/helm/setup.go Normal file
View File

@ -0,0 +1,134 @@
/*
Copyright (c) Edgeless Systems GmbH
SPDX-License-Identifier: AGPL-3.0-only
*/
package helm
import (
"context"
"encoding/base64"
"encoding/json"
"fmt"
"github.com/edgelesssys/constellation/v2/cli/internal/terraform"
"github.com/edgelesssys/constellation/v2/internal/cloud/azureshared"
"github.com/edgelesssys/constellation/v2/internal/cloud/cloudprovider"
"github.com/edgelesssys/constellation/v2/internal/cloud/gcpshared"
"github.com/edgelesssys/constellation/v2/internal/constants"
)
// setupMicroserviceVals returns the values for the microservice chart.
func setupMicroserviceVals(ctx context.Context, provider cloudprovider.Provider, measurementSalt []byte, uid, serviceAccURI string) (map[string]any, error) {
tfClient, err := terraform.New(ctx, constants.TerraformWorkingDir)
if err != nil {
return nil, fmt.Errorf("creating Terraform client: %w", err)
}
output, err := tfClient.ShowCluster(ctx, provider)
if err != nil {
return nil, fmt.Errorf("getting Terraform output: %w", err)
}
extraVals := map[string]any{
"join-service": map[string]any{
"measurementSalt": base64.StdEncoding.EncodeToString(measurementSalt),
},
"verification-service": map[string]any{
"loadBalancerIP": output.IP,
},
"konnectivity": map[string]any{
"loadBalancerIP": output.IP,
},
}
switch provider {
case cloudprovider.GCP:
serviceAccountKey, err := gcpshared.ServiceAccountKeyFromURI(serviceAccURI)
if err != nil {
return nil, fmt.Errorf("getting service account key: %w", err)
}
rawKey, err := json.Marshal(serviceAccountKey)
if err != nil {
return nil, fmt.Errorf("marshaling service account key: %w", err)
}
if output.GCP == nil {
return nil, fmt.Errorf("no GCP output from Terraform")
}
extraVals["ccm"] = map[string]any{
"GCP": map[string]any{
"projectID": output.GCP.ProjectID,
"uid": uid,
"secretData": string(rawKey),
"subnetworkPodCIDR": output.GCP.IPCidrPod,
},
}
case cloudprovider.Azure:
if output.Azure == nil {
return nil, fmt.Errorf("no Azure output from Terraform")
}
ccmConfig, err := getCCMConfig(*output.Azure, serviceAccURI)
if err != nil {
return nil, fmt.Errorf("getting Azure CCM config: %w", err)
}
extraVals["ccm"] = map[string]any{
"Azure": map[string]any{
"azureConfig": string(ccmConfig),
},
}
}
return extraVals, nil
}
// setupOperatorVals returns the values for the constellation-operator chart.
func setupOperatorVals(_ context.Context, uid string) map[string]any {
return map[string]any{
"constellation-operator": map[string]any{
"constellationUID": uid,
},
}
}
type cloudConfig struct {
Cloud string `json:"cloud,omitempty"`
TenantID string `json:"tenantId,omitempty"`
SubscriptionID string `json:"subscriptionId,omitempty"`
ResourceGroup string `json:"resourceGroup,omitempty"`
Location string `json:"location,omitempty"`
SubnetName string `json:"subnetName,omitempty"`
SecurityGroupName string `json:"securityGroupName,omitempty"`
SecurityGroupResourceGroup string `json:"securityGroupResourceGroup,omitempty"`
LoadBalancerName string `json:"loadBalancerName,omitempty"`
LoadBalancerSku string `json:"loadBalancerSku,omitempty"`
VNetName string `json:"vnetName,omitempty"`
VNetResourceGroup string `json:"vnetResourceGroup,omitempty"`
CloudProviderBackoff bool `json:"cloudProviderBackoff,omitempty"`
UseInstanceMetadata bool `json:"useInstanceMetadata,omitempty"`
VMType string `json:"vmType,omitempty"`
UseManagedIdentityExtension bool `json:"useManagedIdentityExtension,omitempty"`
UserAssignedIdentityID string `json:"userAssignedIdentityID,omitempty"`
}
// GetCCMConfig returns the configuration needed for the Kubernetes Cloud Controller Manager on Azure.
func getCCMConfig(tfOutput terraform.AzureApplyOutput, serviceAccURI string) ([]byte, error) {
creds, err := azureshared.ApplicationCredentialsFromURI(serviceAccURI)
if err != nil {
return nil, fmt.Errorf("getting service account key: %w", err)
}
useManagedIdentityExtension := creds.PreferredAuthMethod == azureshared.AuthMethodUserAssignedIdentity
config := cloudConfig{
Cloud: "AzurePublicCloud",
TenantID: creds.TenantID,
SubscriptionID: tfOutput.SubscriptionID,
ResourceGroup: tfOutput.ResourceGroup,
LoadBalancerSku: "standard",
SecurityGroupName: tfOutput.NetworkSecurityGroupName,
LoadBalancerName: tfOutput.LoadBalancerName,
UseInstanceMetadata: true,
VMType: "vmss",
Location: creds.Location,
UseManagedIdentityExtension: useManagedIdentityExtension,
UserAssignedIdentityID: tfOutput.UserAssignedIdentity,
}
return json.Marshal(config)
}

View File

@ -78,172 +78,8 @@ func (c *Client) WithManualStateMigration(migration StateMigration) *Client {
return c return c
} }
// Show reads the default state path and outputs the state. // ShowIAM reads the state of Constellation IAM resources from Terraform.
func (c *Client) Show(ctx context.Context) (*tfjson.State, error) { func (c *Client) ShowIAM(ctx context.Context, provider cloudprovider.Provider) (IAMOutput, error) {
return c.tf.Show(ctx)
}
// PrepareWorkspace prepares a Terraform workspace for a Constellation cluster.
func (c *Client) PrepareWorkspace(path string, vars Variables) error {
if err := prepareWorkspace(path, c.file, c.workingDir); err != nil {
return fmt.Errorf("prepare workspace: %w", err)
}
return c.writeVars(vars)
}
// PrepareUpgradeWorkspace prepares a Terraform workspace for a Constellation version upgrade.
// It copies the Terraform state from the old working dir and the embedded Terraform files into the new working dir.
func (c *Client) PrepareUpgradeWorkspace(path, oldWorkingDir, newWorkingDir, backupDir string, vars Variables) error {
if err := prepareUpgradeWorkspace(path, c.file, oldWorkingDir, newWorkingDir, backupDir); err != nil {
return fmt.Errorf("prepare upgrade workspace: %w", err)
}
return c.writeVars(vars)
}
// PrepareIAMUpgradeWorkspace prepares a Terraform workspace for a Constellation IAM upgrade.
func PrepareIAMUpgradeWorkspace(file file.Handler, path, oldWorkingDir, newWorkingDir, backupDir string) error {
if err := prepareUpgradeWorkspace(path, file, oldWorkingDir, newWorkingDir, backupDir); err != nil {
return fmt.Errorf("prepare upgrade workspace: %w", err)
}
// copy the vars file from the old working dir to the new working dir
if err := file.CopyFile(filepath.Join(oldWorkingDir, terraformVarsFile), filepath.Join(newWorkingDir, terraformVarsFile)); err != nil {
return fmt.Errorf("copying vars file: %w", err)
}
return nil
}
// CreateCluster creates a Constellation cluster using Terraform.
func (c *Client) CreateCluster(ctx context.Context, logLevel LogLevel) (ApplyOutput, error) {
if err := c.setLogLevel(logLevel); err != nil {
return ApplyOutput{}, fmt.Errorf("set terraform log level %s: %w", logLevel.String(), err)
}
if err := c.tf.Init(ctx); err != nil {
return ApplyOutput{}, fmt.Errorf("terraform init: %w", err)
}
if err := c.applyManualStateMigrations(ctx); err != nil {
return ApplyOutput{}, fmt.Errorf("apply manual state migrations: %w", err)
}
if err := c.tf.Apply(ctx); err != nil {
return ApplyOutput{}, fmt.Errorf("terraform apply: %w", err)
}
tfState, err := c.tf.Show(ctx)
if err != nil {
return ApplyOutput{}, fmt.Errorf("terraform show: %w", err)
}
ipOutput, ok := tfState.Values.Outputs["ip"]
if !ok {
return ApplyOutput{}, errors.New("no IP output found")
}
ip, ok := ipOutput.Value.(string)
if !ok {
return ApplyOutput{}, errors.New("invalid type in IP output: not a string")
}
apiServerCertSANsOutput, ok := tfState.Values.Outputs["api_server_cert_sans"]
if !ok {
return ApplyOutput{}, errors.New("no api_server_cert_sans output found")
}
apiServerCertSANsUntyped, ok := apiServerCertSANsOutput.Value.([]any)
if !ok {
return ApplyOutput{}, fmt.Errorf("invalid type in api_server_cert_sans output: %s is not a list of elements", apiServerCertSANsOutput.Type.FriendlyName())
}
apiServerCertSANs, err := toStringSlice(apiServerCertSANsUntyped)
if err != nil {
return ApplyOutput{}, fmt.Errorf("convert api_server_cert_sans output: %w", err)
}
secretOutput, ok := tfState.Values.Outputs["initSecret"]
if !ok {
return ApplyOutput{}, errors.New("no initSecret output found")
}
secret, ok := secretOutput.Value.(string)
if !ok {
return ApplyOutput{}, errors.New("invalid type in initSecret output: not a string")
}
uidOutput, ok := tfState.Values.Outputs["uid"]
if !ok {
return ApplyOutput{}, errors.New("no uid output found")
}
uid, ok := uidOutput.Value.(string)
if !ok {
return ApplyOutput{}, errors.New("invalid type in uid output: not a string")
}
var attestationURL string
if attestationURLOutput, ok := tfState.Values.Outputs["attestationURL"]; ok {
if attestationURLString, ok := attestationURLOutput.Value.(string); ok {
attestationURL = attestationURLString
}
}
return ApplyOutput{
IP: ip,
APIServerCertSANs: apiServerCertSANs,
Secret: secret,
UID: uid,
AttestationURL: attestationURL,
}, nil
}
// ApplyOutput contains the Terraform output values of a cluster creation
// or apply operation.
type ApplyOutput struct {
IP string
APIServerCertSANs []string
Secret string
UID string
// AttestationURL is the URL of the attestation provider.
// It is only set if the cluster is created on Azure.
AttestationURL string
}
// IAMOutput contains the output information of the Terraform IAM operations.
type IAMOutput struct {
GCP GCPIAMOutput
Azure AzureIAMOutput
AWS AWSIAMOutput
}
// GCPIAMOutput contains the output information of the Terraform IAM operation on GCP.
type GCPIAMOutput struct {
SaKey string
}
// AzureIAMOutput contains the output information of the Terraform IAM operation on Microsoft Azure.
type AzureIAMOutput struct {
SubscriptionID string
TenantID string
UAMIID string
}
// AWSIAMOutput contains the output information of the Terraform IAM operation on GCP.
type AWSIAMOutput struct {
ControlPlaneInstanceProfile string
WorkerNodeInstanceProfile string
}
// ApplyIAMConfig creates an IAM configuration using Terraform.
func (c *Client) ApplyIAMConfig(ctx context.Context, provider cloudprovider.Provider, logLevel LogLevel) (IAMOutput, error) {
if err := c.setLogLevel(logLevel); err != nil {
return IAMOutput{}, fmt.Errorf("set terraform log level %s: %w", logLevel.String(), err)
}
if err := c.tf.Init(ctx); err != nil {
return IAMOutput{}, err
}
if err := c.tf.Apply(ctx); err != nil {
return IAMOutput{}, err
}
tfState, err := c.tf.Show(ctx) tfState, err := c.tf.Show(ctx)
if err != nil { if err != nil {
return IAMOutput{}, err return IAMOutput{}, err
@ -324,6 +160,277 @@ func (c *Client) ApplyIAMConfig(ctx context.Context, provider cloudprovider.Prov
} }
} }
// ShowCluster reads the state of Constellation cluster resources from Terraform.
func (c *Client) ShowCluster(ctx context.Context, provider cloudprovider.Provider) (ApplyOutput, error) {
tfState, err := c.tf.Show(ctx)
if err != nil {
return ApplyOutput{}, fmt.Errorf("terraform show: %w", err)
}
ipOutput, ok := tfState.Values.Outputs["ip"]
if !ok {
return ApplyOutput{}, errors.New("no IP output found")
}
ip, ok := ipOutput.Value.(string)
if !ok {
return ApplyOutput{}, errors.New("invalid type in IP output: not a string")
}
apiServerCertSANsOutput, ok := tfState.Values.Outputs["api_server_cert_sans"]
if !ok {
return ApplyOutput{}, errors.New("no api_server_cert_sans output found")
}
apiServerCertSANsUntyped, ok := apiServerCertSANsOutput.Value.([]any)
if !ok {
return ApplyOutput{}, fmt.Errorf("invalid type in api_server_cert_sans output: %s is not a list of elements", apiServerCertSANsOutput.Type.FriendlyName())
}
apiServerCertSANs, err := toStringSlice(apiServerCertSANsUntyped)
if err != nil {
return ApplyOutput{}, fmt.Errorf("convert api_server_cert_sans output: %w", err)
}
secretOutput, ok := tfState.Values.Outputs["initSecret"]
if !ok {
return ApplyOutput{}, errors.New("no initSecret output found")
}
secret, ok := secretOutput.Value.(string)
if !ok {
return ApplyOutput{}, errors.New("invalid type in initSecret output: not a string")
}
uidOutput, ok := tfState.Values.Outputs["uid"]
if !ok {
return ApplyOutput{}, errors.New("no uid output found")
}
uid, ok := uidOutput.Value.(string)
if !ok {
return ApplyOutput{}, errors.New("invalid type in uid output: not a string")
}
res := ApplyOutput{
IP: ip,
APIServerCertSANs: apiServerCertSANs,
Secret: secret,
UID: uid,
}
// TODO add provider
switch provider {
case cloudprovider.GCP:
gcpProjectOutput, ok := tfState.Values.Outputs["project"]
if ok {
gcpProject, ok := gcpProjectOutput.Value.(string)
if !ok {
return ApplyOutput{}, errors.New("invalid type in project output: not a string")
}
cidrNodesOutput, ok := tfState.Values.Outputs["ip_cidr_nodes"]
if !ok {
return ApplyOutput{}, errors.New("no ip_cidr_nodes output found")
}
cidrNodes, ok := cidrNodesOutput.Value.(string)
if !ok {
return ApplyOutput{}, errors.New("invalid type in ip_cidr_nodes output: not a string")
}
cidrPodsOutput, ok := tfState.Values.Outputs["ip_cidr_pods"]
if !ok {
return ApplyOutput{}, errors.New("no ip_cidr_pods output found")
}
cidrPods, ok := cidrPodsOutput.Value.(string)
if !ok {
return ApplyOutput{}, errors.New("invalid type in ip_cidr_pods output: not a string")
}
res.GCP = &GCPApplyOutput{
ProjectID: gcpProject,
IPCidrNode: cidrNodes,
IPCidrPod: cidrPods,
}
}
case cloudprovider.Azure:
var attestationURL string
if ok {
if attestationURLOutput, ok := tfState.Values.Outputs["attestationURL"]; ok {
if attestationURLString, ok := attestationURLOutput.Value.(string); ok {
attestationURL = attestationURLString
}
}
}
azureUAMIOutput, ok := tfState.Values.Outputs["user_assigned_identity"]
if !ok {
return ApplyOutput{}, errors.New("no user_assigned_identity output found")
}
azureUAMI, ok := azureUAMIOutput.Value.(string)
if !ok {
return ApplyOutput{}, errors.New("invalid type in user_assigned_identity output: not a string")
}
rgOutput, ok := tfState.Values.Outputs["resource_group"]
if !ok {
return ApplyOutput{}, errors.New("no resource_group output found")
}
rg, ok := rgOutput.Value.(string)
if !ok {
return ApplyOutput{}, errors.New("invalid type in resource_group output: not a string")
}
subscriptionOutput, ok := tfState.Values.Outputs["subscription_id"]
if !ok {
return ApplyOutput{}, errors.New("no subscription_id output found")
}
subscriptionID, ok := subscriptionOutput.Value.(string)
if !ok {
return ApplyOutput{}, errors.New("invalid type in subscription_id output: not a string")
}
networkSGNameOutput, ok := tfState.Values.Outputs["network_security_group_name"]
if !ok {
return ApplyOutput{}, errors.New("no network_security_group_name output found")
}
networkSGName, ok := networkSGNameOutput.Value.(string)
if !ok {
return ApplyOutput{}, errors.New("invalid type in network_security_group_name output: not a string")
}
loadBalancerNameOutput, ok := tfState.Values.Outputs["loadbalancer_name"]
if !ok {
return ApplyOutput{}, errors.New("no loadbalancer_name output found")
}
loadBalancerName, ok := loadBalancerNameOutput.Value.(string)
if !ok {
return ApplyOutput{}, errors.New("invalid type in loadbalancer_name output: not a string")
}
res.Azure = &AzureApplyOutput{
ResourceGroup: rg,
SubscriptionID: subscriptionID,
UserAssignedIdentity: azureUAMI,
NetworkSecurityGroupName: networkSGName,
LoadBalancerName: loadBalancerName,
AttestationURL: attestationURL,
}
}
return res, nil
}
// PrepareWorkspace prepares a Terraform workspace for a Constellation cluster.
func (c *Client) PrepareWorkspace(path string, vars Variables) error {
if err := prepareWorkspace(path, c.file, c.workingDir); err != nil {
return fmt.Errorf("prepare workspace: %w", err)
}
return c.writeVars(vars)
}
// PrepareUpgradeWorkspace prepares a Terraform workspace for a Constellation version upgrade.
// It copies the Terraform state from the old working dir and the embedded Terraform files into the new working dir.
func (c *Client) PrepareUpgradeWorkspace(path, oldWorkingDir, newWorkingDir, backupDir string, vars Variables) error {
if err := prepareUpgradeWorkspace(path, c.file, oldWorkingDir, newWorkingDir, backupDir); err != nil {
return fmt.Errorf("prepare upgrade workspace: %w", err)
}
return c.writeVars(vars)
}
// PrepareIAMUpgradeWorkspace prepares a Terraform workspace for a Constellation IAM upgrade.
func PrepareIAMUpgradeWorkspace(file file.Handler, path, oldWorkingDir, newWorkingDir, backupDir string) error {
if err := prepareUpgradeWorkspace(path, file, oldWorkingDir, newWorkingDir, backupDir); err != nil {
return fmt.Errorf("prepare upgrade workspace: %w", err)
}
// copy the vars file from the old working dir to the new working dir
if err := file.CopyFile(filepath.Join(oldWorkingDir, terraformVarsFile), filepath.Join(newWorkingDir, terraformVarsFile)); err != nil {
return fmt.Errorf("copying vars file: %w", err)
}
return nil
}
// CreateCluster creates a Constellation cluster using Terraform.
func (c *Client) CreateCluster(ctx context.Context, provider cloudprovider.Provider, logLevel LogLevel) (ApplyOutput, error) {
if err := c.setLogLevel(logLevel); err != nil {
return ApplyOutput{}, fmt.Errorf("set terraform log level %s: %w", logLevel.String(), err)
}
if err := c.tf.Init(ctx); err != nil {
return ApplyOutput{}, fmt.Errorf("terraform init: %w", err)
}
if err := c.applyManualStateMigrations(ctx); err != nil {
return ApplyOutput{}, fmt.Errorf("apply manual state migrations: %w", err)
}
if err := c.tf.Apply(ctx); err != nil {
return ApplyOutput{}, fmt.Errorf("terraform apply: %w", err)
}
return c.ShowCluster(ctx, provider)
}
// ApplyOutput contains the Terraform output values of a cluster creation
// or apply operation.
type ApplyOutput struct {
IP string
APIServerCertSANs []string
Secret string
UID string
GCP *GCPApplyOutput
Azure *AzureApplyOutput
}
// AzureApplyOutput contains the Terraform output values of a terraform apply operation on Microsoft Azure.
type AzureApplyOutput struct {
ResourceGroup string
SubscriptionID string
NetworkSecurityGroupName string
LoadBalancerName string
UserAssignedIdentity string
// AttestationURL is the URL of the attestation provider.
AttestationURL string
}
// GCPApplyOutput contains the Terraform output values of a terraform apply operation on GCP.
type GCPApplyOutput struct {
ProjectID string
IPCidrNode string
IPCidrPod string
}
// IAMOutput contains the output information of the Terraform IAM operations.
type IAMOutput struct {
GCP GCPIAMOutput
Azure AzureIAMOutput
AWS AWSIAMOutput
}
// GCPIAMOutput contains the output information of the Terraform IAM operation on GCP.
type GCPIAMOutput struct {
SaKey string
}
// AzureIAMOutput contains the output information of the Terraform IAM operation on Microsoft Azure.
type AzureIAMOutput struct {
SubscriptionID string
TenantID string
UAMIID string
}
// AWSIAMOutput contains the output information of the Terraform IAM operation on GCP.
type AWSIAMOutput struct {
ControlPlaneInstanceProfile string
WorkerNodeInstanceProfile string
}
// ApplyIAMConfig creates an IAM configuration using Terraform.
func (c *Client) ApplyIAMConfig(ctx context.Context, provider cloudprovider.Provider, logLevel LogLevel) (IAMOutput, error) {
if err := c.setLogLevel(logLevel); err != nil {
return IAMOutput{}, fmt.Errorf("set terraform log level %s: %w", logLevel.String(), err)
}
if err := c.tf.Init(ctx); err != nil {
return IAMOutput{}, err
}
if err := c.tf.Apply(ctx); err != nil {
return IAMOutput{}, err
}
return c.ShowIAM(ctx, provider)
}
// Plan determines the diff that will be applied by Terraform. The plan output is written to the planFile. // Plan determines the diff that will be applied by Terraform. The plan output is written to the planFile.
// If there is a diff, the returned bool is true. Otherwise, it is false. // If there is a diff, the returned bool is true. Otherwise, it is false.
func (c *Client) Plan(ctx context.Context, logLevel LogLevel, planFile string) (bool, error) { func (c *Client) Plan(ctx context.Context, logLevel LogLevel, planFile string) (bool, error) {

View File

@ -277,6 +277,9 @@ module "scale_set_group" {
] ]
} }
data "azurerm_subscription" "current" {
}
moved { moved {
from = module.scale_set_control_plane from = module.scale_set_control_plane
to = module.scale_set_group["control_plane_default"] to = module.scale_set_group["control_plane_default"]

View File

@ -18,3 +18,24 @@ output "initSecret" {
output "attestationURL" { output "attestationURL" {
value = var.create_maa ? azurerm_attestation_provider.attestation_provider[0].attestation_uri : "" value = var.create_maa ? azurerm_attestation_provider.attestation_provider[0].attestation_uri : ""
} }
output "network_security_group_name" {
value = azurerm_network_security_group.security_group.name
}
output "loadbalancer_name" {
value = azurerm_lb.loadbalancer.name
}
output "user_assigned_identity" {
value = var.user_assigned_identity
}
output "resource_group" {
value = var.resource_group
}
output "subscription_id" {
value = data.azurerm_subscription.current.subscription_id
}

View File

@ -18,3 +18,15 @@ output "initSecret" {
value = random_password.initSecret.result value = random_password.initSecret.result
sensitive = true sensitive = true
} }
output "project" {
value = var.project
}
output "ip_cidr_nodes" {
value = local.cidr_vpc_subnet_nodes
}
output "ip_cidr_pods" {
value = local.cidr_vpc_subnet_pods
}

View File

@ -248,6 +248,21 @@ func TestCreateCluster(t *testing.T) {
"api_server_cert_sans": { "api_server_cert_sans": {
Value: []any{"192.0.2.100"}, Value: []any{"192.0.2.100"},
}, },
"user_assigned_identity": {
Value: "test_uami_id",
},
"resource_group": {
Value: "test_rg",
},
"subscription_id": {
Value: "test_subscription_id",
},
"network_security_group_name": {
Value: "test_nsg_name",
},
"loadbalancer_name": {
Value: "test_lb_name",
},
}, },
}, },
} }
@ -435,7 +450,7 @@ func TestCreateCluster(t *testing.T) {
path := path.Join(tc.pathBase, strings.ToLower(tc.provider.String())) path := path.Join(tc.pathBase, strings.ToLower(tc.provider.String()))
require.NoError(c.PrepareWorkspace(path, tc.vars)) require.NoError(c.PrepareWorkspace(path, tc.vars))
tfOutput, err := c.CreateCluster(context.Background(), LogLevelDebug) tfOutput, err := c.CreateCluster(context.Background(), tc.provider, LogLevelDebug)
if tc.wantErr { if tc.wantErr {
assert.Error(err) assert.Error(err)
@ -445,7 +460,9 @@ func TestCreateCluster(t *testing.T) {
assert.Equal("192.0.2.100", tfOutput.IP) assert.Equal("192.0.2.100", tfOutput.IP)
assert.Equal("initSecret", tfOutput.Secret) assert.Equal("initSecret", tfOutput.Secret)
assert.Equal("12345abc", tfOutput.UID) assert.Equal("12345abc", tfOutput.UID)
assert.Equal(tc.expectedAttestationURL, tfOutput.AttestationURL) if tc.provider == cloudprovider.Azure {
assert.Equal(tc.expectedAttestationURL, tfOutput.Azure.AttestationURL)
}
}) })
} }
} }

View File

@ -153,25 +153,24 @@ func CleanUpTerraformMigrations(upgradeID string, fileHandler file.Handler) erro
// In case of a successful upgrade, the output will be written to the specified file and the old Terraform directory is replaced // In case of a successful upgrade, the output will be written to the specified file and the old Terraform directory is replaced
// By the new one. // By the new one.
func (u *TerraformUpgrader) ApplyTerraformMigrations(ctx context.Context, opts TerraformUpgradeOptions, upgradeID string) error { func (u *TerraformUpgrader) ApplyTerraformMigrations(ctx context.Context, opts TerraformUpgradeOptions, upgradeID string) error {
tfOutput, err := u.tf.CreateCluster(ctx, opts.LogLevel) tfOutput, err := u.tf.CreateCluster(ctx, opts.CSP, opts.LogLevel)
if err != nil { if err != nil {
return fmt.Errorf("terraform apply: %w", err) return fmt.Errorf("terraform apply: %w", err)
} }
// AttestationURL is only set for Azure.
if tfOutput.AttestationURL != "" {
if err := u.policyPatcher.Patch(ctx, tfOutput.AttestationURL); err != nil {
return fmt.Errorf("patching policies: %w", err)
}
}
outputFileContents := clusterid.File{ outputFileContents := clusterid.File{
CloudProvider: opts.CSP, CloudProvider: opts.CSP,
InitSecret: []byte(tfOutput.Secret), InitSecret: []byte(tfOutput.Secret),
IP: tfOutput.IP, IP: tfOutput.IP,
APIServerCertSANs: tfOutput.APIServerCertSANs, APIServerCertSANs: tfOutput.APIServerCertSANs,
UID: tfOutput.UID, UID: tfOutput.UID,
AttestationURL: tfOutput.AttestationURL, }
// AttestationURL is only set for Azure.
if tfOutput.Azure != nil {
if err := u.policyPatcher.Patch(ctx, tfOutput.Azure.AttestationURL); err != nil {
return fmt.Errorf("patching policies: %w", err)
}
outputFileContents.AttestationURL = tfOutput.Azure.AttestationURL
} }
if err := u.fileHandler.RemoveAll(constants.TerraformWorkingDir); err != nil { if err := u.fileHandler.RemoveAll(constants.TerraformWorkingDir); err != nil {
@ -215,7 +214,7 @@ type tfClientCommon interface {
// tfResourceClient is a Terraform client for managing cluster resources. // tfResourceClient is a Terraform client for managing cluster resources.
type tfResourceClient interface { type tfResourceClient interface {
PrepareUpgradeWorkspace(path, oldWorkingDir, newWorkingDir, backupDir string, vars terraform.Variables) error PrepareUpgradeWorkspace(path, oldWorkingDir, newWorkingDir, backupDir string, vars terraform.Variables) error
CreateCluster(ctx context.Context, logLevel terraform.LogLevel) (terraform.ApplyOutput, error) CreateCluster(ctx context.Context, provider cloudprovider.Provider, logLevel terraform.LogLevel) (terraform.ApplyOutput, error)
tfClientCommon tfClientCommon
} }

View File

@ -371,7 +371,7 @@ func (u *stubTerraformClient) Plan(context.Context, terraform.LogLevel, string)
return u.hasDiff, u.planErr return u.hasDiff, u.planErr
} }
func (u *stubTerraformClient) CreateCluster(context.Context, terraform.LogLevel) (terraform.ApplyOutput, error) { func (u *stubTerraformClient) CreateCluster(context.Context, cloudprovider.Provider, terraform.LogLevel) (terraform.ApplyOutput, error) {
return terraform.ApplyOutput{}, u.CreateClusterErr return terraform.ApplyOutput{}, u.CreateClusterErr
} }

View File

@ -17,7 +17,6 @@ package azure
import ( import (
"context" "context"
"encoding/json"
"errors" "errors"
"fmt" "fmt"
"path" "path"
@ -99,61 +98,6 @@ func New(ctx context.Context) (*Cloud, error) {
}, nil }, nil
} }
// GetCCMConfig returns the configuration needed for the Kubernetes Cloud Controller Manager on Azure.
func (c *Cloud) GetCCMConfig(ctx context.Context, providerID string, cloudServiceAccountURI string) ([]byte, error) {
subscriptionID, resourceGroup, err := azureshared.BasicsFromProviderID(providerID)
if err != nil {
return nil, fmt.Errorf("parsing provider ID: %w", err)
}
creds, err := azureshared.ApplicationCredentialsFromURI(cloudServiceAccountURI)
if err != nil {
return nil, fmt.Errorf("parsing service account URI: %w", err)
}
uid, err := c.imds.uid(ctx)
if err != nil {
return nil, fmt.Errorf("retrieving instance UID: %w", err)
}
securityGroupName, err := c.getNetworkSecurityGroupName(ctx, resourceGroup, uid)
if err != nil {
return nil, fmt.Errorf("retrieving network security group name: %w", err)
}
loadBalancer, err := c.getLoadBalancer(ctx, resourceGroup, uid)
if err != nil {
return nil, fmt.Errorf("retrieving load balancer: %w", err)
}
if loadBalancer == nil || loadBalancer.Name == nil {
return nil, fmt.Errorf("could not dereference load balancer name")
}
var uamiClientID string
useManagedIdentityExtension := creds.PreferredAuthMethod == azureshared.AuthMethodUserAssignedIdentity
if useManagedIdentityExtension {
uamiClientID, err = c.getUAMIClientIDFromURI(ctx, providerID, creds.UamiResourceID)
if err != nil {
return nil, fmt.Errorf("retrieving user-assigned managed identity client ID: %w", err)
}
}
config := cloudConfig{
Cloud: "AzurePublicCloud",
TenantID: creds.TenantID,
SubscriptionID: subscriptionID,
ResourceGroup: resourceGroup,
LoadBalancerSku: "standard",
SecurityGroupName: securityGroupName,
LoadBalancerName: *loadBalancer.Name,
UseInstanceMetadata: true,
VMType: "vmss",
Location: creds.Location,
UseManagedIdentityExtension: useManagedIdentityExtension,
UserAssignedIdentityID: uamiClientID,
}
return json.Marshal(config)
}
// GetLoadBalancerEndpoint retrieves the first load balancer IP from cloud provider metadata. // GetLoadBalancerEndpoint retrieves the first load balancer IP from cloud provider metadata.
// //
// The returned string is an IP address without a port, but the method name needs to satisfy the // The returned string is an IP address without a port, but the method name needs to satisfy the
@ -286,24 +230,6 @@ func (c *Cloud) getInstance(ctx context.Context, providerID string) (metadata.In
return instance, nil return instance, nil
} }
func (c *Cloud) getUAMIClientIDFromURI(ctx context.Context, providerID, resourceID string) (string, error) {
// userAssignedIdentityURI := "/subscriptions/{subscription-id}/resourcegroups/{resource-group}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/{identity-name}"
_, resourceGroup, scaleSet, instanceID, err := azureshared.ScaleSetInformationFromProviderID(providerID)
if err != nil {
return "", fmt.Errorf("invalid provider ID: %w", err)
}
vmResp, err := c.scaleSetsVMAPI.Get(ctx, resourceGroup, scaleSet, instanceID, nil)
if err != nil {
return "", fmt.Errorf("retrieving instance: %w", err)
}
for rID, v := range vmResp.Identity.UserAssignedIdentities {
if rID == resourceID {
return *v.ClientID, nil
}
}
return "", fmt.Errorf("no user assinged identity found for resource ID %s", resourceID)
}
// getNetworkSecurityGroupName returns the security group name of the resource group. // getNetworkSecurityGroupName returns the security group name of the resource group.
func (c *Cloud) getNetworkSecurityGroupName(ctx context.Context, resourceGroup, uid string) (string, error) { func (c *Cloud) getNetworkSecurityGroupName(ctx context.Context, resourceGroup, uid string) (string, error) {
pager := c.secGroupAPI.NewListPager(resourceGroup, nil) pager := c.secGroupAPI.NewListPager(resourceGroup, nil)
@ -462,26 +388,6 @@ func (c *Cloud) getLoadBalancerDNSName(ctx context.Context) (string, error) {
} }
*/ */
type cloudConfig struct {
Cloud string `json:"cloud,omitempty"`
TenantID string `json:"tenantId,omitempty"`
SubscriptionID string `json:"subscriptionId,omitempty"`
ResourceGroup string `json:"resourceGroup,omitempty"`
Location string `json:"location,omitempty"`
SubnetName string `json:"subnetName,omitempty"`
SecurityGroupName string `json:"securityGroupName,omitempty"`
SecurityGroupResourceGroup string `json:"securityGroupResourceGroup,omitempty"`
LoadBalancerName string `json:"loadBalancerName,omitempty"`
LoadBalancerSku string `json:"loadBalancerSku,omitempty"`
VNetName string `json:"vnetName,omitempty"`
VNetResourceGroup string `json:"vnetResourceGroup,omitempty"`
CloudProviderBackoff bool `json:"cloudProviderBackoff,omitempty"`
UseInstanceMetadata bool `json:"useInstanceMetadata,omitempty"`
VMType string `json:"vmType,omitempty"`
UseManagedIdentityExtension bool `json:"useManagedIdentityExtension,omitempty"`
UserAssignedIdentityID string `json:"userAssignedIdentityID,omitempty"`
}
// convertToInstanceMetadata converts a armcomputev2.VirtualMachineScaleSetVM to a metadata.InstanceMetadata. // convertToInstanceMetadata converts a armcomputev2.VirtualMachineScaleSetVM to a metadata.InstanceMetadata.
func convertToInstanceMetadata(vm armcompute.VirtualMachineScaleSetVM, networkInterfaces []armnetwork.Interface, func convertToInstanceMetadata(vm armcompute.VirtualMachineScaleSetVM, networkInterfaces []armnetwork.Interface,
) (metadata.InstanceMetadata, error) { ) (metadata.InstanceMetadata, error) {

View File

@ -8,7 +8,6 @@ package azure
import ( import (
"context" "context"
"encoding/json"
"errors" "errors"
"testing" "testing"
@ -28,336 +27,6 @@ func TestMain(m *testing.M) {
goleak.VerifyTestMain(m) goleak.VerifyTestMain(m)
} }
func TestGetCCMConfig(t *testing.T) {
someErr := errors.New("failed")
goodLB := armnetwork.LoadBalancer{
Name: to.Ptr("load-balancer"),
Tags: map[string]*string{
cloud.TagUID: to.Ptr("uid"),
},
Properties: &armnetwork.LoadBalancerPropertiesFormat{
FrontendIPConfigurations: []*armnetwork.FrontendIPConfiguration{
{
Properties: &armnetwork.FrontendIPConfigurationPropertiesFormat{
PublicIPAddress: &armnetwork.PublicIPAddress{ID: to.Ptr("/subscriptions/subscription/resourceGroups/resourceGroup/providers/Microsoft.Network/publicIPAddresses/pubIPName")},
},
},
},
},
}
goodSecurityGroup := armnetwork.SecurityGroup{
Tags: map[string]*string{
cloud.TagUID: to.Ptr("uid"),
},
Name: to.Ptr("security-group"),
}
uamiClientID := "uami-client-id"
testCases := map[string]struct {
imdsAPI imdsAPI
loadBalancerAPI loadBalancerAPI
secGroupAPI securityGroupsAPI
scaleSetsVMAPI virtualMachineScaleSetVMsAPI
providerID string
cloudServiceAccountURI string
wantErr bool
wantConfig cloudConfig
}{
"success": {
imdsAPI: &stubIMDSAPI{
uidVal: "uid",
},
loadBalancerAPI: &stubLoadBalancersAPI{
pager: &stubLoadBalancersClientListPager{
list: []armnetwork.LoadBalancer{goodLB},
},
},
secGroupAPI: &stubSecurityGroupsAPI{
pager: &stubSecurityGroupsClientListPager{
list: []armnetwork.SecurityGroup{goodSecurityGroup},
},
},
scaleSetsVMAPI: &stubVirtualMachineScaleSetVMsAPI{
getVM: armcompute.VirtualMachineScaleSetVM{
Identity: &armcompute.VirtualMachineIdentity{
UserAssignedIdentities: map[string]*armcompute.UserAssignedIdentitiesValue{
"subscriptions/9b352db0-82af-408c-a02c-36fbffbf7015/resourceGroups/resourceGroupName/providers/Microsoft.ManagedIdentity/userAssignedIdentities/UAMIName": {ClientID: &uamiClientID},
},
},
},
},
providerID: "azure:///subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/virtualMachineScaleSets/scale-set/virtualMachines/0",
cloudServiceAccountURI: "serviceaccount://azure?tenant_id=tenant-id&client_id=client-id&client_secret=client-secret&location=westeurope&preferred_auth_method=userassignedidentity&uami_resource_id=subscriptions%2F9b352db0-82af-408c-a02c-36fbffbf7015%2FresourceGroups%2FresourceGroupName%2Fproviders%2FMicrosoft.ManagedIdentity%2FuserAssignedIdentities%2FUAMIName",
wantConfig: cloudConfig{
Cloud: "AzurePublicCloud",
TenantID: "tenant-id",
SubscriptionID: "subscription-id",
ResourceGroup: "resource-group",
LoadBalancerSku: "standard",
SecurityGroupName: "security-group",
LoadBalancerName: "load-balancer",
UseInstanceMetadata: true,
UseManagedIdentityExtension: true,
UserAssignedIdentityID: uamiClientID,
VMType: "vmss",
Location: "westeurope",
},
},
"no app registration": {
imdsAPI: &stubIMDSAPI{
uidVal: "uid",
},
loadBalancerAPI: &stubLoadBalancersAPI{
pager: &stubLoadBalancersClientListPager{
list: []armnetwork.LoadBalancer{goodLB},
},
},
secGroupAPI: &stubSecurityGroupsAPI{
pager: &stubSecurityGroupsClientListPager{
list: []armnetwork.SecurityGroup{goodSecurityGroup},
},
},
providerID: "azure:///subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/virtualMachineScaleSets/scale-set/virtualMachines/0",
cloudServiceAccountURI: "serviceaccount://azure?tenant_id=tenant-id&location=westeurope",
wantConfig: cloudConfig{
Cloud: "AzurePublicCloud",
TenantID: "tenant-id",
SubscriptionID: "subscription-id",
ResourceGroup: "resource-group",
LoadBalancerSku: "standard",
SecurityGroupName: "security-group",
LoadBalancerName: "load-balancer",
UseInstanceMetadata: true,
VMType: "vmss",
Location: "westeurope",
},
},
"missing UID tag": {
imdsAPI: &stubIMDSAPI{
uidVal: "uid",
},
loadBalancerAPI: &stubLoadBalancersAPI{
pager: &stubLoadBalancersClientListPager{
list: []armnetwork.LoadBalancer{
{
Name: to.Ptr("load-balancer"),
Properties: &armnetwork.LoadBalancerPropertiesFormat{
FrontendIPConfigurations: []*armnetwork.FrontendIPConfiguration{
{
Properties: &armnetwork.FrontendIPConfigurationPropertiesFormat{
PublicIPAddress: &armnetwork.PublicIPAddress{ID: to.Ptr("/subscriptions/subscription/resourceGroups/resourceGroup/providers/Microsoft.Network/publicIPAddresses/pubIPName")},
},
},
},
},
},
},
},
},
secGroupAPI: &stubSecurityGroupsAPI{
pager: &stubSecurityGroupsClientListPager{
list: []armnetwork.SecurityGroup{goodSecurityGroup},
},
},
providerID: "azure:///subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/virtualMachineScaleSets/scale-set/virtualMachines/0",
cloudServiceAccountURI: "serviceaccount://azure?tenant_id=tenant-id&client_id=client-id&client_secret=client-secret&location=westeurope",
wantErr: true,
},
"only correct UID is chosen": {
imdsAPI: &stubIMDSAPI{
uidVal: "uid",
},
loadBalancerAPI: &stubLoadBalancersAPI{
pager: &stubLoadBalancersClientListPager{
list: []armnetwork.LoadBalancer{
{
Name: to.Ptr("load-balancer"),
Tags: map[string]*string{
cloud.TagUID: to.Ptr("different-uid"),
},
Properties: &armnetwork.LoadBalancerPropertiesFormat{
FrontendIPConfigurations: []*armnetwork.FrontendIPConfiguration{
{
Properties: &armnetwork.FrontendIPConfigurationPropertiesFormat{
PublicIPAddress: &armnetwork.PublicIPAddress{ID: to.Ptr("/subscriptions/subscription/resourceGroups/resourceGroup/providers/Microsoft.Network/publicIPAddresses/pubIPName")},
},
},
},
},
},
goodLB,
},
},
},
secGroupAPI: &stubSecurityGroupsAPI{
pager: &stubSecurityGroupsClientListPager{
list: []armnetwork.SecurityGroup{goodSecurityGroup},
},
},
providerID: "azure:///subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/virtualMachineScaleSets/scale-set/virtualMachines/0",
cloudServiceAccountURI: "serviceaccount://azure?tenant_id=tenant-id&client_id=client-id&client_secret=client-secret&location=westeurope",
wantConfig: cloudConfig{
Cloud: "AzurePublicCloud",
TenantID: "tenant-id",
SubscriptionID: "subscription-id",
ResourceGroup: "resource-group",
LoadBalancerSku: "standard",
SecurityGroupName: "security-group",
LoadBalancerName: "load-balancer",
UseInstanceMetadata: true,
VMType: "vmss",
Location: "westeurope",
},
},
"load balancer list error": {
imdsAPI: &stubIMDSAPI{
uidVal: "uid",
},
loadBalancerAPI: &stubLoadBalancersAPI{
pager: &stubLoadBalancersClientListPager{
fetchErr: someErr,
},
},
secGroupAPI: &stubSecurityGroupsAPI{
pager: &stubSecurityGroupsClientListPager{
list: []armnetwork.SecurityGroup{goodSecurityGroup},
},
},
providerID: "azure:///subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/virtualMachineScaleSets/scale-set/virtualMachines/0",
cloudServiceAccountURI: "serviceaccount://azure?tenant_id=tenant-id&client_id=client-id&client_secret=client-secret&location=westeurope",
wantErr: true,
},
"missing load balancer name": {
imdsAPI: &stubIMDSAPI{
uidVal: "uid",
},
loadBalancerAPI: &stubLoadBalancersAPI{
pager: &stubLoadBalancersClientListPager{
list: []armnetwork.LoadBalancer{{
Tags: map[string]*string{
cloud.TagUID: to.Ptr("uid"),
},
Properties: &armnetwork.LoadBalancerPropertiesFormat{
FrontendIPConfigurations: []*armnetwork.FrontendIPConfiguration{
{
Properties: &armnetwork.FrontendIPConfigurationPropertiesFormat{
PublicIPAddress: &armnetwork.PublicIPAddress{ID: to.Ptr("/subscriptions/subscription/resourceGroups/resourceGroup/providers/Microsoft.Network/publicIPAddresses/pubIPName")},
},
},
},
},
}},
},
},
secGroupAPI: &stubSecurityGroupsAPI{
pager: &stubSecurityGroupsClientListPager{
list: []armnetwork.SecurityGroup{goodSecurityGroup},
},
},
providerID: "azure:///subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/virtualMachineScaleSets/scale-set/virtualMachines/0",
cloudServiceAccountURI: "serviceaccount://azure?tenant_id=tenant-id&client_id=client-id&client_secret=client-secret&location=westeurope",
wantErr: true,
},
"security group list error": {
imdsAPI: &stubIMDSAPI{
uidVal: "uid",
},
loadBalancerAPI: &stubLoadBalancersAPI{
pager: &stubLoadBalancersClientListPager{
list: []armnetwork.LoadBalancer{goodLB},
},
},
secGroupAPI: &stubSecurityGroupsAPI{
pager: &stubSecurityGroupsClientListPager{
fetchErr: someErr,
},
},
providerID: "azure:///subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/virtualMachineScaleSets/scale-set/virtualMachines/0",
cloudServiceAccountURI: "serviceaccount://azure?tenant_id=tenant-id&client_id=client-id&client_secret=client-secret&location=westeurope",
wantErr: true,
},
"invalid provider ID": {
imdsAPI: &stubIMDSAPI{
uidVal: "uid",
},
loadBalancerAPI: &stubLoadBalancersAPI{
pager: &stubLoadBalancersClientListPager{
list: []armnetwork.LoadBalancer{goodLB},
},
},
secGroupAPI: &stubSecurityGroupsAPI{
pager: &stubSecurityGroupsClientListPager{
list: []armnetwork.SecurityGroup{goodSecurityGroup},
},
},
providerID: "invalid:///subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/virtualMachineScaleSets/scale-set/virtualMachines/0",
cloudServiceAccountURI: "serviceaccount://azure?tenant_id=tenant-id&client_id=client-id&client_secret=client-secret&location=westeurope",
wantErr: true,
},
"invalid cloud service account URI": {
imdsAPI: &stubIMDSAPI{
uidVal: "uid",
},
loadBalancerAPI: &stubLoadBalancersAPI{
pager: &stubLoadBalancersClientListPager{
list: []armnetwork.LoadBalancer{goodLB},
},
},
secGroupAPI: &stubSecurityGroupsAPI{
pager: &stubSecurityGroupsClientListPager{
list: []armnetwork.SecurityGroup{goodSecurityGroup},
},
},
providerID: "azure:///subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/virtualMachineScaleSets/scale-set/virtualMachines/0",
cloudServiceAccountURI: "invalid://azure?tenant_id=tenant-id&client_id=client-id&client_secret=client-secret&location=westeurope",
wantErr: true,
},
"imds error": {
imdsAPI: &stubIMDSAPI{
uidErr: someErr,
},
loadBalancerAPI: &stubLoadBalancersAPI{
pager: &stubLoadBalancersClientListPager{
list: []armnetwork.LoadBalancer{goodLB},
},
},
secGroupAPI: &stubSecurityGroupsAPI{
pager: &stubSecurityGroupsClientListPager{
list: []armnetwork.SecurityGroup{goodSecurityGroup},
},
},
providerID: "azure:///subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/virtualMachineScaleSets/scale-set/virtualMachines/0",
cloudServiceAccountURI: "serviceaccount://azure?tenant_id=tenant-id&client_id=client-id&client_secret=client-secret&location=westeurope",
wantErr: true,
},
}
for name, tc := range testCases {
t.Run(name, func(t *testing.T) {
assert := assert.New(t)
require := require.New(t)
cloud := &Cloud{
imds: tc.imdsAPI,
loadBalancerAPI: tc.loadBalancerAPI,
secGroupAPI: tc.secGroupAPI,
scaleSetsVMAPI: tc.scaleSetsVMAPI,
}
config, err := cloud.GetCCMConfig(context.Background(), tc.providerID, tc.cloudServiceAccountURI)
if tc.wantErr {
assert.Error(err)
return
}
assert.NoError(err)
wantConfig, err := json.Marshal(tc.wantConfig)
require.NoError(err)
assert.JSONEq(string(wantConfig), string(config))
})
}
}
func TestGetInstance(t *testing.T) { func TestGetInstance(t *testing.T) {
someErr := errors.New("failed") someErr := errors.New("failed")
sampleProviderID := "azure:///subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/virtualMachineScaleSets/scale-set-name/virtualMachines/instance-id" sampleProviderID := "azure:///subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/virtualMachineScaleSets/scale-set-name/virtualMachines/instance-id"

View File

@ -9,13 +9,21 @@ package azureshared
import ( import (
"fmt" "fmt"
"net/url" "net/url"
"regexp"
"strings" "strings"
) )
var (
subscriptionPattern = regexp.MustCompile(`subscriptions/([^/]+)/`)
rgPattern = regexp.MustCompile(`resourceGroups/([^/]+)/`)
)
// ApplicationCredentials is a set of Azure API credentials. // ApplicationCredentials is a set of Azure API credentials.
// It can contain a client secret and carries the preferred authentication method. // It can contain a client secret and carries the preferred authentication method.
// It is the equivalent of a service account key in other cloud providers. // It is the equivalent of a service account key in other cloud providers.
type ApplicationCredentials struct { type ApplicationCredentials struct {
SubscriptionID string
ResourceGroup string
TenantID string TenantID string
AppClientID string AppClientID string
ClientSecretValue string ClientSecretValue string
@ -37,8 +45,14 @@ func ApplicationCredentialsFromURI(cloudServiceAccountURI string) (ApplicationCr
return ApplicationCredentials{}, fmt.Errorf("invalid service account URI: invalid host: %s", uri.Host) return ApplicationCredentials{}, fmt.Errorf("invalid service account URI: invalid host: %s", uri.Host)
} }
query := uri.Query() query := uri.Query()
subscriptionID := getFirstMatchOrEmpty(subscriptionPattern, query.Get("uami_resource_id"))
resourceGroup := getFirstMatchOrEmpty(rgPattern, query.Get("uami_resource_id"))
preferredAuthMethod := FromString(query.Get("preferred_auth_method")) preferredAuthMethod := FromString(query.Get("preferred_auth_method"))
return ApplicationCredentials{ return ApplicationCredentials{
SubscriptionID: subscriptionID,
ResourceGroup: resourceGroup,
TenantID: query.Get("tenant_id"), TenantID: query.Get("tenant_id"),
AppClientID: query.Get("client_id"), AppClientID: query.Get("client_id"),
ClientSecretValue: query.Get("client_secret"), ClientSecretValue: query.Get("client_secret"),
@ -48,6 +62,15 @@ func ApplicationCredentialsFromURI(cloudServiceAccountURI string) (ApplicationCr
}, nil }, nil
} }
func getFirstMatchOrEmpty(pattern *regexp.Regexp, str string) string {
subscriptionMatches := pattern.FindStringSubmatch(str)
var subscriptionID string
if len(subscriptionMatches) > 1 {
subscriptionID = subscriptionMatches[1]
}
return subscriptionID
}
// ToCloudServiceAccountURI converts the ApplicationCredentials into a cloud service account URI. // ToCloudServiceAccountURI converts the ApplicationCredentials into a cloud service account URI.
func (c ApplicationCredentials) ToCloudServiceAccountURI() string { func (c ApplicationCredentials) ToCloudServiceAccountURI() string {
query := url.Values{} query := url.Values{}

View File

@ -27,12 +27,16 @@ func TestApplicationCredentialsFromURI(t *testing.T) {
Location: "location", Location: "location",
UamiResourceID: "subscriptions/9b352db0-82af-408c-a02c-36fbffbf7015/resourceGroups/resourceGroupName/providers/Microsoft.ManagedIdentity/userAssignedIdentities/UAMIName", UamiResourceID: "subscriptions/9b352db0-82af-408c-a02c-36fbffbf7015/resourceGroups/resourceGroupName/providers/Microsoft.ManagedIdentity/userAssignedIdentities/UAMIName",
PreferredAuthMethod: AuthMethodServicePrincipal, PreferredAuthMethod: AuthMethodServicePrincipal,
SubscriptionID: "9b352db0-82af-408c-a02c-36fbffbf7015",
ResourceGroup: "resourceGroupName",
} }
credsWithoutSecret := ApplicationCredentials{ credsWithoutSecret := ApplicationCredentials{
TenantID: "tenant-id", TenantID: "tenant-id",
Location: "location", Location: "location",
UamiResourceID: "subscriptions/9b352db0-82af-408c-a02c-36fbffbf7015/resourceGroups/resourceGroupName/providers/Microsoft.ManagedIdentity/userAssignedIdentities/UAMIName", UamiResourceID: "subscriptions/9b352db0-82af-408c-a02c-36fbffbf7015/resourceGroups/resourceGroupName/providers/Microsoft.ManagedIdentity/userAssignedIdentities/UAMIName",
PreferredAuthMethod: AuthMethodUserAssignedIdentity, PreferredAuthMethod: AuthMethodUserAssignedIdentity,
SubscriptionID: "9b352db0-82af-408c-a02c-36fbffbf7015",
ResourceGroup: "resourceGroupName",
} }
credsWithoutPreferrredAuthMethod := ApplicationCredentials{ credsWithoutPreferrredAuthMethod := ApplicationCredentials{
TenantID: "tenant-id", TenantID: "tenant-id",

View File

@ -11,10 +11,8 @@ go_library(
visibility = ["//:__subpackages__"], visibility = ["//:__subpackages__"],
deps = [ deps = [
"//internal/constants", "//internal/constants",
"//internal/logger",
"//internal/retry", "//internal/retry",
"@io_k8s_apimachinery//pkg/util/wait", "@io_k8s_apimachinery//pkg/util/wait",
"@org_uber_go_zap//:zap",
"@sh_helm_helm_v3//pkg/action", "@sh_helm_helm_v3//pkg/action",
"@sh_helm_helm_v3//pkg/chart", "@sh_helm_helm_v3//pkg/chart",
"@sh_helm_helm_v3//pkg/chart/loader", "@sh_helm_helm_v3//pkg/chart/loader",

View File

@ -14,9 +14,7 @@ import (
"time" "time"
"github.com/edgelesssys/constellation/v2/internal/constants" "github.com/edgelesssys/constellation/v2/internal/constants"
"github.com/edgelesssys/constellation/v2/internal/logger"
"github.com/edgelesssys/constellation/v2/internal/retry" "github.com/edgelesssys/constellation/v2/internal/retry"
"go.uber.org/zap"
"helm.sh/helm/v3/pkg/action" "helm.sh/helm/v3/pkg/action"
"helm.sh/helm/v3/pkg/chart" "helm.sh/helm/v3/pkg/chart"
"helm.sh/helm/v3/pkg/chart/loader" "helm.sh/helm/v3/pkg/chart/loader"
@ -31,20 +29,25 @@ const (
maximumRetryAttempts = 3 maximumRetryAttempts = 3
) )
type debugLog interface {
Debugf(format string, args ...any)
Sync()
}
// Installer is a wrapper for a helm install action. // Installer is a wrapper for a helm install action.
type Installer struct { type Installer struct {
*action.Install *action.Install
log *logger.Logger log debugLog
} }
// NewInstaller creates a new Installer with the given logger. // NewInstaller creates a new Installer with the given logger.
func NewInstaller(log *logger.Logger, kubeconfig string) (*Installer, error) { func NewInstaller(kubeconfig string, logger debugLog) (*Installer, error) {
settings := cli.New() settings := cli.New()
settings.KubeConfig = kubeconfig settings.KubeConfig = kubeconfig
actionConfig := &action.Configuration{} actionConfig := &action.Configuration{}
if err := actionConfig.Init(settings.RESTClientGetter(), constants.HelmNamespace, if err := actionConfig.Init(settings.RESTClientGetter(), constants.HelmNamespace,
"secret", log.Infof); err != nil { "secret", logger.Debugf); err != nil {
return nil, err return nil, err
} }
@ -53,13 +56,12 @@ func NewInstaller(log *logger.Logger, kubeconfig string) (*Installer, error) {
action.Timeout = timeout action.Timeout = timeout
return &Installer{ return &Installer{
action, Install: action,
log, log: logger,
}, nil }, nil
} }
// InstallChart is the generic install function for helm charts. // InstallChart is the generic install function for helm charts.
// When timeout is nil, the default timeout is used.
func (h *Installer) InstallChart(ctx context.Context, release Release) error { func (h *Installer) InstallChart(ctx context.Context, release Release) error {
return h.InstallChartWithValues(ctx, release, nil) return h.InstallChartWithValues(ctx, release, nil)
} }
@ -115,7 +117,7 @@ func (h *Installer) install(ctx context.Context, chartRaw []byte, values map[str
return fmt.Errorf("helm install: %w", err) return fmt.Errorf("helm install: %w", err)
} }
retryLoopFinishDuration := time.Since(retryLoopStartTime) retryLoopFinishDuration := time.Since(retryLoopStartTime)
h.log.With(zap.String("chart", chart.Name()), zap.Duration("duration", retryLoopFinishDuration)).Infof("Helm chart installation finished") h.log.Debugf("Helm chart %q installation finished after %s", chart.Name(), retryLoopFinishDuration)
return nil return nil
} }
@ -143,15 +145,14 @@ type installDoer struct {
Installer *Installer Installer *Installer
chart *chart.Chart chart *chart.Chart
values map[string]any values map[string]any
log *logger.Logger log debugLog
} }
// Do logs which chart is installed and tries to install it. // Do logs which chart is installed and tries to install it.
func (i installDoer) Do(ctx context.Context) error { func (i installDoer) Do(ctx context.Context) error {
i.log.With(zap.String("chart", i.chart.Name())).Infof("Trying to install Helm chart") i.log.Debugf("Trying to install Helm chart %s", i.chart.Name())
if _, err := i.Installer.RunWithContext(ctx, i.chart, i.values); err != nil { if _, err := i.Installer.RunWithContext(ctx, i.chart, i.values); err != nil {
i.log.With(zap.Error(err), zap.String("chart", i.chart.Name())).Errorf("Helm chart installation failed") i.log.Debugf("Helm chart installation % failed: %v", i.chart.Name(), err)
return err return err
} }