Git-Mirrors/constellation
1
0
Fork 0
mirror of https://github.com/edgelesssys/constellation.git synced 2024-12-24 06:59:40 -05:00
Code Issues Packages Projects Releases Wiki Activity

cli: install helm charts in cli instead of bootstrapper (#2136)

Browse Source 
* init

* fixup! init

* gcp working?

* fixup! fixup! init

* azure cfg for microService installation

* fixup! azure cfg for microService installation

* fixup! azure cfg for microService installation

* cleanup bootstrapper code

* cleanup helminstall code

* fixup! cleanup helminstall code

* Update internal/deploy/helm/install.go

Co-authored-by: Daniel Weiße <66256922+daniel-weisse@users.noreply.github.com>

* daniel feedback

* TODO add provider (also to CreateCluster) so we can ensure that provider specific output

* fixup! daniel feedback

* use debugLog in helm installer

* placeholderHelmInstaller

* rename to stub

---------

Co-authored-by: Daniel Weiße <66256922+daniel-weisse@users.noreply.github.com>
This commit is contained in:
Adrian Stobbe 2023-07-31 10:53:05 +02:00 committed by GitHub
parent ef60d00a60
commit 26305e8f80
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
38 changed files with 775 additions and 970 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
bootstrapper/cmd/bootstrapper/main.go
View File

@ -67,7 +67,7 @@ func main() {
var openDevice vtpm.TPMOpenFunc
var fs afero.Fs
helmClient, err := helm.NewInstaller(log, constants.ControlPlaneAdminConfFilename)
helmClient, err := helm.NewInstaller(constants.ControlPlaneAdminConfFilename, log)
if err != nil {
log.With(zap.Error(err)).Fatalf("Helm client could not be initialized")
}

2
bootstrapper/cmd/bootstrapper/test.go
View File

@ -21,7 +21,7 @@ type clusterFake struct{}
// InitCluster fakes bootstrapping a new cluster with the current node being the master, returning the arguments required to join the cluster.
func (c *clusterFake) InitCluster(
context.Context, string, string, string, []byte,
context.Context, string, string,
[]byte, bool, components.Components, []string, *logger.Logger,
) ([]byte, error) {
return []byte{}, nil

4
bootstrapper/internal/initserver/initserver.go
View File

@ -213,10 +213,8 @@ func (s *Server) Init(req *initproto.InitRequest, stream initproto.API_InitServe
}
kubeconfig, err := s.initializer.InitCluster(stream.Context(),
req.CloudServiceAccountUri,
req.KubernetesVersion,
clusterName,
measurementSalt,
req.HelmDeployments,
req.ConformanceMode,
components.NewComponentsFromInitProto(req.KubernetesComponents),
@ -342,10 +340,8 @@ type ClusterInitializer interface {
// InitCluster initializes a new Kubernetes cluster.
InitCluster(
ctx context.Context,
cloudServiceAccountURI string,
k8sVersion string,
clusterName string,
measurementSalt []byte,
helmDeployments []byte,
conformanceMode bool,
kubernetesComponents components.Components,

2
bootstrapper/internal/initserver/initserver_test.go
View File

@ -406,7 +406,7 @@ type stubClusterInitializer struct {
}
func (i *stubClusterInitializer) InitCluster(
context.Context, string, string, string, []byte,
context.Context, string, string,
[]byte, bool, components.Components, []string, *logger.Logger,
) ([]byte, error) {
return i.initClusterKubeconfig, i.initClusterErr

2
bootstrapper/internal/kubernetes/BUILD.bazel
View File

@ -14,9 +14,7 @@ go_library(
"//bootstrapper/internal/kubernetes/k8sapi",
"//bootstrapper/internal/kubernetes/kubewaiter",
"//internal/cloud/cloudprovider",
"//internal/cloud/gcpshared",
"//internal/cloud/metadata",
"//internal/cloud/openstack",
"//internal/constants",
"//internal/deploy/helm",
"//internal/kubernetes",

199
bootstrapper/internal/kubernetes/kubernetes.go
View File

@ -9,7 +9,6 @@ package kubernetes
import (
"context"
"encoding/base64"
"encoding/json"
"errors"
"fmt"
@ -22,8 +21,6 @@ import (
"github.com/edgelesssys/constellation/v2/bootstrapper/internal/kubernetes/k8sapi"
"github.com/edgelesssys/constellation/v2/bootstrapper/internal/kubernetes/kubewaiter"
"github.com/edgelesssys/constellation/v2/internal/cloud/cloudprovider"
"github.com/edgelesssys/constellation/v2/internal/cloud/gcpshared"
"github.com/edgelesssys/constellation/v2/internal/cloud/openstack"
"github.com/edgelesssys/constellation/v2/internal/constants"
"github.com/edgelesssys/constellation/v2/internal/deploy/helm"
"github.com/edgelesssys/constellation/v2/internal/kubernetes"
@ -78,7 +75,7 @@ func New(cloudProvider string, clusterUtil clusterUtil, configProvider configura
// InitCluster initializes a new Kubernetes cluster and applies pod network provider.
func (k *KubeWrapper) InitCluster(
ctx context.Context, cloudServiceAccountURI, versionString, clusterName string, measurementSalt []byte,
ctx context.Context, versionString, clusterName string,
helmReleasesRaw []byte, conformanceMode bool, kubernetesComponents components.Components, apiServerCertSANs []string, log *logger.Logger,
) ([]byte, error) {
log.With(zap.String("version", versionString)).Infof("Installing Kubernetes components")
@ -223,75 +220,10 @@ func (k *KubeWrapper) InitCluster(
// Continue and don't throw an error here - things might be okay.
}
serviceConfig := constellationServicesConfig{
measurementSalt: measurementSalt,
subnetworkPodCIDR: subnetworkPodCIDR,
cloudServiceAccountURI: cloudServiceAccountURI,
loadBalancerIP: controlPlaneHost,
}
constellationVals, err := k.setupExtraVals(ctx, serviceConfig)
if err != nil {
return nil, fmt.Errorf("setting up extraVals: %w", err)
}
log.Infof("Setting up internal-config ConfigMap")
if err := k.setupInternalConfigMap(ctx); err != nil {
return nil, fmt.Errorf("failed to setup internal ConfigMap: %w", err)
}
log.Infof("Installing Constellation microservices")
if err = k.helmClient.InstallChartWithValues(ctx, helmReleases.ConstellationServices, constellationVals); err != nil {
return nil, fmt.Errorf("installing constellation-services: %w", err)
}
// cert-manager provides CRDs used by other deployments,
// so it should be installed as early as possible, but after the services cert-manager depends on.
log.Infof("Installing cert-manager")
if err = k.helmClient.InstallChart(ctx, helmReleases.CertManager); err != nil {
return nil, fmt.Errorf("installing cert-manager: %w", err)
}
// Install CSI drivers if enabled by the user.
if helmReleases.CSI != nil {
var csiVals map[string]any
if cloudprovider.FromString(k.cloudProvider) == cloudprovider.OpenStack {
creds, err := openstack.AccountKeyFromURI(serviceConfig.cloudServiceAccountURI)
if err != nil {
return nil, err
}
cinderIni := creds.CloudINI().CinderCSIConfiguration()
csiVals = map[string]any{
"cinder-config": map[string]any{
"secretData": cinderIni,
},
}
}
log.Infof("Installing CSI deployments")
if err := k.helmClient.InstallChartWithValues(ctx, *helmReleases.CSI, csiVals); err != nil {
return nil, fmt.Errorf("installing CSI snapshot CRDs: %w", err)
}
}
if helmReleases.AWSLoadBalancerController != nil {
log.Infof("Installing AWS Load Balancer Controller")
if err = k.helmClient.InstallChart(ctx, *helmReleases.AWSLoadBalancerController); err != nil {
return nil, fmt.Errorf("installing AWS Load Balancer Controller: %w", err)
}
}
operatorVals, err := k.setupOperatorVals(ctx)
if err != nil {
return nil, fmt.Errorf("setting up operator vals: %w", err)
}
// Constellation operators require CRDs from cert-manager.
// They must be installed after it.
log.Infof("Installing operators")
if err = k.helmClient.InstallChartWithValues(ctx, helmReleases.ConstellationOperators, operatorVals); err != nil {
return nil, fmt.Errorf("installing operators: %w", err)
}
return kubeConfig, nil
}
@ -447,120 +379,6 @@ func getIPAddr() (string, error) {
return localAddr.IP.String(), nil
}
// setupExtraVals create a helm values map for consumption by helm-install.
// Will move to a more dedicated place once that place becomes apparent.
func (k *KubeWrapper) setupExtraVals(ctx context.Context, serviceConfig constellationServicesConfig) (map[string]any, error) {
extraVals := map[string]any{
"join-service": map[string]any{
"measurementSalt": base64.StdEncoding.EncodeToString(serviceConfig.measurementSalt),
},
"verification-service": map[string]any{
"loadBalancerIP": serviceConfig.loadBalancerIP,
},
"konnectivity": map[string]any{
"loadBalancerIP": serviceConfig.loadBalancerIP,
},
}
instance, err := k.providerMetadata.Self(ctx)
if err != nil {
return nil, fmt.Errorf("retrieving current instance: %w", err)
}
switch cloudprovider.FromString(k.cloudProvider) {
case cloudprovider.GCP:
uid, err := k.providerMetadata.UID(ctx)
if err != nil {
return nil, fmt.Errorf("getting uid: %w", err)
}
projectID, _, _, err := gcpshared.SplitProviderID(instance.ProviderID)
if err != nil {
return nil, fmt.Errorf("splitting providerID: %w", err)
}
serviceAccountKey, err := gcpshared.ServiceAccountKeyFromURI(serviceConfig.cloudServiceAccountURI)
if err != nil {
return nil, fmt.Errorf("getting service account key: %w", err)
}
rawKey, err := json.Marshal(serviceAccountKey)
if err != nil {
return nil, fmt.Errorf("marshaling service account key: %w", err)
}
extraVals["ccm"] = map[string]any{
"GCP": map[string]any{
"projectID": projectID,
"uid": uid,
"secretData": string(rawKey),
"subnetworkPodCIDR": serviceConfig.subnetworkPodCIDR,
},
}
case cloudprovider.Azure:
ccmAzure, ok := k.providerMetadata.(ccmConfigGetter)
if !ok {
return nil, errors.New("invalid cloud provider metadata for Azure")
}
ccmConfig, err := ccmAzure.GetCCMConfig(ctx, instance.ProviderID, serviceConfig.cloudServiceAccountURI)
if err != nil {
return nil, fmt.Errorf("creating ccm secret: %w", err)
}
extraVals["ccm"] = map[string]any{
"Azure": map[string]any{
"azureConfig": string(ccmConfig),
},
}
case cloudprovider.OpenStack:
creds, err := openstack.AccountKeyFromURI(serviceConfig.cloudServiceAccountURI)
if err != nil {
return nil, err
}
credsIni := creds.CloudINI().FullConfiguration()
networkIDsGetter, ok := k.providerMetadata.(openstackMetadata)
if !ok {
return nil, errors.New("generating yawol configuration: cloud provider metadata does not implement OpenStack specific methods")
}
networkIDs, err := networkIDsGetter.GetNetworkIDs(ctx)
if err != nil {
return nil, fmt.Errorf("getting network IDs: %w", err)
}
if len(networkIDs) == 0 {
return nil, errors.New("getting network IDs: no network IDs found")
}
extraVals["ccm"] = map[string]any{
"OpenStack": map[string]any{
"secretData": credsIni,
},
}
yawolIni := creds.CloudINI().YawolConfiguration()
extraVals["yawol-config"] = map[string]any{
"secretData": yawolIni,
}
extraVals["yawol-controller"] = map[string]any{
"yawolNetworkID": networkIDs[0],
"yawolAPIHost": fmt.Sprintf("https://%s:%d", serviceConfig.loadBalancerIP, constants.KubernetesPort),
}
}
return extraVals, nil
}
func (k *KubeWrapper) setupOperatorVals(ctx context.Context) (map[string]any, error) {
uid, err := k.providerMetadata.UID(ctx)
if err != nil {
return nil, fmt.Errorf("retrieving constellation UID: %w", err)
}
return map[string]any{
"constellation-operator": map[string]any{
"constellationUID": uid,
},
}, nil
}
func (k *KubeWrapper) setupCiliumVals(ctx context.Context, in k8sapi.SetupPodNetworkInput) (map[string]any, error) {
vals := map[string]any{
"k8sServiceHost": in.LoadBalancerHost,
@ -585,18 +403,3 @@ func (k *KubeWrapper) setupCiliumVals(ctx context.Context, in k8sapi.SetupPodNet
return vals, nil
}
type ccmConfigGetter interface {
GetCCMConfig(ctx context.Context, providerID, cloudServiceAccountURI string) ([]byte, error)
}
type constellationServicesConfig struct {
measurementSalt []byte
subnetworkPodCIDR string
cloudServiceAccountURI string
loadBalancerIP string
}
type openstackMetadata interface {
GetNetworkIDs(ctx context.Context) ([]string, error)
}

6
bootstrapper/internal/kubernetes/kubernetes_test.go
View File

@ -35,8 +35,6 @@ func TestMain(m *testing.M) {
}
func TestInitCluster(t *testing.T) {
serviceAccountURI := "some-service-account-uri"
nodeName := "node-name"
providerID := "provider-id"
privateIP := "192.0.2.1"
@ -193,8 +191,8 @@ func TestInitCluster(t *testing.T) {
}
_, err := kube.InitCluster(
context.Background(), serviceAccountURI, string(tc.k8sVersion), "kubernetes",
nil, []byte("{}"), false, nil, nil, logger.NewTest(t),
context.Background(), string(tc.k8sVersion), "kubernetes",
[]byte("{}"), false, nil, nil, logger.NewTest(t),
)
if tc.wantErr {

2
cli/internal/cloudcmd/BUILD.bazel
View File

@ -34,7 +34,6 @@ go_library(
"@com_github_azure_azure_sdk_for_go//profiles/latest/attestation/attestation",
"@com_github_azure_azure_sdk_for_go_sdk_azcore//policy",
"@com_github_azure_azure_sdk_for_go_sdk_azidentity//:azidentity",
"@com_github_hashicorp_terraform_json//:terraform-json",
"@com_github_spf13_cobra//:cobra",
],
)
@ -59,7 +58,6 @@ go_test(
"//internal/cloud/cloudprovider",
"//internal/cloud/gcpshared",
"//internal/config",
"@com_github_hashicorp_terraform_json//:terraform-json",
"@com_github_stretchr_testify//assert",
"@com_github_stretchr_testify//require",
"@org_uber_go_goleak//:goleak",

22
cli/internal/cloudcmd/clients.go
View File

@ -13,7 +13,6 @@ import (
"github.com/edgelesssys/constellation/v2/cli/internal/terraform"
"github.com/edgelesssys/constellation/v2/internal/attestation/variant"
"github.com/edgelesssys/constellation/v2/internal/cloud/cloudprovider"
tfjson "github.com/hashicorp/terraform-json"
)
// imageFetcher gets an image reference from the versionsapi.
@ -24,14 +23,23 @@ type imageFetcher interface {
) (string, error)
}
type terraformClient interface {
PrepareWorkspace(path string, input terraform.Variables) error
ApplyIAMConfig(ctx context.Context, provider cloudprovider.Provider, logLevel terraform.LogLevel) (terraform.IAMOutput, error)
CreateCluster(ctx context.Context, logLevel terraform.LogLevel) (terraform.ApplyOutput, error)
Destroy(ctx context.Context, logLevel terraform.LogLevel) error
type tfCommonClient interface {
CleanUpWorkspace() error
Destroy(ctx context.Context, logLevel terraform.LogLevel) error
PrepareWorkspace(path string, input terraform.Variables) error
RemoveInstaller()
Show(ctx context.Context) (*tfjson.State, error)
}
type tfResourceClient interface {
tfCommonClient
CreateCluster(ctx context.Context, provider cloudprovider.Provider, logLevel terraform.LogLevel) (terraform.ApplyOutput, error)
ShowCluster(ctx context.Context, provider cloudprovider.Provider) (terraform.ApplyOutput, error)
}
type tfIAMClient interface {
tfCommonClient
ApplyIAMConfig(ctx context.Context, provider cloudprovider.Provider, logLevel terraform.LogLevel) (terraform.IAMOutput, error)
ShowIAM(ctx context.Context, provider cloudprovider.Provider) (terraform.IAMOutput, error)
}
type libvirtRunner interface {

24
cli/internal/cloudcmd/clients_test.go
View File

@ -14,7 +14,6 @@ import (
"github.com/edgelesssys/constellation/v2/cli/internal/terraform"
"github.com/edgelesssys/constellation/v2/internal/attestation/variant"
"github.com/edgelesssys/constellation/v2/internal/cloud/cloudprovider"
tfjson "github.com/hashicorp/terraform-json"
"go.uber.org/goleak"
)
@ -32,7 +31,7 @@ type stubTerraformClient struct {
iamOutput terraform.IAMOutput
uid string
attestationURL string
tfjsonState *tfjson.State
applyOutput terraform.ApplyOutput
cleanUpWorkspaceCalled bool
removeInstallerCalled bool
destroyCalled bool
@ -45,12 +44,14 @@ type stubTerraformClient struct {
showErr error
}
func (c *stubTerraformClient) CreateCluster(_ context.Context, _ terraform.LogLevel) (terraform.ApplyOutput, error) {
func (c *stubTerraformClient) CreateCluster(_ context.Context, _ cloudprovider.Provider, _ terraform.LogLevel) (terraform.ApplyOutput, error) {
return terraform.ApplyOutput{
IP: c.ip,
Secret: c.initSecret,
UID: c.uid,
AttestationURL: c.attestationURL,
IP: c.ip,
Secret: c.initSecret,
UID: c.uid,
Azure: &terraform.AzureApplyOutput{
AttestationURL: c.attestationURL,
},
}, c.createClusterErr
}
@ -76,9 +77,14 @@ func (c *stubTerraformClient) RemoveInstaller() {
c.removeInstallerCalled = true
}
func (c *stubTerraformClient) Show(_ context.Context) (*tfjson.State, error) {
func (c *stubTerraformClient) ShowCluster(_ context.Context, _ cloudprovider.Provider) (terraform.ApplyOutput, error) {
c.showCalled = true
return c.tfjsonState, c.showErr
return c.applyOutput, c.showErr
}
func (c *stubTerraformClient) ShowIAM(_ context.Context, _ cloudprovider.Provider) (terraform.IAMOutput, error) {
c.showCalled = true
return c.iamOutput, c.showErr
}
type stubLibvirtRunner struct {

35
cli/internal/cloudcmd/create.go
View File

@ -31,7 +31,7 @@ import (
type Creator struct {
out io.Writer
image imageFetcher
newTerraformClient func(ctx context.Context) (terraformClient, error)
newTerraformClient func(ctx context.Context) (tfResourceClient, error)
newLibvirtRunner func() libvirtRunner
newRawDownloader func() rawDownloader
policyPatcher policyPatcher
@ -42,7 +42,7 @@ func NewCreator(out io.Writer) *Creator {
return &Creator{
out: out,
image: imagefetcher.New(),
newTerraformClient: func(ctx context.Context) (terraformClient, error) {
newTerraformClient: func(ctx context.Context) (tfResourceClient, error) {
return terraform.New(ctx, constants.TerraformWorkingDir)
},
newLibvirtRunner: func() libvirtRunner {
@ -115,18 +115,20 @@ func (c *Creator) Create(ctx context.Context, opts CreateOptions) (clusterid.Fil
if err != nil {
return clusterid.File{}, fmt.Errorf("creating cluster: %w", err)
}
return clusterid.File{
res := clusterid.File{
CloudProvider: opts.Provider,
IP: tfOutput.IP,
APIServerCertSANs: tfOutput.APIServerCertSANs,
InitSecret: []byte(tfOutput.Secret),
UID: tfOutput.UID,
AttestationURL: tfOutput.AttestationURL,
}, nil
}
if tfOutput.Azure != nil {
res.AttestationURL = tfOutput.Azure.AttestationURL
}
return res, nil
}
func (c *Creator) createAWS(ctx context.Context, cl terraformClient, opts CreateOptions) (tfOutput terraform.ApplyOutput, retErr error) {
func (c *Creator) createAWS(ctx context.Context, cl tfResourceClient, opts CreateOptions) (tfOutput terraform.ApplyOutput, retErr error) {
vars := awsTerraformVars(opts.Config, opts.image, &opts.ControlPlaneCount, &opts.WorkerCount)
tfOutput, err := runTerraformCreate(ctx, cl, cloudprovider.AWS, vars, c.out, opts.TFLogLevel)
@ -137,7 +139,7 @@ func (c *Creator) createAWS(ctx context.Context, cl terraformClient, opts Create
return tfOutput, nil
}
func (c *Creator) createGCP(ctx context.Context, cl terraformClient, opts CreateOptions) (tfOutput terraform.ApplyOutput, retErr error) {
func (c *Creator) createGCP(ctx context.Context, cl tfResourceClient, opts CreateOptions) (tfOutput terraform.ApplyOutput, retErr error) {
vars := gcpTerraformVars(opts.Config, opts.image, &opts.ControlPlaneCount, &opts.WorkerCount)
tfOutput, err := runTerraformCreate(ctx, cl, cloudprovider.GCP, vars, c.out, opts.TFLogLevel)
@ -148,7 +150,7 @@ func (c *Creator) createGCP(ctx context.Context, cl terraformClient, opts Create
return tfOutput, nil
}
func (c *Creator) createAzure(ctx context.Context, cl terraformClient, opts CreateOptions) (tfOutput terraform.ApplyOutput, retErr error) {
func (c *Creator) createAzure(ctx context.Context, cl tfResourceClient, opts CreateOptions) (tfOutput terraform.ApplyOutput, retErr error) {
vars := azureTerraformVars(opts.Config, opts.image, &opts.ControlPlaneCount, &opts.WorkerCount)
tfOutput, err := runTerraformCreate(ctx, cl, cloudprovider.Azure, vars, c.out, opts.TFLogLevel)
@ -158,7 +160,10 @@ func (c *Creator) createAzure(ctx context.Context, cl terraformClient, opts Crea
if vars.GetCreateMAA() {
// Patch the attestation policy to allow the cluster to boot while having secure boot disabled.
if err := c.policyPatcher.Patch(ctx, tfOutput.AttestationURL); err != nil {
if tfOutput.Azure == nil {
return terraform.ApplyOutput{}, errors.New("no Terraform Azure output found")
}
if err := c.policyPatcher.Patch(ctx, tfOutput.Azure.AttestationURL); err != nil {
return terraform.ApplyOutput{}, err
}
}
@ -197,7 +202,7 @@ func normalizeAzureURIs(vars *terraform.AzureClusterVariables) *terraform.AzureC
return vars
}
func (c *Creator) createOpenStack(ctx context.Context, cl terraformClient, opts CreateOptions) (tfOutput terraform.ApplyOutput, retErr error) {
func (c *Creator) createOpenStack(ctx context.Context, cl tfResourceClient, opts CreateOptions) (tfOutput terraform.ApplyOutput, retErr error) {
// TODO(malt3): Remove this once OpenStack is supported.
if os.Getenv("CONSTELLATION_OPENSTACK_DEV") != "1" {
return terraform.ApplyOutput{}, errors.New("OpenStack isn't supported yet")
@ -221,13 +226,13 @@ func (c *Creator) createOpenStack(ctx context.Context, cl terraformClient, opts
return tfOutput, nil
}
func runTerraformCreate(ctx context.Context, cl terraformClient, provider cloudprovider.Provider, vars terraform.Variables, outWriter io.Writer, loglevel terraform.LogLevel) (output terraform.ApplyOutput, retErr error) {
func runTerraformCreate(ctx context.Context, cl tfResourceClient, provider cloudprovider.Provider, vars terraform.Variables, outWriter io.Writer, loglevel terraform.LogLevel) (output terraform.ApplyOutput, retErr error) {
if err := cl.PrepareWorkspace(path.Join("terraform", strings.ToLower(provider.String())), vars); err != nil {
return terraform.ApplyOutput{}, err
}
defer rollbackOnError(outWriter, &retErr, &rollbackerTerraform{client: cl}, loglevel)
tfOutput, err := cl.CreateCluster(ctx, loglevel)
tfOutput, err := cl.CreateCluster(ctx, provider, loglevel)
if err != nil {
return terraform.ApplyOutput{}, err
}
@ -240,7 +245,7 @@ type qemuCreateOptions struct {
CreateOptions
}
func (c *Creator) createQEMU(ctx context.Context, cl terraformClient, lv libvirtRunner, opts qemuCreateOptions) (tfOutput terraform.ApplyOutput, retErr error) {
func (c *Creator) createQEMU(ctx context.Context, cl tfResourceClient, lv libvirtRunner, opts qemuCreateOptions) (tfOutput terraform.ApplyOutput, retErr error) {
qemuRollbacker := &rollbackerQEMU{client: cl, libvirt: lv, createdWorkspace: false}
defer rollbackOnError(c.out, &retErr, qemuRollbacker, opts.TFLogLevel)
@ -300,7 +305,7 @@ func (c *Creator) createQEMU(ctx context.Context, cl terraformClient, lv libvirt
// Allow rollback of QEMU Terraform workspace from this point on
qemuRollbacker.createdWorkspace = true
tfOutput, err = cl.CreateCluster(ctx, opts.TFLogLevel)
tfOutput, err = cl.CreateCluster(ctx, opts.Provider, opts.TFLogLevel)
if err != nil {
return terraform.ApplyOutput{}, fmt.Errorf("create cluster: %w", err)
}

4
cli/internal/cloudcmd/create_test.go
View File

@ -28,7 +28,7 @@ func TestCreator(t *testing.T) {
someErr := errors.New("failed")
testCases := map[string]struct {
tfClient terraformClient
tfClient tfResourceClient
newTfClientErr error
libvirt *stubLibvirtRunner
provider cloudprovider.Provider
@ -203,7 +203,7 @@ func TestCreator(t *testing.T) {
image: &stubImageFetcher{
reference: "some-image",
},
newTerraformClient: func(ctx context.Context) (terraformClient, error) {
newTerraformClient: func(ctx context.Context) (tfResourceClient, error) {
return tc.tfClient, tc.newTfClientErr
},
newLibvirtRunner: func() libvirtRunner {

50
cli/internal/cloudcmd/iam.go
View File

@ -24,7 +24,7 @@ import (
// IAMDestroyer destroys an IAM configuration.
type IAMDestroyer struct {
client terraformClient
client tfIAMClient
}
// NewIAMDestroyer creates a new IAM Destroyer.
@ -38,35 +38,23 @@ func NewIAMDestroyer(ctx context.Context) (*IAMDestroyer, error) {
// GetTfstateServiceAccountKey returns the sa_key output from the terraform state.
func (d *IAMDestroyer) GetTfstateServiceAccountKey(ctx context.Context) (gcpshared.ServiceAccountKey, error) {
tfState, err := d.client.Show(ctx)
tfState, err := d.client.ShowIAM(ctx, cloudprovider.GCP)
if err != nil {
return gcpshared.ServiceAccountKey{}, err
return gcpshared.ServiceAccountKey{}, fmt.Errorf("getting terraform state: %w", err)
}
if saKeyString := tfState.GCP.SaKey; saKeyString != "" {
saKey, err := base64.StdEncoding.DecodeString(saKeyString)
if err != nil {
return gcpshared.ServiceAccountKey{}, err
}
var tfSaKey gcpshared.ServiceAccountKey
if err := json.Unmarshal(saKey, &tfSaKey); err != nil {
return gcpshared.ServiceAccountKey{}, err
}
if tfState.Values == nil {
return gcpshared.ServiceAccountKey{}, errors.New("no Values field in terraform state")
return tfSaKey, nil
}
saKeyJSON := tfState.Values.Outputs["sa_key"]
if saKeyJSON == nil {
return gcpshared.ServiceAccountKey{}, errors.New("no sa_key in outputs of the terraform state")
}
saKeyString, ok := saKeyJSON.Value.(string)
if !ok {
return gcpshared.ServiceAccountKey{}, errors.New("sa_key field in terraform state is not a string")
}
saKey, err := base64.StdEncoding.DecodeString(saKeyString)
if err != nil {
return gcpshared.ServiceAccountKey{}, err
}
var tfSaKey gcpshared.ServiceAccountKey
if err := json.Unmarshal(saKey, &tfSaKey); err != nil {
return gcpshared.ServiceAccountKey{}, err
}
return tfSaKey, nil
return gcpshared.ServiceAccountKey{}, errors.New("no saKey in terraform state")
}
// DestroyIAMConfiguration destroys the previously created IAM configuration and deletes the local IAM terraform files.
@ -80,7 +68,7 @@ func (d *IAMDestroyer) DestroyIAMConfiguration(ctx context.Context, logLevel ter
// IAMCreator creates the IAM configuration on the cloud provider.
type IAMCreator struct {
out io.Writer
newTerraformClient func(ctx context.Context) (terraformClient, error)
newTerraformClient func(ctx context.Context) (tfIAMClient, error)
}
// IAMConfigOptions holds the necessary values for IAM configuration.
@ -116,7 +104,7 @@ type AWSIAMConfig struct {
func NewIAMCreator(out io.Writer) *IAMCreator {
return &IAMCreator{
out: out,
newTerraformClient: func(ctx context.Context) (terraformClient, error) {
newTerraformClient: func(ctx context.Context) (tfIAMClient, error) {
return terraform.New(ctx, constants.TerraformIAMWorkingDir)
},
}
@ -152,7 +140,7 @@ func (c *IAMCreator) Create(ctx context.Context, provider cloudprovider.Provider
}
// createGCP creates the IAM configuration on GCP.
func (c *IAMCreator) createGCP(ctx context.Context, cl terraformClient, opts *IAMConfigOptions) (retFile iamid.File, retErr error) {
func (c *IAMCreator) createGCP(ctx context.Context, cl tfIAMClient, opts *IAMConfigOptions) (retFile iamid.File, retErr error) {
defer rollbackOnError(c.out, &retErr, &rollbackerTerraform{client: cl}, opts.TFLogLevel)
vars := terraform.GCPIAMVariables{
@ -180,7 +168,7 @@ func (c *IAMCreator) createGCP(ctx context.Context, cl terraformClient, opts *IA
}
// createAzure creates the IAM configuration on Azure.
func (c *IAMCreator) createAzure(ctx context.Context, cl terraformClient, opts *IAMConfigOptions) (retFile iamid.File, retErr error) {
func (c *IAMCreator) createAzure(ctx context.Context, cl tfIAMClient, opts *IAMConfigOptions) (retFile iamid.File, retErr error) {
defer rollbackOnError(c.out, &retErr, &rollbackerTerraform{client: cl}, opts.TFLogLevel)
vars := terraform.AzureIAMVariables{
@ -209,7 +197,7 @@ func (c *IAMCreator) createAzure(ctx context.Context, cl terraformClient, opts *
}
// createAWS creates the IAM configuration on AWS.
func (c *IAMCreator) createAWS(ctx context.Context, cl terraformClient, opts *IAMConfigOptions) (retFile iamid.File, retErr error) {
func (c *IAMCreator) createAWS(ctx context.Context, cl tfIAMClient, opts *IAMConfigOptions) (retFile iamid.File, retErr error) {
defer rollbackOnError(c.out, &retErr, &rollbackerTerraform{client: cl}, opts.TFLogLevel)
vars := terraform.AWSIAMVariables{

63
cli/internal/cloudcmd/iam_test.go
View File

@ -17,7 +17,6 @@ import (
"github.com/edgelesssys/constellation/v2/cli/internal/terraform"
"github.com/edgelesssys/constellation/v2/internal/cloud/cloudprovider"
"github.com/edgelesssys/constellation/v2/internal/cloud/gcpshared"
tfjson "github.com/hashicorp/terraform-json"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
@ -83,7 +82,7 @@ func TestIAMCreator(t *testing.T) {
}
testCases := map[string]struct {
tfClient terraformClient
tfClient tfIAMClient
newTfClientErr error
config *IAMConfigOptions
provider cloudprovider.Provider
@ -125,7 +124,7 @@ func TestIAMCreator(t *testing.T) {
creator := &IAMCreator{
out: &bytes.Buffer{},
newTerraformClient: func(ctx context.Context) (terraformClient, error) {
newTerraformClient: func(ctx context.Context) (tfIAMClient, error) {
return tc.tfClient, tc.newTfClientErr
},
}
@ -225,13 +224,9 @@ func TestGetTfstateServiceAccountKey(t *testing.T) {
}{
"valid": {
cl: &stubTerraformClient{
tfjsonState: &tfjson.State{
Values: &tfjson.StateValues{
Outputs: map[string]*tfjson.StateOutput{
"sa_key": {
Value: gcpFileB64,
},
},
iamOutput: terraform.IAMOutput{
GCP: terraform.GCPIAMOutput{
SaKey: gcpFileB64,
},
},
},
@ -247,31 +242,16 @@ func TestGetTfstateServiceAccountKey(t *testing.T) {
},
"nil tfstate values": {
cl: &stubTerraformClient{
tfjsonState: &tfjson.State{
Values: nil,
},
},
wantErr: true,
wantShowCalled: true,
},
"no key": {
cl: &stubTerraformClient{
tfjsonState: &tfjson.State{
Values: &tfjson.StateValues{},
},
iamOutput: terraform.IAMOutput{},
},
wantErr: true,
wantShowCalled: true,
},
"invalid base64": {
cl: &stubTerraformClient{
tfjsonState: &tfjson.State{
Values: &tfjson.StateValues{
Outputs: map[string]*tfjson.StateOutput{
"sa_key": {
Value: "iamnotvalid",
},
},
iamOutput: terraform.IAMOutput{
GCP: terraform.GCPIAMOutput{
SaKey: "iamnotvalid",
},
},
},
@ -280,28 +260,9 @@ func TestGetTfstateServiceAccountKey(t *testing.T) {
},
"valid base64 invalid json": {
cl: &stubTerraformClient{
tfjsonState: &tfjson.State{
Values: &tfjson.StateValues{
Outputs: map[string]*tfjson.StateOutput{
"sa_key": {
Value: base64.StdEncoding.EncodeToString([]byte("asdf")),
},
},
},
},
},
wantErr: true,
wantShowCalled: true,
},
"not string": {
cl: &stubTerraformClient{
tfjsonState: &tfjson.State{
Values: &tfjson.StateValues{
Outputs: map[string]*tfjson.StateOutput{
"sa_key": {
Value: 1,
},
},
iamOutput: terraform.IAMOutput{
GCP: terraform.GCPIAMOutput{
SaKey: base64.StdEncoding.EncodeToString([]byte("asdf")),
},
},
},

4
cli/internal/cloudcmd/rollback.go
View File

@ -36,7 +36,7 @@ func rollbackOnError(w io.Writer, onErr *error, roll rollbacker, logLevel terraf
}
type rollbackerTerraform struct {
client terraformClient
client tfCommonClient
}
func (r *rollbackerTerraform) rollback(ctx context.Context, logLevel terraform.LogLevel) error {
@ -47,7 +47,7 @@ func (r *rollbackerTerraform) rollback(ctx context.Context, logLevel terraform.L
}
type rollbackerQEMU struct {
client terraformClient
client tfResourceClient
libvirt libvirtRunner
createdWorkspace bool
}

6
cli/internal/cloudcmd/terminate.go
View File

@ -16,14 +16,14 @@ import (
// Terminator deletes cloud provider resources.
type Terminator struct {
newTerraformClient func(ctx context.Context) (terraformClient, error)
newTerraformClient func(ctx context.Context) (tfResourceClient, error)
newLibvirtRunner func() libvirtRunner
}
// NewTerminator create a new cloud terminator.
func NewTerminator() *Terminator {
return &Terminator{
newTerraformClient: func(ctx context.Context) (terraformClient, error) {
newTerraformClient: func(ctx context.Context) (tfResourceClient, error) {
return terraform.New(ctx, constants.TerraformWorkingDir)
},
newLibvirtRunner: func() libvirtRunner {
@ -49,7 +49,7 @@ func (t *Terminator) Terminate(ctx context.Context, logLevel terraform.LogLevel)
return t.terminateTerraform(ctx, cl, logLevel)
}
func (t *Terminator) terminateTerraform(ctx context.Context, cl terraformClient, logLevel terraform.LogLevel) error {
func (t *Terminator) terminateTerraform(ctx context.Context, cl tfResourceClient, logLevel terraform.LogLevel) error {
if err := cl.Destroy(ctx, logLevel); err != nil {
return err
}

4
cli/internal/cloudcmd/terminate_test.go
View File

@ -19,7 +19,7 @@ func TestTerminator(t *testing.T) {
someErr := errors.New("failed")
testCases := map[string]struct {
tfClient terraformClient
tfClient tfResourceClient
newTfClientErr error
libvirt *stubLibvirtRunner
wantErr bool
@ -55,7 +55,7 @@ func TestTerminator(t *testing.T) {
assert := assert.New(t)
terminator := &Terminator{
newTerraformClient: func(ctx context.Context) (terraformClient, error) {
newTerraformClient: func(ctx context.Context) (tfResourceClient, error) {
return tc.tfClient, tc.newTfClientErr
},
newLibvirtRunner: func() libvirtRunner {

1
cli/internal/cmd/BUILD.bazel
View File

@ -153,6 +153,7 @@ go_test(
"//internal/config",
"//internal/constants",
"//internal/crypto/testvector",
"//internal/deploy/helm",
"//internal/file",
"//internal/grpc/atlscredentials",
"//internal/grpc/dialer",

40
cli/internal/cmd/init.go
View File

@ -7,8 +7,10 @@ SPDX-License-Identifier: AGPL-3.0-only
package cmd
import (
"bytes"
"context"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"io"
@ -72,11 +74,12 @@ func NewInitCmd() *cobra.Command {
}
type initCmd struct {
log debugLog
merger configMerger
spinner spinnerInterf
masterSecret uri.MasterSecret
fh *file.Handler
log debugLog
merger configMerger
spinner spinnerInterf
masterSecret uri.MasterSecret
fh *file.Handler
helmInstaller helm.SuiteInstaller
}
// runInitialize runs the initialize command.
@ -100,7 +103,11 @@ func runInitialize(cmd *cobra.Command, _ []string) error {
ctx, cancel := context.WithTimeout(cmd.Context(), time.Hour)
defer cancel()
cmd.SetContext(ctx)
i := &initCmd{log: log, spinner: spinner, merger: &kubeconfigMerger{log: log}, fh: &fileHandler}
helmInstaller, err := helm.NewInstallationClient(log)
if err != nil {
return fmt.Errorf("creating Helm installer: %w", err)
}
i := &initCmd{log: log, spinner: spinner, merger: &kubeconfigMerger{log: log}, fh: &fileHandler, helmInstaller: helmInstaller}
fetcher := attestationconfigapi.NewFetcher()
return i.initialize(cmd, newDialer, fileHandler, license.NewClient(), fetcher)
}
@ -180,7 +187,14 @@ func (i *initCmd) initialize(cmd *cobra.Command, newDialer func(validator atls.V
helmLoader := helm.NewLoader(provider, k8sVersion, clusterName)
i.log.Debugf("Created new Helm loader")
helmDeployments, err := helmLoader.Load(conf, flags.conformance, flags.helmWaitMode, masterSecret.Key, masterSecret.Salt)
releases, err := helmLoader.LoadReleases(conf, flags.conformance, flags.helmWaitMode, masterSecret.Key, masterSecret.Salt)
if err != nil {
return fmt.Errorf("loading Helm charts: %w", err)
}
helmDeployments, err := json.Marshal(releases)
if err != nil {
return err
}
i.log.Debugf("Loaded Helm deployments")
if err != nil {
return fmt.Errorf("loading Helm charts: %w", err)
@ -214,10 +228,20 @@ func (i *initCmd) initialize(cmd *cobra.Command, newDialer func(validator atls.V
return err
}
i.log.Debugf("Initialization request succeeded")
i.log.Debugf("Writing Constellation ID file")
idFile.CloudProvider = provider
return i.writeOutput(idFile, resp, flags.mergeConfigs, cmd.OutOrStdout(), fileHandler)
bufferedOutput := &bytes.Buffer{}
err = i.writeOutput(idFile, resp, flags.mergeConfigs, bufferedOutput, fileHandler)
if err != nil {
return err
}
if err := i.helmInstaller.Install(cmd.Context(), provider, masterSecret, idFile, serviceAccURI, releases); err != nil {
return fmt.Errorf("installing Helm charts: %w", err)
}
cmd.Println(bufferedOutput.String())
return nil
}
func (i *initCmd) initCall(ctx context.Context, dialer grpcDialer, ip string, req *initproto.InitRequest) (*initproto.InitSuccessResponse, error) {

12
cli/internal/cmd/init_test.go
View File

@ -28,6 +28,7 @@ import (
"github.com/edgelesssys/constellation/v2/internal/cloud/gcpshared"
"github.com/edgelesssys/constellation/v2/internal/config"
"github.com/edgelesssys/constellation/v2/internal/constants"
helminstaller "github.com/edgelesssys/constellation/v2/internal/deploy/helm"
"github.com/edgelesssys/constellation/v2/internal/file"
"github.com/edgelesssys/constellation/v2/internal/grpc/atlscredentials"
"github.com/edgelesssys/constellation/v2/internal/grpc/dialer"
@ -174,7 +175,7 @@ func TestInitialize(t *testing.T) {
ctx, cancel := context.WithTimeout(ctx, 4*time.Second)
defer cancel()
cmd.SetContext(ctx)
i := &initCmd{log: logger.NewTest(t), spinner: &nopSpinner{}}
i := &initCmd{log: logger.NewTest(t), spinner: &nopSpinner{}, helmInstaller: &stubHelmInstaller{}}
err := i.initialize(cmd, newDialer, fileHandler, &stubLicenseClient{}, stubAttestationFetcher{})
if tc.wantErr {
@ -666,3 +667,12 @@ func (c stubInitClient) Recv() (*initproto.InitResponse, error) {
return res, err
}
type stubHelmInstaller struct{}
func (i *stubHelmInstaller) Install(_ context.Context, _ cloudprovider.Provider, _ uri.MasterSecret,
_ clusterid.File,
_ string, _ *helminstaller.Releases,
) error {
return nil
}

2
cli/internal/cmd/tfmigrationclient.go
View File

@ -38,7 +38,7 @@ func (u *tfMigrationClient) planMigration(cmd *cobra.Command, file file.Handler,
func (u *tfMigrationClient) applyMigration(cmd *cobra.Command, file file.Handler, migrateCmd upgrade.TfMigrationCmd, yesFlag bool) error {
hasDiff, err := u.planMigration(cmd, file, migrateCmd)
if err != nil {
return fmt.Errorf("planning terraform migrations: %w", err)
return err
}
if hasDiff {
// If there are any Terraform migrations to apply, ask for confirmation

7
cli/internal/helm/BUILD.bazel
View File

@ -7,8 +7,10 @@ go_library(
"backup.go",
"client.go",
"helm.go",
"helminstaller.go",
"loader.go",
"serviceversion.go",
"setup.go",
"values.go",
],
embedsrcs = [
@ -417,12 +419,17 @@ go_library(
deps = [
"//cli/internal/clusterid",
"//cli/internal/helm/imageversion",
"//cli/internal/terraform",
"//internal/cloud/azureshared",
"//internal/cloud/cloudprovider",
"//internal/cloud/gcpshared",
"//internal/cloud/openstack",
"//internal/compatibility",
"//internal/config",
"//internal/constants",
"//internal/deploy/helm",
"//internal/file",
"//internal/kms/uri",
"//internal/semver",
"//internal/versions",
"@com_github_pkg_errors//:errors",

100
cli/internal/helm/helminstaller.go Normal file
View File

@ -0,0 +1,100 @@
/*
Copyright (c) Edgeless Systems GmbH
SPDX-License-Identifier: AGPL-3.0-only
*/
package helm
import (
"context"
"fmt"
"github.com/edgelesssys/constellation/v2/cli/internal/clusterid"
"github.com/edgelesssys/constellation/v2/internal/cloud/cloudprovider"
"github.com/edgelesssys/constellation/v2/internal/cloud/openstack"
"github.com/edgelesssys/constellation/v2/internal/constants"
helminstaller "github.com/edgelesssys/constellation/v2/internal/deploy/helm"
"github.com/edgelesssys/constellation/v2/internal/kms/uri"
)
// SuiteInstaller installs all Helm charts required for a constellation cluster.
type SuiteInstaller interface {
Install(ctx context.Context, provider cloudprovider.Provider, masterSecret uri.MasterSecret,
idFile clusterid.File,
serviceAccURI string, releases *helminstaller.Releases,
) error
}
type helmInstallationClient struct {
log debugLog
installer helmInstaller
}
// NewInstallationClient creates a new Helm installation client to install all Helm charts required for a constellation cluster.
func NewInstallationClient(log debugLog) (SuiteInstaller, error) {
installer, err := helminstaller.NewInstaller(constants.AdminConfFilename, log)
if err != nil {
return nil, fmt.Errorf("creating Helm installer: %w", err)
}
return &helmInstallationClient{log: log, installer: installer}, nil
}
func (h helmInstallationClient) Install(ctx context.Context, provider cloudprovider.Provider, masterSecret uri.MasterSecret,
idFile clusterid.File,
serviceAccURI string, releases *helminstaller.Releases,
) error {
serviceVals, err := setupMicroserviceVals(ctx, provider, masterSecret.Salt, idFile.UID, serviceAccURI)
if err != nil {
return fmt.Errorf("setting up microservice values: %w", err)
}
if err := h.installer.InstallChartWithValues(ctx, releases.ConstellationServices, serviceVals); err != nil {
return fmt.Errorf("installing microservices: %w", err)
}
h.log.Debugf("Installing cert-manager")
if err := h.installer.InstallChart(ctx, releases.CertManager); err != nil {
return fmt.Errorf("installing cert-manager: %w", err)
}
if releases.CSI != nil {
var csiVals map[string]any
if provider == cloudprovider.OpenStack {
creds, err := openstack.AccountKeyFromURI(serviceAccURI)
if err != nil {
return err
}
cinderIni := creds.CloudINI().CinderCSIConfiguration()
csiVals = map[string]any{
"cinder-config": map[string]any{
"secretData": cinderIni,
},
}
}
h.log.Debugf("Installing CSI deployments")
if err := h.installer.InstallChartWithValues(ctx, *releases.CSI, csiVals); err != nil {
return fmt.Errorf("installing CSI snapshot CRDs: %w", err)
}
}
if releases.AWSLoadBalancerController != nil {
h.log.Debugf("Installing AWS Load Balancer Controller")
if err := h.installer.InstallChart(ctx, *releases.AWSLoadBalancerController); err != nil {
return fmt.Errorf("installing AWS Load Balancer Controller: %w", err)
}
}
h.log.Debugf("Installing constellation operators")
operatorVals := setupOperatorVals(ctx, idFile.UID)
if err := h.installer.InstallChartWithValues(ctx, releases.ConstellationOperators, operatorVals); err != nil {
return fmt.Errorf("installing constellation operators: %w", err)
}
// TODO(elchead): AB#3301 do cilium after version upgrade
return nil
}
type helmInstaller interface {
InstallChart(context.Context, helminstaller.Release) error
InstallChartWithValues(ctx context.Context, release helminstaller.Release, extraValues map[string]any) error
}

20
cli/internal/helm/loader.go
View File

@ -110,6 +110,19 @@ func NewLoader(csp cloudprovider.Provider, k8sVersion versions.ValidK8sVersion,
// Load the embedded helm charts.
func (i *ChartLoader) Load(config *config.Config, conformanceMode bool, helmWaitMode helm.WaitMode, masterSecret, salt []byte) ([]byte, error) {
releases, err := i.LoadReleases(config, conformanceMode, helmWaitMode, masterSecret, salt)
if err != nil {
return nil, fmt.Errorf("loading releases: %w", err)
}
rel, err := json.Marshal(releases)
if err != nil {
return nil, err
}
return rel, nil
}
// LoadReleases loads the embedded helm charts and returns them as a HelmReleases object.
func (i *ChartLoader) LoadReleases(config *config.Config, conformanceMode bool, helmWaitMode helm.WaitMode, masterSecret, salt []byte) (*helm.Releases, error) {
ciliumRelease, err := i.loadRelease(ciliumInfo, helmWaitMode)
if err != nil {
return nil, fmt.Errorf("loading cilium: %w", err)
@ -150,12 +163,7 @@ func (i *ChartLoader) Load(config *config.Config, conformanceMode bool, helmWait
}
releases.CSI = &csi
}
rel, err := json.Marshal(releases)
if err != nil {
return nil, err
}
return rel, nil
return &releases, nil
}
// loadRelease loads the embedded chart and values depending on the given info argument.

134
cli/internal/helm/setup.go Normal file
View File

@ -0,0 +1,134 @@
/*
Copyright (c) Edgeless Systems GmbH
SPDX-License-Identifier: AGPL-3.0-only
*/
package helm
import (
"context"
"encoding/base64"
"encoding/json"
"fmt"
"github.com/edgelesssys/constellation/v2/cli/internal/terraform"
"github.com/edgelesssys/constellation/v2/internal/cloud/azureshared"
"github.com/edgelesssys/constellation/v2/internal/cloud/cloudprovider"
"github.com/edgelesssys/constellation/v2/internal/cloud/gcpshared"
"github.com/edgelesssys/constellation/v2/internal/constants"
)
// setupMicroserviceVals returns the values for the microservice chart.
func setupMicroserviceVals(ctx context.Context, provider cloudprovider.Provider, measurementSalt []byte, uid, serviceAccURI string) (map[string]any, error) {
tfClient, err := terraform.New(ctx, constants.TerraformWorkingDir)
if err != nil {
return nil, fmt.Errorf("creating Terraform client: %w", err)
}
output, err := tfClient.ShowCluster(ctx, provider)
if err != nil {
return nil, fmt.Errorf("getting Terraform output: %w", err)
}
extraVals := map[string]any{
"join-service": map[string]any{
"measurementSalt": base64.StdEncoding.EncodeToString(measurementSalt),
},
"verification-service": map[string]any{
"loadBalancerIP": output.IP,
},
"konnectivity": map[string]any{
"loadBalancerIP": output.IP,
},
}
switch provider {
case cloudprovider.GCP:
serviceAccountKey, err := gcpshared.ServiceAccountKeyFromURI(serviceAccURI)
if err != nil {
return nil, fmt.Errorf("getting service account key: %w", err)
}
rawKey, err := json.Marshal(serviceAccountKey)
if err != nil {
return nil, fmt.Errorf("marshaling service account key: %w", err)
}
if output.GCP == nil {
return nil, fmt.Errorf("no GCP output from Terraform")
}
extraVals["ccm"] = map[string]any{
"GCP": map[string]any{
"projectID": output.GCP.ProjectID,
"uid": uid,
"secretData": string(rawKey),
"subnetworkPodCIDR": output.GCP.IPCidrPod,
},
}
case cloudprovider.Azure:
if output.Azure == nil {
return nil, fmt.Errorf("no Azure output from Terraform")
}
ccmConfig, err := getCCMConfig(*output.Azure, serviceAccURI)
if err != nil {
return nil, fmt.Errorf("getting Azure CCM config: %w", err)
}
extraVals["ccm"] = map[string]any{
"Azure": map[string]any{
"azureConfig": string(ccmConfig),
},
}
}
return extraVals, nil
}
// setupOperatorVals returns the values for the constellation-operator chart.
func setupOperatorVals(_ context.Context, uid string) map[string]any {
return map[string]any{
"constellation-operator": map[string]any{
"constellationUID": uid,
},
}
}
type cloudConfig struct {
Cloud string `json:"cloud,omitempty"`
TenantID string `json:"tenantId,omitempty"`
SubscriptionID string `json:"subscriptionId,omitempty"`
ResourceGroup string `json:"resourceGroup,omitempty"`
Location string `json:"location,omitempty"`
SubnetName string `json:"subnetName,omitempty"`
SecurityGroupName string `json:"securityGroupName,omitempty"`
SecurityGroupResourceGroup string `json:"securityGroupResourceGroup,omitempty"`
LoadBalancerName string `json:"loadBalancerName,omitempty"`
LoadBalancerSku string `json:"loadBalancerSku,omitempty"`
VNetName string `json:"vnetName,omitempty"`
VNetResourceGroup string `json:"vnetResourceGroup,omitempty"`
CloudProviderBackoff bool `json:"cloudProviderBackoff,omitempty"`
UseInstanceMetadata bool `json:"useInstanceMetadata,omitempty"`
VMType string `json:"vmType,omitempty"`
UseManagedIdentityExtension bool `json:"useManagedIdentityExtension,omitempty"`
UserAssignedIdentityID string `json:"userAssignedIdentityID,omitempty"`
}
// GetCCMConfig returns the configuration needed for the Kubernetes Cloud Controller Manager on Azure.
func getCCMConfig(tfOutput terraform.AzureApplyOutput, serviceAccURI string) ([]byte, error) {
creds, err := azureshared.ApplicationCredentialsFromURI(serviceAccURI)
if err != nil {
return nil, fmt.Errorf("getting service account key: %w", err)
}
useManagedIdentityExtension := creds.PreferredAuthMethod == azureshared.AuthMethodUserAssignedIdentity
config := cloudConfig{
Cloud: "AzurePublicCloud",
TenantID: creds.TenantID,
SubscriptionID: tfOutput.SubscriptionID,
ResourceGroup: tfOutput.ResourceGroup,
LoadBalancerSku: "standard",
SecurityGroupName: tfOutput.NetworkSecurityGroupName,
LoadBalancerName: tfOutput.LoadBalancerName,
UseInstanceMetadata: true,
VMType: "vmss",
Location: creds.Location,
UseManagedIdentityExtension: useManagedIdentityExtension,
UserAssignedIdentityID: tfOutput.UserAssignedIdentity,
}
return json.Marshal(config)
}

439
cli/internal/terraform/terraform.go
View File

@ -78,172 +78,8 @@ func (c *Client) WithManualStateMigration(migration StateMigration) *Client {
return c
}
// Show reads the default state path and outputs the state.
func (c *Client) Show(ctx context.Context) (*tfjson.State, error) {
return c.tf.Show(ctx)
}
// PrepareWorkspace prepares a Terraform workspace for a Constellation cluster.
func (c *Client) PrepareWorkspace(path string, vars Variables) error {
if err := prepareWorkspace(path, c.file, c.workingDir); err != nil {
return fmt.Errorf("prepare workspace: %w", err)
}
return c.writeVars(vars)
}
// PrepareUpgradeWorkspace prepares a Terraform workspace for a Constellation version upgrade.
// It copies the Terraform state from the old working dir and the embedded Terraform files into the new working dir.
func (c *Client) PrepareUpgradeWorkspace(path, oldWorkingDir, newWorkingDir, backupDir string, vars Variables) error {
if err := prepareUpgradeWorkspace(path, c.file, oldWorkingDir, newWorkingDir, backupDir); err != nil {
return fmt.Errorf("prepare upgrade workspace: %w", err)
}
return c.writeVars(vars)
}
// PrepareIAMUpgradeWorkspace prepares a Terraform workspace for a Constellation IAM upgrade.
func PrepareIAMUpgradeWorkspace(file file.Handler, path, oldWorkingDir, newWorkingDir, backupDir string) error {
if err := prepareUpgradeWorkspace(path, file, oldWorkingDir, newWorkingDir, backupDir); err != nil {
return fmt.Errorf("prepare upgrade workspace: %w", err)
}
// copy the vars file from the old working dir to the new working dir
if err := file.CopyFile(filepath.Join(oldWorkingDir, terraformVarsFile), filepath.Join(newWorkingDir, terraformVarsFile)); err != nil {
return fmt.Errorf("copying vars file: %w", err)
}
return nil
}
// CreateCluster creates a Constellation cluster using Terraform.
func (c *Client) CreateCluster(ctx context.Context, logLevel LogLevel) (ApplyOutput, error) {
if err := c.setLogLevel(logLevel); err != nil {
return ApplyOutput{}, fmt.Errorf("set terraform log level %s: %w", logLevel.String(), err)
}
if err := c.tf.Init(ctx); err != nil {
return ApplyOutput{}, fmt.Errorf("terraform init: %w", err)
}
if err := c.applyManualStateMigrations(ctx); err != nil {
return ApplyOutput{}, fmt.Errorf("apply manual state migrations: %w", err)
}
if err := c.tf.Apply(ctx); err != nil {
return ApplyOutput{}, fmt.Errorf("terraform apply: %w", err)
}
tfState, err := c.tf.Show(ctx)
if err != nil {
return ApplyOutput{}, fmt.Errorf("terraform show: %w", err)
}
ipOutput, ok := tfState.Values.Outputs["ip"]
if !ok {
return ApplyOutput{}, errors.New("no IP output found")
}
ip, ok := ipOutput.Value.(string)
if !ok {
return ApplyOutput{}, errors.New("invalid type in IP output: not a string")
}
apiServerCertSANsOutput, ok := tfState.Values.Outputs["api_server_cert_sans"]
if !ok {
return ApplyOutput{}, errors.New("no api_server_cert_sans output found")
}
apiServerCertSANsUntyped, ok := apiServerCertSANsOutput.Value.([]any)
if !ok {
return ApplyOutput{}, fmt.Errorf("invalid type in api_server_cert_sans output: %s is not a list of elements", apiServerCertSANsOutput.Type.FriendlyName())
}
apiServerCertSANs, err := toStringSlice(apiServerCertSANsUntyped)
if err != nil {
return ApplyOutput{}, fmt.Errorf("convert api_server_cert_sans output: %w", err)
}
secretOutput, ok := tfState.Values.Outputs["initSecret"]
if !ok {
return ApplyOutput{}, errors.New("no initSecret output found")
}
secret, ok := secretOutput.Value.(string)
if !ok {
return ApplyOutput{}, errors.New("invalid type in initSecret output: not a string")
}
uidOutput, ok := tfState.Values.Outputs["uid"]
if !ok {
return ApplyOutput{}, errors.New("no uid output found")
}
uid, ok := uidOutput.Value.(string)
if !ok {
return ApplyOutput{}, errors.New("invalid type in uid output: not a string")
}
var attestationURL string
if attestationURLOutput, ok := tfState.Values.Outputs["attestationURL"]; ok {
if attestationURLString, ok := attestationURLOutput.Value.(string); ok {
attestationURL = attestationURLString
}
}
return ApplyOutput{
IP: ip,
APIServerCertSANs: apiServerCertSANs,
Secret: secret,
UID: uid,
AttestationURL: attestationURL,
}, nil
}
// ApplyOutput contains the Terraform output values of a cluster creation
// or apply operation.
type ApplyOutput struct {
IP string
APIServerCertSANs []string
Secret string
UID string
// AttestationURL is the URL of the attestation provider.
// It is only set if the cluster is created on Azure.
AttestationURL string
}
// IAMOutput contains the output information of the Terraform IAM operations.
type IAMOutput struct {
GCP GCPIAMOutput
Azure AzureIAMOutput
AWS AWSIAMOutput
}
// GCPIAMOutput contains the output information of the Terraform IAM operation on GCP.
type GCPIAMOutput struct {
SaKey string
}
// AzureIAMOutput contains the output information of the Terraform IAM operation on Microsoft Azure.
type AzureIAMOutput struct {
SubscriptionID string
TenantID string
UAMIID string
}
// AWSIAMOutput contains the output information of the Terraform IAM operation on GCP.
type AWSIAMOutput struct {
ControlPlaneInstanceProfile string
WorkerNodeInstanceProfile string
}
// ApplyIAMConfig creates an IAM configuration using Terraform.
func (c *Client) ApplyIAMConfig(ctx context.Context, provider cloudprovider.Provider, logLevel LogLevel) (IAMOutput, error) {
if err := c.setLogLevel(logLevel); err != nil {
return IAMOutput{}, fmt.Errorf("set terraform log level %s: %w", logLevel.String(), err)
}
if err := c.tf.Init(ctx); err != nil {
return IAMOutput{}, err
}
if err := c.tf.Apply(ctx); err != nil {
return IAMOutput{}, err
}
// ShowIAM reads the state of Constellation IAM resources from Terraform.
func (c *Client) ShowIAM(ctx context.Context, provider cloudprovider.Provider) (IAMOutput, error) {
tfState, err := c.tf.Show(ctx)
if err != nil {
return IAMOutput{}, err
@ -324,6 +160,277 @@ func (c *Client) ApplyIAMConfig(ctx context.Context, provider cloudprovider.Prov
}
}
// ShowCluster reads the state of Constellation cluster resources from Terraform.
func (c *Client) ShowCluster(ctx context.Context, provider cloudprovider.Provider) (ApplyOutput, error) {
tfState, err := c.tf.Show(ctx)
if err != nil {
return ApplyOutput{}, fmt.Errorf("terraform show: %w", err)
}
ipOutput, ok := tfState.Values.Outputs["ip"]
if !ok {
return ApplyOutput{}, errors.New("no IP output found")
}
ip, ok := ipOutput.Value.(string)
if !ok {
return ApplyOutput{}, errors.New("invalid type in IP output: not a string")
}
apiServerCertSANsOutput, ok := tfState.Values.Outputs["api_server_cert_sans"]
if !ok {
return ApplyOutput{}, errors.New("no api_server_cert_sans output found")
}
apiServerCertSANsUntyped, ok := apiServerCertSANsOutput.Value.([]any)
if !ok {
return ApplyOutput{}, fmt.Errorf("invalid type in api_server_cert_sans output: %s is not a list of elements", apiServerCertSANsOutput.Type.FriendlyName())
}
apiServerCertSANs, err := toStringSlice(apiServerCertSANsUntyped)
if err != nil {
return ApplyOutput{}, fmt.Errorf("convert api_server_cert_sans output: %w", err)
}
secretOutput, ok := tfState.Values.Outputs["initSecret"]
if !ok {
return ApplyOutput{}, errors.New("no initSecret output found")
}
secret, ok := secretOutput.Value.(string)
if !ok {
return ApplyOutput{}, errors.New("invalid type in initSecret output: not a string")
}
uidOutput, ok := tfState.Values.Outputs["uid"]
if !ok {
return ApplyOutput{}, errors.New("no uid output found")
}
uid, ok := uidOutput.Value.(string)
if !ok {
return ApplyOutput{}, errors.New("invalid type in uid output: not a string")
}
res := ApplyOutput{
IP: ip,
APIServerCertSANs: apiServerCertSANs,
Secret: secret,
UID: uid,
}
// TODO add provider
switch provider {
case cloudprovider.GCP:
gcpProjectOutput, ok := tfState.Values.Outputs["project"]
if ok {
gcpProject, ok := gcpProjectOutput.Value.(string)
if !ok {
return ApplyOutput{}, errors.New("invalid type in project output: not a string")
}
cidrNodesOutput, ok := tfState.Values.Outputs["ip_cidr_nodes"]
if !ok {
return ApplyOutput{}, errors.New("no ip_cidr_nodes output found")
}
cidrNodes, ok := cidrNodesOutput.Value.(string)
if !ok {
return ApplyOutput{}, errors.New("invalid type in ip_cidr_nodes output: not a string")
}
cidrPodsOutput, ok := tfState.Values.Outputs["ip_cidr_pods"]
if !ok {
return ApplyOutput{}, errors.New("no ip_cidr_pods output found")
}
cidrPods, ok := cidrPodsOutput.Value.(string)
if !ok {
return ApplyOutput{}, errors.New("invalid type in ip_cidr_pods output: not a string")
}
res.GCP = &GCPApplyOutput{
ProjectID: gcpProject,
IPCidrNode: cidrNodes,
IPCidrPod: cidrPods,
}
}
case cloudprovider.Azure:
var attestationURL string
if ok {
if attestationURLOutput, ok := tfState.Values.Outputs["attestationURL"]; ok {
if attestationURLString, ok := attestationURLOutput.Value.(string); ok {
attestationURL = attestationURLString
}
}
}
azureUAMIOutput, ok := tfState.Values.Outputs["user_assigned_identity"]
if !ok {
return ApplyOutput{}, errors.New("no user_assigned_identity output found")
}
azureUAMI, ok := azureUAMIOutput.Value.(string)
if !ok {
return ApplyOutput{}, errors.New("invalid type in user_assigned_identity output: not a string")
}
rgOutput, ok := tfState.Values.Outputs["resource_group"]
if !ok {
return ApplyOutput{}, errors.New("no resource_group output found")
}
rg, ok := rgOutput.Value.(string)
if !ok {
return ApplyOutput{}, errors.New("invalid type in resource_group output: not a string")
}
subscriptionOutput, ok := tfState.Values.Outputs["subscription_id"]
if !ok {
return ApplyOutput{}, errors.New("no subscription_id output found")
}
subscriptionID, ok := subscriptionOutput.Value.(string)
if !ok {
return ApplyOutput{}, errors.New("invalid type in subscription_id output: not a string")
}
networkSGNameOutput, ok := tfState.Values.Outputs["network_security_group_name"]
if !ok {
return ApplyOutput{}, errors.New("no network_security_group_name output found")
}
networkSGName, ok := networkSGNameOutput.Value.(string)
if !ok {
return ApplyOutput{}, errors.New("invalid type in network_security_group_name output: not a string")
}
loadBalancerNameOutput, ok := tfState.Values.Outputs["loadbalancer_name"]
if !ok {
return ApplyOutput{}, errors.New("no loadbalancer_name output found")
}
loadBalancerName, ok := loadBalancerNameOutput.Value.(string)
if !ok {
return ApplyOutput{}, errors.New("invalid type in loadbalancer_name output: not a string")
}
res.Azure = &AzureApplyOutput{
ResourceGroup: rg,
SubscriptionID: subscriptionID,
UserAssignedIdentity: azureUAMI,
NetworkSecurityGroupName: networkSGName,
LoadBalancerName: loadBalancerName,
AttestationURL: attestationURL,
}
}
return res, nil
}
// PrepareWorkspace prepares a Terraform workspace for a Constellation cluster.
func (c *Client) PrepareWorkspace(path string, vars Variables) error {
if err := prepareWorkspace(path, c.file, c.workingDir); err != nil {
return fmt.Errorf("prepare workspace: %w", err)
}
return c.writeVars(vars)
}
// PrepareUpgradeWorkspace prepares a Terraform workspace for a Constellation version upgrade.
// It copies the Terraform state from the old working dir and the embedded Terraform files into the new working dir.
func (c *Client) PrepareUpgradeWorkspace(path, oldWorkingDir, newWorkingDir, backupDir string, vars Variables) error {
if err := prepareUpgradeWorkspace(path, c.file, oldWorkingDir, newWorkingDir, backupDir); err != nil {
return fmt.Errorf("prepare upgrade workspace: %w", err)
}
return c.writeVars(vars)
}
// PrepareIAMUpgradeWorkspace prepares a Terraform workspace for a Constellation IAM upgrade.
func PrepareIAMUpgradeWorkspace(file file.Handler, path, oldWorkingDir, newWorkingDir, backupDir string) error {
if err := prepareUpgradeWorkspace(path, file, oldWorkingDir, newWorkingDir, backupDir); err != nil {
return fmt.Errorf("prepare upgrade workspace: %w", err)
}
// copy the vars file from the old working dir to the new working dir
if err := file.CopyFile(filepath.Join(oldWorkingDir, terraformVarsFile), filepath.Join(newWorkingDir, terraformVarsFile)); err != nil {
return fmt.Errorf("copying vars file: %w", err)
}
return nil
}
// CreateCluster creates a Constellation cluster using Terraform.
func (c *Client) CreateCluster(ctx context.Context, provider cloudprovider.Provider, logLevel LogLevel) (ApplyOutput, error) {
if err := c.setLogLevel(logLevel); err != nil {
return ApplyOutput{}, fmt.Errorf("set terraform log level %s: %w", logLevel.String(), err)
}
if err := c.tf.Init(ctx); err != nil {
return ApplyOutput{}, fmt.Errorf("terraform init: %w", err)
}
if err := c.applyManualStateMigrations(ctx); err != nil {
return ApplyOutput{}, fmt.Errorf("apply manual state migrations: %w", err)
}
if err := c.tf.Apply(ctx); err != nil {
return ApplyOutput{}, fmt.Errorf("terraform apply: %w", err)
}
return c.ShowCluster(ctx, provider)
}
// ApplyOutput contains the Terraform output values of a cluster creation
// or apply operation.
type ApplyOutput struct {
IP string
APIServerCertSANs []string
Secret string
UID string
GCP *GCPApplyOutput
Azure *AzureApplyOutput
}
// AzureApplyOutput contains the Terraform output values of a terraform apply operation on Microsoft Azure.
type AzureApplyOutput struct {
ResourceGroup string
SubscriptionID string
NetworkSecurityGroupName string
LoadBalancerName string
UserAssignedIdentity string
// AttestationURL is the URL of the attestation provider.
AttestationURL string
}
// GCPApplyOutput contains the Terraform output values of a terraform apply operation on GCP.
type GCPApplyOutput struct {
ProjectID string
IPCidrNode string
IPCidrPod string
}
// IAMOutput contains the output information of the Terraform IAM operations.
type IAMOutput struct {
GCP GCPIAMOutput
Azure AzureIAMOutput
AWS AWSIAMOutput
}
// GCPIAMOutput contains the output information of the Terraform IAM operation on GCP.
type GCPIAMOutput struct {
SaKey string
}
// AzureIAMOutput contains the output information of the Terraform IAM operation on Microsoft Azure.
type AzureIAMOutput struct {
SubscriptionID string
TenantID string
UAMIID string
}
// AWSIAMOutput contains the output information of the Terraform IAM operation on GCP.
type AWSIAMOutput struct {
ControlPlaneInstanceProfile string
WorkerNodeInstanceProfile string
}
// ApplyIAMConfig creates an IAM configuration using Terraform.
func (c *Client) ApplyIAMConfig(ctx context.Context, provider cloudprovider.Provider, logLevel LogLevel) (IAMOutput, error) {
if err := c.setLogLevel(logLevel); err != nil {
return IAMOutput{}, fmt.Errorf("set terraform log level %s: %w", logLevel.String(), err)
}
if err := c.tf.Init(ctx); err != nil {
return IAMOutput{}, err
}
if err := c.tf.Apply(ctx); err != nil {
return IAMOutput{}, err
}
return c.ShowIAM(ctx, provider)
}
// Plan determines the diff that will be applied by Terraform. The plan output is written to the planFile.
// If there is a diff, the returned bool is true. Otherwise, it is false.
func (c *Client) Plan(ctx context.Context, logLevel LogLevel, planFile string) (bool, error) {

3
cli/internal/terraform/terraform/azure/main.tf
View File

@ -277,6 +277,9 @@ module "scale_set_group" {
]
}
data "azurerm_subscription" "current" {
}
moved {
from = module.scale_set_control_plane
to = module.scale_set_group["control_plane_default"]

21
cli/internal/terraform/terraform/azure/outputs.tf
View File

@ -18,3 +18,24 @@ output "initSecret" {
output "attestationURL" {
value = var.create_maa ? azurerm_attestation_provider.attestation_provider[0].attestation_uri : ""
}
output "network_security_group_name" {
value = azurerm_network_security_group.security_group.name
}
output "loadbalancer_name" {
value = azurerm_lb.loadbalancer.name
}
output "user_assigned_identity" {
value = var.user_assigned_identity
}
output "resource_group" {
value = var.resource_group
}
output "subscription_id" {
value = data.azurerm_subscription.current.subscription_id
}

12
cli/internal/terraform/terraform/gcp/outputs.tf
View File

@ -18,3 +18,15 @@ output "initSecret" {
value = random_password.initSecret.result
sensitive = true
}
output "project" {
value = var.project
}
output "ip_cidr_nodes" {
value = local.cidr_vpc_subnet_nodes
}
output "ip_cidr_pods" {
value = local.cidr_vpc_subnet_pods
}

21
cli/internal/terraform/terraform_test.go
View File

@ -248,6 +248,21 @@ func TestCreateCluster(t *testing.T) {
"api_server_cert_sans": {
Value: []any{"192.0.2.100"},
},
"user_assigned_identity": {
Value: "test_uami_id",
},
"resource_group": {
Value: "test_rg",
},
"subscription_id": {
Value: "test_subscription_id",
},
"network_security_group_name": {
Value: "test_nsg_name",
},
"loadbalancer_name": {
Value: "test_lb_name",
},
},
},
}
@ -435,7 +450,7 @@ func TestCreateCluster(t *testing.T) {
path := path.Join(tc.pathBase, strings.ToLower(tc.provider.String()))
require.NoError(c.PrepareWorkspace(path, tc.vars))
tfOutput, err := c.CreateCluster(context.Background(), LogLevelDebug)
tfOutput, err := c.CreateCluster(context.Background(), tc.provider, LogLevelDebug)
if tc.wantErr {
assert.Error(err)
@ -445,7 +460,9 @@ func TestCreateCluster(t *testing.T) {
assert.Equal("192.0.2.100", tfOutput.IP)
assert.Equal("initSecret", tfOutput.Secret)
assert.Equal("12345abc", tfOutput.UID)
assert.Equal(tc.expectedAttestationURL, tfOutput.AttestationURL)
if tc.provider == cloudprovider.Azure {
assert.Equal(tc.expectedAttestationURL, tfOutput.Azure.AttestationURL)
}
})
}
}

19
cli/internal/upgrade/terraform.go
View File

@ -153,25 +153,24 @@ func CleanUpTerraformMigrations(upgradeID string, fileHandler file.Handler) erro
// In case of a successful upgrade, the output will be written to the specified file and the old Terraform directory is replaced
// By the new one.
func (u *TerraformUpgrader) ApplyTerraformMigrations(ctx context.Context, opts TerraformUpgradeOptions, upgradeID string) error {
tfOutput, err := u.tf.CreateCluster(ctx, opts.LogLevel)
tfOutput, err := u.tf.CreateCluster(ctx, opts.CSP, opts.LogLevel)
if err != nil {
return fmt.Errorf("terraform apply: %w", err)
}
// AttestationURL is only set for Azure.
if tfOutput.AttestationURL != "" {
if err := u.policyPatcher.Patch(ctx, tfOutput.AttestationURL); err != nil {
return fmt.Errorf("patching policies: %w", err)
}
}
outputFileContents := clusterid.File{
CloudProvider: opts.CSP,
InitSecret: []byte(tfOutput.Secret),
IP: tfOutput.IP,
APIServerCertSANs: tfOutput.APIServerCertSANs,
UID: tfOutput.UID,
AttestationURL: tfOutput.AttestationURL,
}
// AttestationURL is only set for Azure.
if tfOutput.Azure != nil {
if err := u.policyPatcher.Patch(ctx, tfOutput.Azure.AttestationURL); err != nil {
return fmt.Errorf("patching policies: %w", err)
}
outputFileContents.AttestationURL = tfOutput.Azure.AttestationURL
}
if err := u.fileHandler.RemoveAll(constants.TerraformWorkingDir); err != nil {
@ -215,7 +214,7 @@ type tfClientCommon interface {
// tfResourceClient is a Terraform client for managing cluster resources.
type tfResourceClient interface {
PrepareUpgradeWorkspace(path, oldWorkingDir, newWorkingDir, backupDir string, vars terraform.Variables) error
CreateCluster(ctx context.Context, logLevel terraform.LogLevel) (terraform.ApplyOutput, error)
CreateCluster(ctx context.Context, provider cloudprovider.Provider, logLevel terraform.LogLevel) (terraform.ApplyOutput, error)
tfClientCommon
}

2
cli/internal/upgrade/terraform_test.go
View File

@ -371,7 +371,7 @@ func (u *stubTerraformClient) Plan(context.Context, terraform.LogLevel, string)
return u.hasDiff, u.planErr
}
func (u *stubTerraformClient) CreateCluster(context.Context, terraform.LogLevel) (terraform.ApplyOutput, error) {
func (u *stubTerraformClient) CreateCluster(context.Context, cloudprovider.Provider, terraform.LogLevel) (terraform.ApplyOutput, error) {
return terraform.ApplyOutput{}, u.CreateClusterErr
}

94
internal/cloud/azure/azure.go
View File

@ -17,7 +17,6 @@ package azure
import (
"context"
"encoding/json"
"errors"
"fmt"
"path"
@ -99,61 +98,6 @@ func New(ctx context.Context) (*Cloud, error) {
}, nil
}
// GetCCMConfig returns the configuration needed for the Kubernetes Cloud Controller Manager on Azure.
func (c *Cloud) GetCCMConfig(ctx context.Context, providerID string, cloudServiceAccountURI string) ([]byte, error) {
subscriptionID, resourceGroup, err := azureshared.BasicsFromProviderID(providerID)
if err != nil {
return nil, fmt.Errorf("parsing provider ID: %w", err)
}
creds, err := azureshared.ApplicationCredentialsFromURI(cloudServiceAccountURI)
if err != nil {
return nil, fmt.Errorf("parsing service account URI: %w", err)
}
uid, err := c.imds.uid(ctx)
if err != nil {
return nil, fmt.Errorf("retrieving instance UID: %w", err)
}
securityGroupName, err := c.getNetworkSecurityGroupName(ctx, resourceGroup, uid)
if err != nil {
return nil, fmt.Errorf("retrieving network security group name: %w", err)
}
loadBalancer, err := c.getLoadBalancer(ctx, resourceGroup, uid)
if err != nil {
return nil, fmt.Errorf("retrieving load balancer: %w", err)
}
if loadBalancer == nil || loadBalancer.Name == nil {
return nil, fmt.Errorf("could not dereference load balancer name")
}
var uamiClientID string
useManagedIdentityExtension := creds.PreferredAuthMethod == azureshared.AuthMethodUserAssignedIdentity
if useManagedIdentityExtension {
uamiClientID, err = c.getUAMIClientIDFromURI(ctx, providerID, creds.UamiResourceID)
if err != nil {
return nil, fmt.Errorf("retrieving user-assigned managed identity client ID: %w", err)
}
}
config := cloudConfig{
Cloud: "AzurePublicCloud",
TenantID: creds.TenantID,
SubscriptionID: subscriptionID,
ResourceGroup: resourceGroup,
LoadBalancerSku: "standard",
SecurityGroupName: securityGroupName,
LoadBalancerName: *loadBalancer.Name,
UseInstanceMetadata: true,
VMType: "vmss",
Location: creds.Location,
UseManagedIdentityExtension: useManagedIdentityExtension,
UserAssignedIdentityID: uamiClientID,
}
return json.Marshal(config)
}
// GetLoadBalancerEndpoint retrieves the first load balancer IP from cloud provider metadata.
//
// The returned string is an IP address without a port, but the method name needs to satisfy the
@ -286,24 +230,6 @@ func (c *Cloud) getInstance(ctx context.Context, providerID string) (metadata.In
return instance, nil
}
func (c *Cloud) getUAMIClientIDFromURI(ctx context.Context, providerID, resourceID string) (string, error) {
// userAssignedIdentityURI := "/subscriptions/{subscription-id}/resourcegroups/{resource-group}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/{identity-name}"
_, resourceGroup, scaleSet, instanceID, err := azureshared.ScaleSetInformationFromProviderID(providerID)
if err != nil {
return "", fmt.Errorf("invalid provider ID: %w", err)
}
vmResp, err := c.scaleSetsVMAPI.Get(ctx, resourceGroup, scaleSet, instanceID, nil)
if err != nil {
return "", fmt.Errorf("retrieving instance: %w", err)
}
for rID, v := range vmResp.Identity.UserAssignedIdentities {
if rID == resourceID {
return *v.ClientID, nil
}
}
return "", fmt.Errorf("no user assinged identity found for resource ID %s", resourceID)
}
// getNetworkSecurityGroupName returns the security group name of the resource group.
func (c *Cloud) getNetworkSecurityGroupName(ctx context.Context, resourceGroup, uid string) (string, error) {
pager := c.secGroupAPI.NewListPager(resourceGroup, nil)
@ -462,26 +388,6 @@ func (c *Cloud) getLoadBalancerDNSName(ctx context.Context) (string, error) {
}
*/
type cloudConfig struct {
Cloud string `json:"cloud,omitempty"`
TenantID string `json:"tenantId,omitempty"`
SubscriptionID string `json:"subscriptionId,omitempty"`
ResourceGroup string `json:"resourceGroup,omitempty"`
Location string `json:"location,omitempty"`
SubnetName string `json:"subnetName,omitempty"`
SecurityGroupName string `json:"securityGroupName,omitempty"`
SecurityGroupResourceGroup string `json:"securityGroupResourceGroup,omitempty"`
LoadBalancerName string `json:"loadBalancerName,omitempty"`
LoadBalancerSku string `json:"loadBalancerSku,omitempty"`
VNetName string `json:"vnetName,omitempty"`
VNetResourceGroup string `json:"vnetResourceGroup,omitempty"`
CloudProviderBackoff bool `json:"cloudProviderBackoff,omitempty"`
UseInstanceMetadata bool `json:"useInstanceMetadata,omitempty"`
VMType string `json:"vmType,omitempty"`
UseManagedIdentityExtension bool `json:"useManagedIdentityExtension,omitempty"`
UserAssignedIdentityID string `json:"userAssignedIdentityID,omitempty"`
}
// convertToInstanceMetadata converts a armcomputev2.VirtualMachineScaleSetVM to a metadata.InstanceMetadata.
func convertToInstanceMetadata(vm armcompute.VirtualMachineScaleSetVM, networkInterfaces []armnetwork.Interface,
) (metadata.InstanceMetadata, error) {

331
internal/cloud/azure/azure_test.go
View File

@ -8,7 +8,6 @@ package azure
import (
"context"
"encoding/json"
"errors"
"testing"
@ -28,336 +27,6 @@ func TestMain(m *testing.M) {
goleak.VerifyTestMain(m)
}
func TestGetCCMConfig(t *testing.T) {
someErr := errors.New("failed")
goodLB := armnetwork.LoadBalancer{
Name: to.Ptr("load-balancer"),
Tags: map[string]*string{
cloud.TagUID: to.Ptr("uid"),
},
Properties: &armnetwork.LoadBalancerPropertiesFormat{
FrontendIPConfigurations: []*armnetwork.FrontendIPConfiguration{
{
Properties: &armnetwork.FrontendIPConfigurationPropertiesFormat{
PublicIPAddress: &armnetwork.PublicIPAddress{ID: to.Ptr("/subscriptions/subscription/resourceGroups/resourceGroup/providers/Microsoft.Network/publicIPAddresses/pubIPName")},
},
},
},
},
}
goodSecurityGroup := armnetwork.SecurityGroup{
Tags: map[string]*string{
cloud.TagUID: to.Ptr("uid"),
},
Name: to.Ptr("security-group"),
}
uamiClientID := "uami-client-id"
testCases := map[string]struct {
imdsAPI imdsAPI
loadBalancerAPI loadBalancerAPI
secGroupAPI securityGroupsAPI
scaleSetsVMAPI virtualMachineScaleSetVMsAPI
providerID string
cloudServiceAccountURI string
wantErr bool
wantConfig cloudConfig
}{
"success": {
imdsAPI: &stubIMDSAPI{
uidVal: "uid",
},
loadBalancerAPI: &stubLoadBalancersAPI{
pager: &stubLoadBalancersClientListPager{
list: []armnetwork.LoadBalancer{goodLB},
},
},
secGroupAPI: &stubSecurityGroupsAPI{
pager: &stubSecurityGroupsClientListPager{
list: []armnetwork.SecurityGroup{goodSecurityGroup},
},
},
scaleSetsVMAPI: &stubVirtualMachineScaleSetVMsAPI{
getVM: armcompute.VirtualMachineScaleSetVM{
Identity: &armcompute.VirtualMachineIdentity{
UserAssignedIdentities: map[string]*armcompute.UserAssignedIdentitiesValue{
"subscriptions/9b352db0-82af-408c-a02c-36fbffbf7015/resourceGroups/resourceGroupName/providers/Microsoft.ManagedIdentity/userAssignedIdentities/UAMIName": {ClientID: &uamiClientID},
},
},
},
},
providerID: "azure:///subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/virtualMachineScaleSets/scale-set/virtualMachines/0",
cloudServiceAccountURI: "serviceaccount://azure?tenant_id=tenant-id&client_id=client-id&client_secret=client-secret&location=westeurope&preferred_auth_method=userassignedidentity&uami_resource_id=subscriptions%2F9b352db0-82af-408c-a02c-36fbffbf7015%2FresourceGroups%2FresourceGroupName%2Fproviders%2FMicrosoft.ManagedIdentity%2FuserAssignedIdentities%2FUAMIName",
wantConfig: cloudConfig{
Cloud: "AzurePublicCloud",
TenantID: "tenant-id",
SubscriptionID: "subscription-id",
ResourceGroup: "resource-group",
LoadBalancerSku: "standard",
SecurityGroupName: "security-group",
LoadBalancerName: "load-balancer",
UseInstanceMetadata: true,
UseManagedIdentityExtension: true,
UserAssignedIdentityID: uamiClientID,
VMType: "vmss",
Location: "westeurope",
},
},
"no app registration": {
imdsAPI: &stubIMDSAPI{
uidVal: "uid",
},
loadBalancerAPI: &stubLoadBalancersAPI{
pager: &stubLoadBalancersClientListPager{
list: []armnetwork.LoadBalancer{goodLB},
},
},
secGroupAPI: &stubSecurityGroupsAPI{
pager: &stubSecurityGroupsClientListPager{
list: []armnetwork.SecurityGroup{goodSecurityGroup},
},
},
providerID: "azure:///subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/virtualMachineScaleSets/scale-set/virtualMachines/0",
cloudServiceAccountURI: "serviceaccount://azure?tenant_id=tenant-id&location=westeurope",
wantConfig: cloudConfig{
Cloud: "AzurePublicCloud",
TenantID: "tenant-id",
SubscriptionID: "subscription-id",
ResourceGroup: "resource-group",
LoadBalancerSku: "standard",
SecurityGroupName: "security-group",
LoadBalancerName: "load-balancer",
UseInstanceMetadata: true,
VMType: "vmss",
Location: "westeurope",
},
},
"missing UID tag": {
imdsAPI: &stubIMDSAPI{
uidVal: "uid",
},
loadBalancerAPI: &stubLoadBalancersAPI{
pager: &stubLoadBalancersClientListPager{
list: []armnetwork.LoadBalancer{
{
Name: to.Ptr("load-balancer"),
Properties: &armnetwork.LoadBalancerPropertiesFormat{
FrontendIPConfigurations: []*armnetwork.FrontendIPConfiguration{
{
Properties: &armnetwork.FrontendIPConfigurationPropertiesFormat{
PublicIPAddress: &armnetwork.PublicIPAddress{ID: to.Ptr("/subscriptions/subscription/resourceGroups/resourceGroup/providers/Microsoft.Network/publicIPAddresses/pubIPName")},
},
},
},
},
},
},
},
},
secGroupAPI: &stubSecurityGroupsAPI{
pager: &stubSecurityGroupsClientListPager{
list: []armnetwork.SecurityGroup{goodSecurityGroup},
},
},
providerID: "azure:///subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/virtualMachineScaleSets/scale-set/virtualMachines/0",
cloudServiceAccountURI: "serviceaccount://azure?tenant_id=tenant-id&client_id=client-id&client_secret=client-secret&location=westeurope",
wantErr: true,
},
"only correct UID is chosen": {
imdsAPI: &stubIMDSAPI{
uidVal: "uid",
},
loadBalancerAPI: &stubLoadBalancersAPI{
pager: &stubLoadBalancersClientListPager{
list: []armnetwork.LoadBalancer{
{
Name: to.Ptr("load-balancer"),
Tags: map[string]*string{
cloud.TagUID: to.Ptr("different-uid"),
},
Properties: &armnetwork.LoadBalancerPropertiesFormat{
FrontendIPConfigurations: []*armnetwork.FrontendIPConfiguration{
{
Properties: &armnetwork.FrontendIPConfigurationPropertiesFormat{
PublicIPAddress: &armnetwork.PublicIPAddress{ID: to.Ptr("/subscriptions/subscription/resourceGroups/resourceGroup/providers/Microsoft.Network/publicIPAddresses/pubIPName")},
},
},
},
},
},
goodLB,
},
},
},
secGroupAPI: &stubSecurityGroupsAPI{
pager: &stubSecurityGroupsClientListPager{
list: []armnetwork.SecurityGroup{goodSecurityGroup},
},
},
providerID: "azure:///subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/virtualMachineScaleSets/scale-set/virtualMachines/0",
cloudServiceAccountURI: "serviceaccount://azure?tenant_id=tenant-id&client_id=client-id&client_secret=client-secret&location=westeurope",
wantConfig: cloudConfig{
Cloud: "AzurePublicCloud",
TenantID: "tenant-id",
SubscriptionID: "subscription-id",
ResourceGroup: "resource-group",
LoadBalancerSku: "standard",
SecurityGroupName: "security-group",
LoadBalancerName: "load-balancer",
UseInstanceMetadata: true,
VMType: "vmss",
Location: "westeurope",
},
},
"load balancer list error": {
imdsAPI: &stubIMDSAPI{
uidVal: "uid",
},
loadBalancerAPI: &stubLoadBalancersAPI{
pager: &stubLoadBalancersClientListPager{
fetchErr: someErr,
},
},
secGroupAPI: &stubSecurityGroupsAPI{
pager: &stubSecurityGroupsClientListPager{
list: []armnetwork.SecurityGroup{goodSecurityGroup},
},
},
providerID: "azure:///subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/virtualMachineScaleSets/scale-set/virtualMachines/0",
cloudServiceAccountURI: "serviceaccount://azure?tenant_id=tenant-id&client_id=client-id&client_secret=client-secret&location=westeurope",
wantErr: true,
},
"missing load balancer name": {
imdsAPI: &stubIMDSAPI{
uidVal: "uid",
},
loadBalancerAPI: &stubLoadBalancersAPI{
pager: &stubLoadBalancersClientListPager{
list: []armnetwork.LoadBalancer{{
Tags: map[string]*string{
cloud.TagUID: to.Ptr("uid"),
},
Properties: &armnetwork.LoadBalancerPropertiesFormat{
FrontendIPConfigurations: []*armnetwork.FrontendIPConfiguration{
{
Properties: &armnetwork.FrontendIPConfigurationPropertiesFormat{
PublicIPAddress: &armnetwork.PublicIPAddress{ID: to.Ptr("/subscriptions/subscription/resourceGroups/resourceGroup/providers/Microsoft.Network/publicIPAddresses/pubIPName")},
},
},
},
},
}},
},
},
secGroupAPI: &stubSecurityGroupsAPI{
pager: &stubSecurityGroupsClientListPager{
list: []armnetwork.SecurityGroup{goodSecurityGroup},
},
},
providerID: "azure:///subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/virtualMachineScaleSets/scale-set/virtualMachines/0",
cloudServiceAccountURI: "serviceaccount://azure?tenant_id=tenant-id&client_id=client-id&client_secret=client-secret&location=westeurope",
wantErr: true,
},
"security group list error": {
imdsAPI: &stubIMDSAPI{
uidVal: "uid",
},
loadBalancerAPI: &stubLoadBalancersAPI{
pager: &stubLoadBalancersClientListPager{
list: []armnetwork.LoadBalancer{goodLB},
},
},
secGroupAPI: &stubSecurityGroupsAPI{
pager: &stubSecurityGroupsClientListPager{
fetchErr: someErr,
},
},
providerID: "azure:///subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/virtualMachineScaleSets/scale-set/virtualMachines/0",
cloudServiceAccountURI: "serviceaccount://azure?tenant_id=tenant-id&client_id=client-id&client_secret=client-secret&location=westeurope",
wantErr: true,
},
"invalid provider ID": {
imdsAPI: &stubIMDSAPI{
uidVal: "uid",
},
loadBalancerAPI: &stubLoadBalancersAPI{
pager: &stubLoadBalancersClientListPager{
list: []armnetwork.LoadBalancer{goodLB},
},
},
secGroupAPI: &stubSecurityGroupsAPI{
pager: &stubSecurityGroupsClientListPager{
list: []armnetwork.SecurityGroup{goodSecurityGroup},
},
},
providerID: "invalid:///subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/virtualMachineScaleSets/scale-set/virtualMachines/0",
cloudServiceAccountURI: "serviceaccount://azure?tenant_id=tenant-id&client_id=client-id&client_secret=client-secret&location=westeurope",
wantErr: true,
},
"invalid cloud service account URI": {
imdsAPI: &stubIMDSAPI{
uidVal: "uid",
},
loadBalancerAPI: &stubLoadBalancersAPI{
pager: &stubLoadBalancersClientListPager{
list: []armnetwork.LoadBalancer{goodLB},
},
},
secGroupAPI: &stubSecurityGroupsAPI{
pager: &stubSecurityGroupsClientListPager{
list: []armnetwork.SecurityGroup{goodSecurityGroup},
},
},
providerID: "azure:///subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/virtualMachineScaleSets/scale-set/virtualMachines/0",
cloudServiceAccountURI: "invalid://azure?tenant_id=tenant-id&client_id=client-id&client_secret=client-secret&location=westeurope",
wantErr: true,
},
"imds error": {
imdsAPI: &stubIMDSAPI{
uidErr: someErr,
},
loadBalancerAPI: &stubLoadBalancersAPI{
pager: &stubLoadBalancersClientListPager{
list: []armnetwork.LoadBalancer{goodLB},
},
},
secGroupAPI: &stubSecurityGroupsAPI{
pager: &stubSecurityGroupsClientListPager{
list: []armnetwork.SecurityGroup{goodSecurityGroup},
},
},
providerID: "azure:///subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/virtualMachineScaleSets/scale-set/virtualMachines/0",
cloudServiceAccountURI: "serviceaccount://azure?tenant_id=tenant-id&client_id=client-id&client_secret=client-secret&location=westeurope",
wantErr: true,
},
}
for name, tc := range testCases {
t.Run(name, func(t *testing.T) {
assert := assert.New(t)
require := require.New(t)
cloud := &Cloud{
imds: tc.imdsAPI,
loadBalancerAPI: tc.loadBalancerAPI,
secGroupAPI: tc.secGroupAPI,
scaleSetsVMAPI: tc.scaleSetsVMAPI,
}
config, err := cloud.GetCCMConfig(context.Background(), tc.providerID, tc.cloudServiceAccountURI)
if tc.wantErr {
assert.Error(err)
return
}
assert.NoError(err)
wantConfig, err := json.Marshal(tc.wantConfig)
require.NoError(err)
assert.JSONEq(string(wantConfig), string(config))
})
}
}
func TestGetInstance(t *testing.T) {
someErr := errors.New("failed")
sampleProviderID := "azure:///subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/virtualMachineScaleSets/scale-set-name/virtualMachines/instance-id"

23
internal/cloud/azureshared/appcredentials.go
View File

@ -9,13 +9,21 @@ package azureshared
import (
"fmt"
"net/url"
"regexp"
"strings"
)
var (
subscriptionPattern = regexp.MustCompile(`subscriptions/([^/]+)/`)
rgPattern = regexp.MustCompile(`resourceGroups/([^/]+)/`)
)
// ApplicationCredentials is a set of Azure API credentials.
// It can contain a client secret and carries the preferred authentication method.
// It is the equivalent of a service account key in other cloud providers.
type ApplicationCredentials struct {
SubscriptionID string
ResourceGroup string
TenantID string
AppClientID string
ClientSecretValue string
@ -37,8 +45,14 @@ func ApplicationCredentialsFromURI(cloudServiceAccountURI string) (ApplicationCr
return ApplicationCredentials{}, fmt.Errorf("invalid service account URI: invalid host: %s", uri.Host)
}
query := uri.Query()
subscriptionID := getFirstMatchOrEmpty(subscriptionPattern, query.Get("uami_resource_id"))
resourceGroup := getFirstMatchOrEmpty(rgPattern, query.Get("uami_resource_id"))
preferredAuthMethod := FromString(query.Get("preferred_auth_method"))
return ApplicationCredentials{
SubscriptionID: subscriptionID,
ResourceGroup: resourceGroup,
TenantID: query.Get("tenant_id"),
AppClientID: query.Get("client_id"),
ClientSecretValue: query.Get("client_secret"),
@ -48,6 +62,15 @@ func ApplicationCredentialsFromURI(cloudServiceAccountURI string) (ApplicationCr
}, nil
}
func getFirstMatchOrEmpty(pattern *regexp.Regexp, str string) string {
subscriptionMatches := pattern.FindStringSubmatch(str)
var subscriptionID string
if len(subscriptionMatches) > 1 {
subscriptionID = subscriptionMatches[1]
}
return subscriptionID
}
// ToCloudServiceAccountURI converts the ApplicationCredentials into a cloud service account URI.
func (c ApplicationCredentials) ToCloudServiceAccountURI() string {
query := url.Values{}

4
internal/cloud/azureshared/appcredentials_test.go
View File

@ -27,12 +27,16 @@ func TestApplicationCredentialsFromURI(t *testing.T) {
Location: "location",
UamiResourceID: "subscriptions/9b352db0-82af-408c-a02c-36fbffbf7015/resourceGroups/resourceGroupName/providers/Microsoft.ManagedIdentity/userAssignedIdentities/UAMIName",
PreferredAuthMethod: AuthMethodServicePrincipal,
SubscriptionID: "9b352db0-82af-408c-a02c-36fbffbf7015",
ResourceGroup: "resourceGroupName",
}
credsWithoutSecret := ApplicationCredentials{
TenantID: "tenant-id",
Location: "location",
UamiResourceID: "subscriptions/9b352db0-82af-408c-a02c-36fbffbf7015/resourceGroups/resourceGroupName/providers/Microsoft.ManagedIdentity/userAssignedIdentities/UAMIName",
PreferredAuthMethod: AuthMethodUserAssignedIdentity,
SubscriptionID: "9b352db0-82af-408c-a02c-36fbffbf7015",
ResourceGroup: "resourceGroupName",
}
credsWithoutPreferrredAuthMethod := ApplicationCredentials{
TenantID: "tenant-id",

2
internal/deploy/helm/BUILD.bazel
View File

@ -11,10 +11,8 @@ go_library(
visibility = ["//:__subpackages__"],
deps = [
"//internal/constants",
"//internal/logger",
"//internal/retry",
"@io_k8s_apimachinery//pkg/util/wait",
"@org_uber_go_zap//:zap",
"@sh_helm_helm_v3//pkg/action",
"@sh_helm_helm_v3//pkg/chart",
"@sh_helm_helm_v3//pkg/chart/loader",

27
internal/deploy/helm/install.go
View File

@ -14,9 +14,7 @@ import (
"time"
"github.com/edgelesssys/constellation/v2/internal/constants"
"github.com/edgelesssys/constellation/v2/internal/logger"
"github.com/edgelesssys/constellation/v2/internal/retry"
"go.uber.org/zap"
"helm.sh/helm/v3/pkg/action"
"helm.sh/helm/v3/pkg/chart"
"helm.sh/helm/v3/pkg/chart/loader"
@ -31,20 +29,25 @@ const (
maximumRetryAttempts = 3
)
type debugLog interface {
Debugf(format string, args ...any)
Sync()
}
// Installer is a wrapper for a helm install action.
type Installer struct {
*action.Install
log *logger.Logger
log debugLog
}
// NewInstaller creates a new Installer with the given logger.
func NewInstaller(log *logger.Logger, kubeconfig string) (*Installer, error) {
func NewInstaller(kubeconfig string, logger debugLog) (*Installer, error) {
settings := cli.New()
settings.KubeConfig = kubeconfig
actionConfig := &action.Configuration{}
if err := actionConfig.Init(settings.RESTClientGetter(), constants.HelmNamespace,
"secret", log.Infof); err != nil {
"secret", logger.Debugf); err != nil {
return nil, err
}
@ -53,13 +56,12 @@ func NewInstaller(log *logger.Logger, kubeconfig string) (*Installer, error) {
action.Timeout = timeout
return &Installer{
action,
log,
Install: action,
log: logger,
}, nil
}
// InstallChart is the generic install function for helm charts.
// When timeout is nil, the default timeout is used.
func (h *Installer) InstallChart(ctx context.Context, release Release) error {
return h.InstallChartWithValues(ctx, release, nil)
}
@ -115,7 +117,7 @@ func (h *Installer) install(ctx context.Context, chartRaw []byte, values map[str
return fmt.Errorf("helm install: %w", err)
}
retryLoopFinishDuration := time.Since(retryLoopStartTime)
h.log.With(zap.String("chart", chart.Name()), zap.Duration("duration", retryLoopFinishDuration)).Infof("Helm chart installation finished")
h.log.Debugf("Helm chart %q installation finished after %s", chart.Name(), retryLoopFinishDuration)
return nil
}
@ -143,15 +145,14 @@ type installDoer struct {
Installer *Installer
chart *chart.Chart
values map[string]any
log *logger.Logger
log debugLog
}
// Do logs which chart is installed and tries to install it.
func (i installDoer) Do(ctx context.Context) error {
i.log.With(zap.String("chart", i.chart.Name())).Infof("Trying to install Helm chart")
i.log.Debugf("Trying to install Helm chart %s", i.chart.Name())
if _, err := i.Installer.RunWithContext(ctx, i.chart, i.values); err != nil {
i.log.With(zap.Error(err), zap.String("chart", i.chart.Name())).Errorf("Helm chart installation failed")
i.log.Debugf("Helm chart installation % failed: %v", i.chart.Name(), err)
return err
}