From c69e6777bd62503322fa03bdaddbe194240a6037 Mon Sep 17 00:00:00 2001 From: Moritz Sanft <58110325+msanft@users.noreply.github.com> Date: Mon, 22 May 2023 13:31:20 +0200 Subject: [PATCH] cli: Terraform migrations on upgrade (#1685) * add terraform planning * overwrite terraform files in upgrade workspace * Revert "overwrite terraform files in upgrade workspace" This reverts commit 8bdacfb8bef23ef2cdbdb06bad0855b3bbc42df0. * prepare terraform workspace * test upgrade integration * print upgrade abort * rename plan file * write output to file * add show plan test * add upgrade tf workdir * fix workspace preparing * squash to 1 command * test * bazel build * plan test * register flag manually * bazel tidy * fix linter * remove MAA variable * fix workdir * accept tf variables * variable fetching * fix resource indices * accept Terraform targets * refactor upgrade command * Terraform migration apply unit test * pass down image fetcher to test * use new flags in e2e test * move file name to constant * update buildfiles * fix version constant * conditionally create MAA * move interface down * upgrade dir * update buildfiles * fix interface * fix createMAA check * fix imports * update buildfiles * wip: workspace backup * copy utils * backup upgrade workspace * remove debug print * replace old state after upgrade * check if flag exists * prepare test workspace * remove prefix Co-authored-by: Otto Bittner * respect file permissions * refactor tf upgrader * check workspace before upgrades * remove temp upgrade dir after completion * clean up workspace after abortion * fix upgrade apply test * fix linter --------- Co-authored-by: Otto Bittner --- cli/internal/cloudcmd/clients.go | 2 +- cli/internal/cloudcmd/clients_test.go | 2 +- cli/internal/cmd/BUILD.bazel | 3 + cli/internal/cmd/upgradeapply.go | 181 ++++++++++++- cli/internal/cmd/upgradeapply_test.go | 112 +++++++- cli/internal/cmd/upgradecheck.go | 2 +- cli/internal/helm/backup.go | 7 +- cli/internal/kubernetes/BUILD.bazel | 3 + cli/internal/kubernetes/upgrade.go | 45 +++- cli/internal/terraform/loader.go | 33 ++- cli/internal/terraform/loader_test.go | 99 ++++++- cli/internal/terraform/terraform.go | 63 ++++- cli/internal/terraform/terraform_test.go | 152 ++++++++++- cli/internal/upgrade/BUILD.bazel | 34 +++ cli/internal/upgrade/terraform.go | 175 +++++++++++++ cli/internal/upgrade/terraform_test.go | 313 +++++++++++++++++++++++ cli/internal/upgrade/upgrade.go | 10 + e2e/internal/upgrade/upgrade_test.go | 17 +- internal/constants/constants.go | 10 + internal/file/file.go | 50 ++++ internal/file/file_test.go | 122 +++++++++ 21 files changed, 1391 insertions(+), 44 deletions(-) create mode 100644 cli/internal/upgrade/BUILD.bazel create mode 100644 cli/internal/upgrade/terraform.go create mode 100644 cli/internal/upgrade/terraform_test.go create mode 100644 cli/internal/upgrade/upgrade.go diff --git a/cli/internal/cloudcmd/clients.go b/cli/internal/cloudcmd/clients.go index 897be8ea8..e016de080 100644 --- a/cli/internal/cloudcmd/clients.go +++ b/cli/internal/cloudcmd/clients.go @@ -23,7 +23,7 @@ type imageFetcher interface { type terraformClient interface { PrepareWorkspace(path string, input terraform.Variables) error - CreateCluster(ctx context.Context, logLevel terraform.LogLevel) (terraform.CreateOutput, error) + CreateCluster(ctx context.Context, logLevel terraform.LogLevel, targets ...string) (terraform.CreateOutput, error) CreateIAMConfig(ctx context.Context, provider cloudprovider.Provider, logLevel terraform.LogLevel) (terraform.IAMOutput, error) Destroy(ctx context.Context, logLevel terraform.LogLevel) error CleanUpWorkspace() error diff --git a/cli/internal/cloudcmd/clients_test.go b/cli/internal/cloudcmd/clients_test.go index 750362a4f..a6b11487b 100644 --- a/cli/internal/cloudcmd/clients_test.go +++ b/cli/internal/cloudcmd/clients_test.go @@ -45,7 +45,7 @@ type stubTerraformClient struct { showErr error } -func (c *stubTerraformClient) CreateCluster(_ context.Context, _ terraform.LogLevel) (terraform.CreateOutput, error) { +func (c *stubTerraformClient) CreateCluster(_ context.Context, _ terraform.LogLevel, _ ...string) (terraform.CreateOutput, error) { return terraform.CreateOutput{ IP: c.ip, Secret: c.initSecret, diff --git a/cli/internal/cmd/BUILD.bazel b/cli/internal/cmd/BUILD.bazel index f0793f838..386adaa65 100644 --- a/cli/internal/cmd/BUILD.bazel +++ b/cli/internal/cmd/BUILD.bazel @@ -41,9 +41,11 @@ go_library( "//cli/internal/clusterid", "//cli/internal/helm", "//cli/internal/iamid", + "//cli/internal/image", "//cli/internal/kubernetes", "//cli/internal/libvirt", "//cli/internal/terraform", + "//cli/internal/upgrade", "//disk-mapper/recoverproto", "//internal/atls", "//internal/attestation/measurements", @@ -122,6 +124,7 @@ go_test( "//cli/internal/iamid", "//cli/internal/kubernetes", "//cli/internal/terraform", + "//cli/internal/upgrade", "//disk-mapper/recoverproto", "//internal/atls", "//internal/attestation/measurements", diff --git a/cli/internal/cmd/upgradeapply.go b/cli/internal/cmd/upgradeapply.go index 28157172d..09e662437 100644 --- a/cli/internal/cmd/upgradeapply.go +++ b/cli/internal/cmd/upgradeapply.go @@ -10,11 +10,16 @@ import ( "context" "errors" "fmt" + "path/filepath" + "strings" "time" "github.com/edgelesssys/constellation/v2/cli/internal/clusterid" "github.com/edgelesssys/constellation/v2/cli/internal/helm" + "github.com/edgelesssys/constellation/v2/cli/internal/image" "github.com/edgelesssys/constellation/v2/cli/internal/kubernetes" + "github.com/edgelesssys/constellation/v2/cli/internal/terraform" + "github.com/edgelesssys/constellation/v2/cli/internal/upgrade" "github.com/edgelesssys/constellation/v2/internal/cloud/cloudprovider" "github.com/edgelesssys/constellation/v2/internal/compatibility" "github.com/edgelesssys/constellation/v2/internal/config" @@ -55,17 +60,20 @@ func runUpgradeApply(cmd *cobra.Command, _ []string) error { defer log.Sync() fileHandler := file.NewHandler(afero.NewOsFs()) - upgrader, err := kubernetes.NewUpgrader(cmd.OutOrStdout(), log) + upgrader, err := kubernetes.NewUpgrader(cmd.Context(), cmd.OutOrStdout(), log) if err != nil { return err } - applyCmd := upgradeApplyCmd{upgrader: upgrader, log: log} + fetcher := image.New() + + applyCmd := upgradeApplyCmd{upgrader: upgrader, log: log, fetcher: fetcher} return applyCmd.upgradeApply(cmd, fileHandler) } type upgradeApplyCmd struct { upgrader cloudUpgrader + fetcher imageFetcher log debugLog } @@ -94,6 +102,10 @@ func (u *upgradeApplyCmd) upgradeApply(cmd *cobra.Command, fileHandler file.Hand return fmt.Errorf("upgrading measurements: %w", err) } + if err := u.migrateTerraform(cmd, fileHandler, u.fetcher, conf, flags); err != nil { + return fmt.Errorf("performing Terraform migrations: %w", err) + } + if conf.GetProvider() == cloudprovider.Azure || conf.GetProvider() == cloudprovider.GCP || conf.GetProvider() == cloudprovider.AWS { err = u.handleServiceUpgrade(cmd, conf, flags) upgradeErr := &compatibility.InvalidUpgradeError{} @@ -120,6 +132,141 @@ func (u *upgradeApplyCmd) upgradeApply(cmd *cobra.Command, fileHandler file.Hand return nil } +// migrateTerraform checks if the Constellation version the cluster is being upgraded to requires a migration +// of cloud resources with Terraform. If so, the migration is performed. +func (u *upgradeApplyCmd) migrateTerraform(cmd *cobra.Command, file file.Handler, fetcher imageFetcher, conf *config.Config, flags upgradeApplyFlags) error { + u.log.Debugf("Planning Terraform migrations") + + if err := u.upgrader.CheckTerraformMigrations(file); err != nil { + return fmt.Errorf("checking workspace: %w", err) + } + + targets, vars, err := u.parseUpgradeVars(cmd, conf, fetcher) + if err != nil { + return fmt.Errorf("parsing upgrade variables: %w", err) + } + u.log.Debugf("Using migration targets:\n%v", targets) + u.log.Debugf("Using Terraform variables:\n%v", vars) + + opts := upgrade.TerraformUpgradeOptions{ + LogLevel: flags.terraformLogLevel, + CSP: conf.GetProvider(), + Vars: vars, + Targets: targets, + OutputFile: constants.TerraformMigrationOutputFile, + } + + // Check if there are any Terraform migrations to apply + hasDiff, err := u.upgrader.PlanTerraformMigrations(cmd.Context(), opts) + if err != nil { + return fmt.Errorf("planning terraform migrations: %w", err) + } + + if hasDiff { + // If there are any Terraform migrations to apply, ask for confirmation + if !flags.yes { + ok, err := askToConfirm(cmd, "Do you want to apply the Terraform migrations?") + if err != nil { + return fmt.Errorf("asking for confirmation: %w", err) + } + if !ok { + cmd.Println("Aborting upgrade.") + if err := u.upgrader.CleanUpTerraformMigrations(file); err != nil { + return fmt.Errorf("cleaning up workspace: %w", err) + } + return fmt.Errorf("aborted by user") + } + } + u.log.Debugf("Applying Terraform migrations") + err := u.upgrader.ApplyTerraformMigrations(cmd.Context(), file, opts) + if err != nil { + return fmt.Errorf("applying terraform migrations: %w", err) + } + cmd.Printf("Terraform migrations applied successfully and output written to: %s\n"+ + "A backup of the pre-upgrade Terraform state has been written to: %s\n", + opts.OutputFile, filepath.Join(constants.UpgradeDir, constants.TerraformUpgradeBackupDir)) + } else { + u.log.Debugf("No Terraform diff detected") + } + + return nil +} + +func (u *upgradeApplyCmd) parseUpgradeVars(cmd *cobra.Command, conf *config.Config, fetcher imageFetcher) ([]string, terraform.Variables, error) { + // Fetch variables to execute Terraform script with + imageRef, err := fetcher.FetchReference(cmd.Context(), conf) + if err != nil { + return nil, nil, fmt.Errorf("fetching image reference: %w", err) + } + + commonVariables := terraform.CommonVariables{ + Name: conf.Name, + StateDiskSizeGB: conf.StateDiskSizeGB, + // Ignore node count as their values are only being respected for creation + // See here: https://developer.hashicorp.com/terraform/language/meta-arguments/lifecycle#ignore_changes + } + + switch conf.GetProvider() { + case cloudprovider.AWS: + targets := []string{} + + vars := &terraform.AWSClusterVariables{ + CommonVariables: commonVariables, + StateDiskType: conf.Provider.AWS.StateDiskType, + Region: conf.Provider.AWS.Region, + Zone: conf.Provider.AWS.Zone, + InstanceType: conf.Provider.AWS.InstanceType, + AMIImageID: imageRef, + IAMProfileControlPlane: conf.Provider.AWS.IAMProfileControlPlane, + IAMProfileWorkerNodes: conf.Provider.AWS.IAMProfileWorkerNodes, + Debug: conf.IsDebugCluster(), + } + return targets, vars, nil + case cloudprovider.Azure: + targets := []string{"azurerm_attestation_provider.attestation_provider"} + + // Azure Terraform provider is very strict about it's casing + imageRef = strings.Replace(imageRef, "CommunityGalleries", "communityGalleries", 1) + imageRef = strings.Replace(imageRef, "Images", "images", 1) + imageRef = strings.Replace(imageRef, "Versions", "versions", 1) + + vars := &terraform.AzureClusterVariables{ + CommonVariables: commonVariables, + Location: conf.Provider.Azure.Location, + ResourceGroup: conf.Provider.Azure.ResourceGroup, + UserAssignedIdentity: conf.Provider.Azure.UserAssignedIdentity, + InstanceType: conf.Provider.Azure.InstanceType, + StateDiskType: conf.Provider.Azure.StateDiskType, + ImageID: imageRef, + SecureBoot: *conf.Provider.Azure.SecureBoot, + CreateMAA: conf.GetAttestationConfig().GetVariant().Equal(variant.AzureSEVSNP{}), + Debug: conf.IsDebugCluster(), + } + return targets, vars, nil + case cloudprovider.GCP: + targets := []string{} + + vars := &terraform.GCPClusterVariables{ + CommonVariables: commonVariables, + Project: conf.Provider.GCP.Project, + Region: conf.Provider.GCP.Region, + Zone: conf.Provider.GCP.Zone, + CredentialsFile: conf.Provider.GCP.ServiceAccountKeyPath, + InstanceType: conf.Provider.GCP.InstanceType, + StateDiskType: conf.Provider.GCP.StateDiskType, + ImageID: imageRef, + Debug: conf.IsDebugCluster(), + } + return targets, vars, nil + default: + return nil, nil, fmt.Errorf("unsupported provider: %s", conf.GetProvider()) + } +} + +type imageFetcher interface { + FetchReference(ctx context.Context, conf *config.Config) (string, error) +} + // upgradeAttestConfigIfDiff checks if the locally configured measurements are different from the cluster's measurements. // If so the function will ask the user to confirm (if --yes is not set) and upgrade the measurements only. func (u *upgradeApplyCmd) upgradeAttestConfigIfDiff(cmd *cobra.Command, newConfig config.AttestationCfg, flags upgradeApplyFlags) error { @@ -193,14 +340,30 @@ func parseUpgradeApplyFlags(cmd *cobra.Command) (upgradeApplyFlags, error) { return upgradeApplyFlags{}, fmt.Errorf("parsing force argument: %w", err) } - return upgradeApplyFlags{configPath: configPath, yes: yes, upgradeTimeout: timeout, force: force}, nil + logLevelString, err := cmd.Flags().GetString("tf-log") + if err != nil { + return upgradeApplyFlags{}, fmt.Errorf("parsing tf-log string: %w", err) + } + logLevel, err := terraform.ParseLogLevel(logLevelString) + if err != nil { + return upgradeApplyFlags{}, fmt.Errorf("parsing Terraform log level %s: %w", logLevelString, err) + } + + return upgradeApplyFlags{ + configPath: configPath, + yes: yes, + upgradeTimeout: timeout, + force: force, + terraformLogLevel: logLevel, + }, nil } type upgradeApplyFlags struct { - configPath string - yes bool - upgradeTimeout time.Duration - force bool + configPath string + yes bool + upgradeTimeout time.Duration + force bool + terraformLogLevel terraform.LogLevel } type cloudUpgrader interface { @@ -208,4 +371,8 @@ type cloudUpgrader interface { UpgradeHelmServices(ctx context.Context, config *config.Config, timeout time.Duration, allowDestructive bool) error UpdateAttestationConfig(ctx context.Context, newConfig config.AttestationCfg) error GetClusterAttestationConfig(ctx context.Context, variant variant.Variant) (config.AttestationCfg, *corev1.ConfigMap, error) + PlanTerraformMigrations(ctx context.Context, opts upgrade.TerraformUpgradeOptions) (bool, error) + ApplyTerraformMigrations(ctx context.Context, fileHandler file.Handler, opts upgrade.TerraformUpgradeOptions) error + CheckTerraformMigrations(fileHandler file.Handler) error + CleanUpTerraformMigrations(fileHandler file.Handler) error } diff --git a/cli/internal/cmd/upgradeapply_test.go b/cli/internal/cmd/upgradeapply_test.go index c9e2f7b90..6b6cc933b 100644 --- a/cli/internal/cmd/upgradeapply_test.go +++ b/cli/internal/cmd/upgradeapply_test.go @@ -7,6 +7,7 @@ SPDX-License-Identifier: AGPL-3.0-only package cmd import ( + "bytes" "context" "errors" "testing" @@ -14,6 +15,7 @@ import ( "github.com/edgelesssys/constellation/v2/cli/internal/clusterid" "github.com/edgelesssys/constellation/v2/cli/internal/kubernetes" + "github.com/edgelesssys/constellation/v2/cli/internal/upgrade" "github.com/edgelesssys/constellation/v2/internal/cloud/cloudprovider" "github.com/edgelesssys/constellation/v2/internal/config" "github.com/edgelesssys/constellation/v2/internal/constants" @@ -30,10 +32,14 @@ func TestUpgradeApply(t *testing.T) { someErr := errors.New("some error") testCases := map[string]struct { upgrader stubUpgrader + fetcher stubImageFetcher wantErr bool + yesFlag bool + stdin string }{ "success": { upgrader: stubUpgrader{currentConfig: config.DefaultForAzureSEVSNP()}, + yesFlag: true, }, "nodeVersion some error": { upgrader: stubUpgrader{ @@ -41,12 +47,14 @@ func TestUpgradeApply(t *testing.T) { nodeVersionErr: someErr, }, wantErr: true, + yesFlag: true, }, "nodeVersion in progress error": { upgrader: stubUpgrader{ currentConfig: config.DefaultForAzureSEVSNP(), nodeVersionErr: kubernetes.ErrInProgress, }, + yesFlag: true, }, "helm other error": { upgrader: stubUpgrader{ @@ -54,6 +62,63 @@ func TestUpgradeApply(t *testing.T) { helmErr: someErr, }, wantErr: true, + fetcher: stubImageFetcher{}, + yesFlag: true, + }, + "check terraform error": { + upgrader: stubUpgrader{ + currentConfig: config.DefaultForAzureSEVSNP(), + checkTerraformErr: someErr, + }, + fetcher: stubImageFetcher{}, + wantErr: true, + yesFlag: true, + }, + "abort": { + upgrader: stubUpgrader{ + currentConfig: config.DefaultForAzureSEVSNP(), + terraformDiff: true, + }, + fetcher: stubImageFetcher{}, + wantErr: true, + stdin: "no\n", + }, + "clean terraform error": { + upgrader: stubUpgrader{ + currentConfig: config.DefaultForAzureSEVSNP(), + cleanTerraformErr: someErr, + terraformDiff: true, + }, + fetcher: stubImageFetcher{}, + wantErr: true, + stdin: "no\n", + }, + "plan terraform error": { + upgrader: stubUpgrader{ + currentConfig: config.DefaultForAzureSEVSNP(), + planTerraformErr: someErr, + }, + fetcher: stubImageFetcher{}, + wantErr: true, + yesFlag: true, + }, + "apply terraform error": { + upgrader: stubUpgrader{ + currentConfig: config.DefaultForAzureSEVSNP(), + applyTerraformErr: someErr, + terraformDiff: true, + }, + fetcher: stubImageFetcher{}, + wantErr: true, + yesFlag: true, + }, + "fetch reference error": { + upgrader: stubUpgrader{ + currentConfig: config.DefaultForAzureSEVSNP(), + }, + fetcher: stubImageFetcher{fetchReferenceErr: someErr}, + wantErr: true, + yesFlag: true, }, } @@ -62,19 +127,23 @@ func TestUpgradeApply(t *testing.T) { assert := assert.New(t) require := require.New(t) cmd := newUpgradeApplyCmd() + cmd.SetIn(bytes.NewBufferString(tc.stdin)) cmd.Flags().String("config", constants.ConfigFilename, "") // register persistent flag manually cmd.Flags().Bool("force", true, "") // register persistent flag manually + cmd.Flags().String("tf-log", "DEBUG", "") // register persistent flag manually - err := cmd.Flags().Set("yes", "true") - require.NoError(err) + if tc.yesFlag { + err := cmd.Flags().Set("yes", "true") + require.NoError(err) + } handler := file.NewHandler(afero.NewMemMapFs()) cfg := defaultConfigWithExpectedMeasurements(t, config.Default(), cloudprovider.Azure) require.NoError(handler.WriteYAML(constants.ConfigFilename, cfg)) require.NoError(handler.WriteJSON(constants.ClusterIDsFileName, clusterid.File{})) - upgrader := upgradeApplyCmd{upgrader: tc.upgrader, log: logger.NewTest(t)} - err = upgrader.upgradeApply(cmd, handler) + upgrader := upgradeApplyCmd{upgrader: tc.upgrader, log: logger.NewTest(t), fetcher: tc.fetcher} + err := upgrader.upgradeApply(cmd, handler) if tc.wantErr { assert.Error(err) } else { @@ -85,9 +154,14 @@ func TestUpgradeApply(t *testing.T) { } type stubUpgrader struct { - currentConfig config.AttestationCfg - nodeVersionErr error - helmErr error + currentConfig config.AttestationCfg + nodeVersionErr error + helmErr error + terraformDiff bool + planTerraformErr error + checkTerraformErr error + applyTerraformErr error + cleanTerraformErr error } func (u stubUpgrader) UpgradeNodeVersion(_ context.Context, _ *config.Config) error { @@ -105,3 +179,27 @@ func (u stubUpgrader) UpdateAttestationConfig(_ context.Context, _ config.Attest func (u stubUpgrader) GetClusterAttestationConfig(_ context.Context, _ variant.Variant) (config.AttestationCfg, *corev1.ConfigMap, error) { return u.currentConfig, &corev1.ConfigMap{}, nil } + +func (u stubUpgrader) CheckTerraformMigrations(file.Handler) error { + return u.checkTerraformErr +} + +func (u stubUpgrader) CleanUpTerraformMigrations(file.Handler) error { + return u.cleanTerraformErr +} + +func (u stubUpgrader) PlanTerraformMigrations(context.Context, upgrade.TerraformUpgradeOptions) (bool, error) { + return u.terraformDiff, u.planTerraformErr +} + +func (u stubUpgrader) ApplyTerraformMigrations(context.Context, file.Handler, upgrade.TerraformUpgradeOptions) error { + return u.applyTerraformErr +} + +type stubImageFetcher struct { + fetchReferenceErr error +} + +func (s stubImageFetcher) FetchReference(context.Context, *config.Config) (string, error) { + return "", s.fetchReferenceErr +} diff --git a/cli/internal/cmd/upgradecheck.go b/cli/internal/cmd/upgradecheck.go index 8d1458301..a086f5522 100644 --- a/cli/internal/cmd/upgradecheck.go +++ b/cli/internal/cmd/upgradecheck.go @@ -62,7 +62,7 @@ func runUpgradeCheck(cmd *cobra.Command, _ []string) error { if err != nil { return err } - checker, err := kubernetes.NewUpgrader(cmd.OutOrStdout(), log) + checker, err := kubernetes.NewUpgrader(cmd.Context(), cmd.OutOrStdout(), log) if err != nil { return err } diff --git a/cli/internal/helm/backup.go b/cli/internal/helm/backup.go index ad081d678..f0da75bf2 100644 --- a/cli/internal/helm/backup.go +++ b/cli/internal/helm/backup.go @@ -11,14 +11,15 @@ import ( "fmt" "path/filepath" + "github.com/edgelesssys/constellation/v2/internal/constants" apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" "k8s.io/apimachinery/pkg/runtime/schema" "sigs.k8s.io/yaml" ) -const ( - crdBackupFolder = "constellation-upgrade/backups/crds/" - backupFolder = "constellation-upgrade/backups/" +var ( + backupFolder = filepath.Join(constants.UpgradeDir, "backups") + string(filepath.Separator) + crdBackupFolder = filepath.Join(backupFolder, "crds") + string(filepath.Separator) ) func (c *Client) backupCRDs(ctx context.Context) ([]apiextensionsv1.CustomResourceDefinition, error) { diff --git a/cli/internal/kubernetes/BUILD.bazel b/cli/internal/kubernetes/BUILD.bazel index 43a5bb9d1..a676eba4c 100644 --- a/cli/internal/kubernetes/BUILD.bazel +++ b/cli/internal/kubernetes/BUILD.bazel @@ -13,10 +13,13 @@ go_library( deps = [ "//cli/internal/helm", "//cli/internal/image", + "//cli/internal/terraform", + "//cli/internal/upgrade", "//internal/attestation/measurements", "//internal/compatibility", "//internal/config", "//internal/constants", + "//internal/file", "//internal/kubernetes", "//internal/kubernetes/kubectl", "//internal/variant", diff --git a/cli/internal/kubernetes/upgrade.go b/cli/internal/kubernetes/upgrade.go index 52ae2e027..5ffc6c1ff 100644 --- a/cli/internal/kubernetes/upgrade.go +++ b/cli/internal/kubernetes/upgrade.go @@ -12,14 +12,18 @@ import ( "errors" "fmt" "io" + "path/filepath" "time" "github.com/edgelesssys/constellation/v2/cli/internal/helm" "github.com/edgelesssys/constellation/v2/cli/internal/image" + "github.com/edgelesssys/constellation/v2/cli/internal/terraform" + "github.com/edgelesssys/constellation/v2/cli/internal/upgrade" "github.com/edgelesssys/constellation/v2/internal/attestation/measurements" "github.com/edgelesssys/constellation/v2/internal/compatibility" "github.com/edgelesssys/constellation/v2/internal/config" "github.com/edgelesssys/constellation/v2/internal/constants" + "github.com/edgelesssys/constellation/v2/internal/file" internalk8s "github.com/edgelesssys/constellation/v2/internal/kubernetes" "github.com/edgelesssys/constellation/v2/internal/kubernetes/kubectl" "github.com/edgelesssys/constellation/v2/internal/variant" @@ -77,11 +81,12 @@ type Upgrader struct { helmClient helmInterface imageFetcher imageFetcher outWriter io.Writer + tfUpgrader *upgrade.TerraformUpgrader log debugLog } // NewUpgrader returns a new Upgrader. -func NewUpgrader(outWriter io.Writer, log debugLog) (*Upgrader, error) { +func NewUpgrader(ctx context.Context, outWriter io.Writer, log debugLog) (*Upgrader, error) { kubeConfig, err := clientcmd.BuildConfigFromFlags("", constants.AdminConfFilename) if err != nil { return nil, fmt.Errorf("building kubernetes config: %w", err) @@ -103,16 +108,54 @@ func NewUpgrader(outWriter io.Writer, log debugLog) (*Upgrader, error) { return nil, fmt.Errorf("setting up helm client: %w", err) } + tfClient, err := terraform.New(ctx, filepath.Join(constants.UpgradeDir, constants.TerraformUpgradeWorkingDir)) + if err != nil { + return nil, fmt.Errorf("setting up terraform client: %w", err) + } + + tfUpgrader, err := upgrade.NewTerraformUpgrader(tfClient, outWriter) + if err != nil { + return nil, fmt.Errorf("setting up terraform upgrader: %w", err) + } + return &Upgrader{ stableInterface: &stableClient{client: kubeClient}, dynamicInterface: &NodeVersionClient{client: unstructuredClient}, helmClient: helmClient, imageFetcher: image.New(), outWriter: outWriter, + tfUpgrader: tfUpgrader, log: log, }, nil } +// CheckTerraformMigrations checks whether Terraform migrations are possible in the current workspace. +// If the files that will be written during the upgrade already exist, it returns an error. +func (u *Upgrader) CheckTerraformMigrations(fileHandler file.Handler) error { + return u.tfUpgrader.CheckTerraformMigrations(fileHandler) +} + +// CleanUpTerraformMigrations cleans up the Terraform migration workspace, for example when an upgrade is +// aborted by the user. +func (u *Upgrader) CleanUpTerraformMigrations(fileHandler file.Handler) error { + return u.tfUpgrader.CleanUpTerraformMigrations(fileHandler) +} + +// PlanTerraformMigrations prepares the upgrade workspace and plans the Terraform migrations for the Constellation upgrade. +// If a diff exists, it's being written to the upgrader's output writer. It also returns +// a bool indicating whether a diff exists. +func (u *Upgrader) PlanTerraformMigrations(ctx context.Context, opts upgrade.TerraformUpgradeOptions) (bool, error) { + return u.tfUpgrader.PlanTerraformMigrations(ctx, opts) +} + +// ApplyTerraformMigrations applies the migerations planned by PlanTerraformMigrations. +// If PlanTerraformMigrations has not been executed before, it will return an error. +// In case of a successful upgrade, the output will be written to the specified file and the old Terraform directory is replaced +// By the new one. +func (u *Upgrader) ApplyTerraformMigrations(ctx context.Context, fileHandler file.Handler, opts upgrade.TerraformUpgradeOptions) error { + return u.tfUpgrader.ApplyTerraformMigrations(ctx, fileHandler, opts) +} + // UpgradeHelmServices upgrade helm services. func (u *Upgrader) UpgradeHelmServices(ctx context.Context, config *config.Config, timeout time.Duration, allowDestructive bool) error { return u.helmClient.Upgrade(ctx, config, timeout, allowDestructive) diff --git a/cli/internal/terraform/loader.go b/cli/internal/terraform/loader.go index 083784176..ba2f8eeb4 100644 --- a/cli/internal/terraform/loader.go +++ b/cli/internal/terraform/loader.go @@ -10,10 +10,12 @@ import ( "bytes" "embed" "errors" + "fmt" "io/fs" "path/filepath" "strings" + "github.com/edgelesssys/constellation/v2/internal/constants" "github.com/edgelesssys/constellation/v2/internal/file" "github.com/spf13/afero" ) @@ -27,8 +29,35 @@ var terraformFS embed.FS // prepareWorkspace loads the embedded Terraform files, // and writes them into the workspace. -func prepareWorkspace(path string, fileHandler file.Handler, workingDir string) error { - rootDir := path +func prepareWorkspace(rootDir string, fileHandler file.Handler, workingDir string) error { + return terraformCopier(fileHandler, rootDir, workingDir) +} + +// prepareUpgradeWorkspace takes the Terraform state file from the old workspace and the +// embedded Terraform files and writes them into the new workspace. +func prepareUpgradeWorkspace(rootDir string, fileHandler file.Handler, oldWorkingDir, newWorkingDir string) error { + // backup old workspace + if err := fileHandler.CopyDir( + oldWorkingDir, + filepath.Join(constants.UpgradeDir, constants.TerraformUpgradeBackupDir), + ); err != nil { + return fmt.Errorf("backing up old workspace: %w", err) + } + + // copy state file + if err := fileHandler.CopyFile( + filepath.Join(oldWorkingDir, "terraform.tfstate"), + filepath.Join(newWorkingDir, "terraform.tfstate"), + file.OptMkdirAll, + ); err != nil { + return fmt.Errorf("copying state file: %w", err) + } + + return terraformCopier(fileHandler, rootDir, newWorkingDir) +} + +// terraformCopier copies the embedded Terraform files into the workspace. +func terraformCopier(fileHandler file.Handler, rootDir, workingDir string) error { return fs.WalkDir(terraformFS, rootDir, func(path string, d fs.DirEntry, err error) error { if err != nil { return err diff --git a/cli/internal/terraform/loader_test.go b/cli/internal/terraform/loader_test.go index 1622223eb..fa1b3a755 100644 --- a/cli/internal/terraform/loader_test.go +++ b/cli/internal/terraform/loader_test.go @@ -21,7 +21,7 @@ import ( "github.com/stretchr/testify/require" ) -func TestLoader(t *testing.T) { +func TestPrepareWorkspace(t *testing.T) { testCases := map[string]struct { pathBase string provider cloudprovider.Provider @@ -109,29 +109,114 @@ func TestLoader(t *testing.T) { err := prepareWorkspace(path, file, constants.TerraformWorkingDir) require.NoError(err) - checkFiles(t, file, func(err error) { assert.NoError(err) }, tc.fileList) + checkFiles(t, file, func(err error) { assert.NoError(err) }, constants.TerraformWorkingDir, tc.fileList) if tc.testAlreadyUnpacked { // Let's try the same again and check if we don't get a "file already exists" error. require.NoError(file.Remove(filepath.Join(constants.TerraformWorkingDir, "variables.tf"))) err := prepareWorkspace(path, file, constants.TerraformWorkingDir) assert.NoError(err) - checkFiles(t, file, func(err error) { assert.NoError(err) }, tc.fileList) + checkFiles(t, file, func(err error) { assert.NoError(err) }, constants.TerraformWorkingDir, tc.fileList) } err = cleanUpWorkspace(file, constants.TerraformWorkingDir) require.NoError(err) - checkFiles(t, file, func(err error) { assert.ErrorIs(err, fs.ErrNotExist) }, tc.fileList) + checkFiles(t, file, func(err error) { assert.ErrorIs(err, fs.ErrNotExist) }, constants.TerraformWorkingDir, tc.fileList) }) } } -func checkFiles(t *testing.T, file file.Handler, assertion func(error), files []string) { +func TestPrepareUpgradeWorkspace(t *testing.T) { + testCases := map[string]struct { + pathBase string + provider cloudprovider.Provider + oldWorkingDir string + newWorkingDir string + oldWorkspaceFiles []string + newWorkspaceFiles []string + expectedFiles []string + testAlreadyUnpacked bool + wantErr bool + }{ + "works": { + pathBase: "terraform", + provider: cloudprovider.AWS, + oldWorkingDir: "old", + newWorkingDir: "new", + oldWorkspaceFiles: []string{"terraform.tfstate"}, + expectedFiles: []string{ + "main.tf", + "variables.tf", + "outputs.tf", + "modules", + "terraform.tfstate", + }, + }, + "state file does not exist": { + pathBase: "terraform", + provider: cloudprovider.AWS, + oldWorkingDir: "old", + newWorkingDir: "new", + oldWorkspaceFiles: []string{}, + expectedFiles: []string{}, + wantErr: true, + }, + "terraform files already exist in new dir": { + pathBase: "terraform", + provider: cloudprovider.AWS, + oldWorkingDir: "old", + newWorkingDir: "new", + oldWorkspaceFiles: []string{"terraform.tfstate"}, + newWorkspaceFiles: []string{"main.tf"}, + wantErr: true, + }, + } + + for name, tc := range testCases { + t.Run(name, func(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + + file := file.NewHandler(afero.NewMemMapFs()) + + path := path.Join(tc.pathBase, strings.ToLower(tc.provider.String())) + + createFiles(t, file, tc.oldWorkspaceFiles, tc.oldWorkingDir) + createFiles(t, file, tc.newWorkspaceFiles, tc.newWorkingDir) + + err := prepareUpgradeWorkspace(path, file, tc.oldWorkingDir, tc.newWorkingDir) + + if tc.wantErr { + require.Error(err) + } else { + require.NoError(err) + } + checkFiles(t, file, func(err error) { assert.NoError(err) }, tc.newWorkingDir, tc.expectedFiles) + checkFiles(t, file, func(err error) { assert.NoError(err) }, + filepath.Join(constants.UpgradeDir, constants.TerraformUpgradeBackupDir), + tc.oldWorkspaceFiles, + ) + }) + } +} + +func checkFiles(t *testing.T, fileHandler file.Handler, assertion func(error), dir string, files []string) { t.Helper() for _, f := range files { - path := filepath.Join(constants.TerraformWorkingDir, f) - _, err := file.Stat(path) + path := filepath.Join(dir, f) + _, err := fileHandler.Stat(path) assertion(err) } } + +func createFiles(t *testing.T, fileHandler file.Handler, fileList []string, targetDir string) { + t.Helper() + require := require.New(t) + + for _, f := range fileList { + path := filepath.Join(targetDir, f) + err := fileHandler.Write(path, []byte("1234"), file.OptOverwrite, file.OptMkdirAll) + require.NoError(err) + } +} diff --git a/cli/internal/terraform/terraform.go b/cli/internal/terraform/terraform.go index 14d3ae117..51e4845f2 100644 --- a/cli/internal/terraform/terraform.go +++ b/cli/internal/terraform/terraform.go @@ -18,6 +18,7 @@ import ( "context" "errors" "fmt" + "io" "path/filepath" "github.com/edgelesssys/constellation/v2/internal/cloud/cloudprovider" @@ -78,14 +79,24 @@ func (c *Client) Show(ctx context.Context) (*tfjson.State, error) { // PrepareWorkspace prepares a Terraform workspace for a Constellation cluster. func (c *Client) PrepareWorkspace(path string, vars Variables) error { if err := prepareWorkspace(path, c.file, c.workingDir); err != nil { - return err + return fmt.Errorf("prepare workspace: %w", err) + } + + return c.writeVars(vars) +} + +// PrepareUpgradeWorkspace prepares a Terraform workspace for a Constellation version upgrade. +// It copies the Terraform state from the old working dir and the embedded Terraform files into the new working dir. +func (c *Client) PrepareUpgradeWorkspace(path, oldWorkingDir, newWorkingDir string, vars Variables) error { + if err := prepareUpgradeWorkspace(path, c.file, oldWorkingDir, newWorkingDir); err != nil { + return fmt.Errorf("prepare upgrade workspace: %w", err) } return c.writeVars(vars) } // CreateCluster creates a Constellation cluster using Terraform. -func (c *Client) CreateCluster(ctx context.Context, logLevel LogLevel) (CreateOutput, error) { +func (c *Client) CreateCluster(ctx context.Context, logLevel LogLevel, targets ...string) (CreateOutput, error) { if err := c.setLogLevel(logLevel); err != nil { return CreateOutput{}, fmt.Errorf("set terraform log level %s: %w", logLevel.String(), err) } @@ -94,7 +105,12 @@ func (c *Client) CreateCluster(ctx context.Context, logLevel LogLevel) (CreateOu return CreateOutput{}, fmt.Errorf("terraform init: %w", err) } - if err := c.tf.Apply(ctx); err != nil { + opts := []tfexec.ApplyOption{} + for _, target := range targets { + opts = append(opts, tfexec.Target(target)) + } + + if err := c.tf.Apply(ctx, opts...); err != nil { return CreateOutput{}, fmt.Errorf("terraform apply: %w", err) } @@ -294,6 +310,45 @@ func (c *Client) CreateIAMConfig(ctx context.Context, provider cloudprovider.Pro } } +// Plan determines the diff that will be applied by Terraform. The plan output is written to the planFile. +// If there is a diff, the returned bool is true. Otherwise, it is false. +func (c *Client) Plan(ctx context.Context, logLevel LogLevel, planFile string, targets ...string) (bool, error) { + if err := c.setLogLevel(logLevel); err != nil { + return false, fmt.Errorf("set terraform log level %s: %w", logLevel.String(), err) + } + + if err := c.tf.Init(ctx); err != nil { + return false, fmt.Errorf("terraform init: %w", err) + } + + opts := []tfexec.PlanOption{ + tfexec.Out(planFile), + } + for _, target := range targets { + opts = append(opts, tfexec.Target(target)) + } + return c.tf.Plan(ctx, opts...) +} + +// ShowPlan formats the diff in planFilePath and writes it to the specified output. +func (c *Client) ShowPlan(ctx context.Context, logLevel LogLevel, planFilePath string, output io.Writer) error { + if err := c.setLogLevel(logLevel); err != nil { + return fmt.Errorf("set terraform log level %s: %w", logLevel.String(), err) + } + + planResult, err := c.tf.ShowPlanFileRaw(ctx, planFilePath) + if err != nil { + return fmt.Errorf("terraform show plan: %w", err) + } + + _, err = output.Write([]byte(planResult)) + if err != nil { + return fmt.Errorf("write plan output: %w", err) + } + + return nil +} + // Destroy destroys Terraform-created cloud resources. func (c *Client) Destroy(ctx context.Context, logLevel LogLevel) error { if err := c.setLogLevel(logLevel); err != nil { @@ -386,6 +441,8 @@ type tfInterface interface { Destroy(context.Context, ...tfexec.DestroyOption) error Init(context.Context, ...tfexec.InitOption) error Show(context.Context, ...tfexec.ShowOption) (*tfjson.State, error) + Plan(ctx context.Context, opts ...tfexec.PlanOption) (bool, error) + ShowPlanFileRaw(ctx context.Context, planPath string, opts ...tfexec.ShowOption) (string, error) SetLog(level string) error SetLogPath(path string) error } diff --git a/cli/internal/terraform/terraform_test.go b/cli/internal/terraform/terraform_test.go index 66094179f..6f1156360 100644 --- a/cli/internal/terraform/terraform_test.go +++ b/cli/internal/terraform/terraform_test.go @@ -7,6 +7,7 @@ SPDX-License-Identifier: AGPL-3.0-only package terraform import ( + "bytes" "context" "errors" "io/fs" @@ -934,14 +935,143 @@ func TestLogLevelString(t *testing.T) { } } +func TestPlan(t *testing.T) { + someError := errors.New("some error") + + testCases := map[string]struct { + pathBase string + tf *stubTerraform + fs afero.Fs + wantErr bool + }{ + "plan succeeds": { + pathBase: "terraform", + tf: &stubTerraform{}, + fs: afero.NewMemMapFs(), + }, + "set log path fails": { + pathBase: "terraform", + tf: &stubTerraform{ + setLogPathErr: someError, + }, + fs: afero.NewMemMapFs(), + wantErr: true, + }, + "set log fails": { + pathBase: "terraform", + tf: &stubTerraform{ + setLogErr: someError, + }, + fs: afero.NewMemMapFs(), + wantErr: true, + }, + "plan fails": { + pathBase: "terraform", + tf: &stubTerraform{ + planJSONErr: someError, + }, + fs: afero.NewMemMapFs(), + wantErr: true, + }, + "init fails": { + pathBase: "terraform", + tf: &stubTerraform{ + initErr: someError, + }, + fs: afero.NewMemMapFs(), + wantErr: true, + }, + } + + for name, tc := range testCases { + t.Run(name, func(t *testing.T) { + require := require.New(t) + + c := &Client{ + file: file.NewHandler(tc.fs), + tf: tc.tf, + workingDir: tc.pathBase, + } + + _, err := c.Plan(context.Background(), LogLevelDebug, constants.TerraformUpgradePlanFile) + if tc.wantErr { + require.Error(err) + } else { + require.NoError(err) + } + }) + } +} + +func TestShowPlan(t *testing.T) { + someError := errors.New("some error") + testCases := map[string]struct { + pathBase string + tf *stubTerraform + fs afero.Fs + wantErr bool + }{ + "show plan succeeds": { + pathBase: "terraform", + tf: &stubTerraform{}, + fs: afero.NewMemMapFs(), + }, + "set log path fails": { + pathBase: "terraform", + tf: &stubTerraform{ + setLogPathErr: someError, + }, + fs: afero.NewMemMapFs(), + wantErr: true, + }, + "set log fails": { + pathBase: "terraform", + tf: &stubTerraform{ + setLogErr: someError, + }, + fs: afero.NewMemMapFs(), + wantErr: true, + }, + "show plan file fails": { + pathBase: "terraform", + tf: &stubTerraform{ + showPlanFileErr: someError, + }, + fs: afero.NewMemMapFs(), + wantErr: true, + }, + } + + for name, tc := range testCases { + t.Run(name, func(t *testing.T) { + require := require.New(t) + + c := &Client{ + file: file.NewHandler(tc.fs), + tf: tc.tf, + workingDir: tc.pathBase, + } + + err := c.ShowPlan(context.Background(), LogLevelDebug, "", bytes.NewBuffer(nil)) + if tc.wantErr { + require.Error(err) + } else { + require.NoError(err) + } + }) + } +} + type stubTerraform struct { - applyErr error - destroyErr error - initErr error - showErr error - setLogErr error - setLogPathErr error - showState *tfjson.State + applyErr error + destroyErr error + initErr error + showErr error + setLogErr error + setLogPathErr error + planJSONErr error + showPlanFileErr error + showState *tfjson.State } func (s *stubTerraform) Apply(context.Context, ...tfexec.ApplyOption) error { @@ -960,6 +1090,14 @@ func (s *stubTerraform) Show(context.Context, ...tfexec.ShowOption) (*tfjson.Sta return s.showState, s.showErr } +func (s *stubTerraform) Plan(context.Context, ...tfexec.PlanOption) (bool, error) { + return false, s.planJSONErr +} + +func (s *stubTerraform) ShowPlanFileRaw(context.Context, string, ...tfexec.ShowOption) (string, error) { + return "", s.showPlanFileErr +} + func (s *stubTerraform) SetLog(_ string) error { return s.setLogErr } diff --git a/cli/internal/upgrade/BUILD.bazel b/cli/internal/upgrade/BUILD.bazel new file mode 100644 index 000000000..9e7d2ace6 --- /dev/null +++ b/cli/internal/upgrade/BUILD.bazel @@ -0,0 +1,34 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library") +load("//bazel/go:go_test.bzl", "go_test") + +go_library( + name = "upgrade", + srcs = [ + "terraform.go", + "upgrade.go", + ], + importpath = "github.com/edgelesssys/constellation/v2/cli/internal/upgrade", + visibility = ["//cli:__subpackages__"], + deps = [ + "//cli/internal/clusterid", + "//cli/internal/terraform", + "//internal/cloud/cloudprovider", + "//internal/constants", + "//internal/file", + ], +) + +go_test( + name = "upgrade_test", + srcs = ["terraform_test.go"], + embed = [":upgrade"], + deps = [ + "//cli/internal/terraform", + "//internal/cloud/cloudprovider", + "//internal/constants", + "//internal/file", + "@com_github_spf13_afero//:afero", + "@com_github_stretchr_testify//assert", + "@com_github_stretchr_testify//require", + ], +) diff --git a/cli/internal/upgrade/terraform.go b/cli/internal/upgrade/terraform.go new file mode 100644 index 000000000..6f7e68770 --- /dev/null +++ b/cli/internal/upgrade/terraform.go @@ -0,0 +1,175 @@ +/* +Copyright (c) Edgeless Systems GmbH + +SPDX-License-Identifier: AGPL-3.0-only +*/ + +package upgrade + +import ( + "context" + "fmt" + "io" + "os" + "path/filepath" + "strings" + + "github.com/edgelesssys/constellation/v2/cli/internal/clusterid" + "github.com/edgelesssys/constellation/v2/cli/internal/terraform" + "github.com/edgelesssys/constellation/v2/internal/cloud/cloudprovider" + "github.com/edgelesssys/constellation/v2/internal/constants" + "github.com/edgelesssys/constellation/v2/internal/file" +) + +// NewTerraformUpgrader returns a new TerraformUpgrader. +func NewTerraformUpgrader(tfClient tfClient, outWriter io.Writer) (*TerraformUpgrader, error) { + return &TerraformUpgrader{ + tf: tfClient, + outWriter: outWriter, + }, nil +} + +// TerraformUpgrader is responsible for performing Terraform migrations on cluster upgrades. +type TerraformUpgrader struct { + tf tfClient + outWriter io.Writer +} + +// TerraformUpgradeOptions are the options used for the Terraform upgrade. +type TerraformUpgradeOptions struct { + // LogLevel is the log level used for Terraform. + LogLevel terraform.LogLevel + // CSP is the cloud provider to perform the upgrade on. + CSP cloudprovider.Provider + // Vars are the Terraform variables used for the upgrade. + Vars terraform.Variables + // Targets are the Terraform targets used for the upgrade. + Targets []string + // OutputFile is the file to write the Terraform output to. + OutputFile string +} + +// CheckTerraformMigrations checks whether Terraform migrations are possible in the current workspace. +// If the files that will be written during the upgrade already exist, it returns an error. +func (u *TerraformUpgrader) CheckTerraformMigrations(fileHandler file.Handler) error { + var existingFiles []string + filesToCheck := []string{ + constants.TerraformMigrationOutputFile, + filepath.Join(constants.UpgradeDir, constants.TerraformUpgradeBackupDir), + } + + for _, f := range filesToCheck { + if err := checkFileExists(fileHandler, &existingFiles, f); err != nil { + return fmt.Errorf("checking terraform migrations: %w", err) + } + } + + if len(existingFiles) > 0 { + return fmt.Errorf("file(s) %s already exist", strings.Join(existingFiles, ", ")) + } + return nil +} + +// checkFileExists checks whether a file exists and adds it to the existingFiles slice if it does. +func checkFileExists(fileHandler file.Handler, existingFiles *[]string, filename string) error { + _, err := fileHandler.Stat(filename) + if err != nil { + if !os.IsNotExist(err) { + return fmt.Errorf("checking %s: %w", filename, err) + } + return nil + } + + *existingFiles = append(*existingFiles, filename) + return nil +} + +// PlanTerraformMigrations prepares the upgrade workspace and plans the Terraform migrations for the Constellation upgrade. +// If a diff exists, it's being written to the upgrader's output writer. It also returns +// a bool indicating whether a diff exists. +func (u *TerraformUpgrader) PlanTerraformMigrations(ctx context.Context, opts TerraformUpgradeOptions) (bool, error) { + err := u.tf.PrepareUpgradeWorkspace( + filepath.Join("terraform", strings.ToLower(opts.CSP.String())), + constants.TerraformWorkingDir, + filepath.Join(constants.UpgradeDir, constants.TerraformUpgradeWorkingDir), + opts.Vars, + ) + if err != nil { + return false, fmt.Errorf("preparing terraform workspace: %w", err) + } + + hasDiff, err := u.tf.Plan(ctx, opts.LogLevel, constants.TerraformUpgradePlanFile, opts.Targets...) + if err != nil { + return false, fmt.Errorf("terraform plan: %w", err) + } + + if hasDiff { + if err := u.tf.ShowPlan(ctx, opts.LogLevel, constants.TerraformUpgradePlanFile, u.outWriter); err != nil { + return false, fmt.Errorf("terraform show plan: %w", err) + } + } + + return hasDiff, nil +} + +// CleanUpTerraformMigrations cleans up the Terraform migration workspace, for example when an upgrade is +// aborted by the user. +func (u *TerraformUpgrader) CleanUpTerraformMigrations(fileHandler file.Handler) error { + cleanupFiles := []string{ + filepath.Join(constants.UpgradeDir, constants.TerraformUpgradeBackupDir), + filepath.Join(constants.UpgradeDir, constants.TerraformUpgradeWorkingDir), + } + + for _, f := range cleanupFiles { + if err := fileHandler.RemoveAll(f); err != nil { + return fmt.Errorf("cleaning up file %s: %w", f, err) + } + } + + return nil +} + +// ApplyTerraformMigrations applies the migerations planned by PlanTerraformMigrations. +// If PlanTerraformMigrations has not been executed before, it will return an error. +// In case of a successful upgrade, the output will be written to the specified file and the old Terraform directory is replaced +// By the new one. +func (u *TerraformUpgrader) ApplyTerraformMigrations(ctx context.Context, fileHandler file.Handler, opts TerraformUpgradeOptions) error { + tfOutput, err := u.tf.CreateCluster(ctx, opts.LogLevel, opts.Targets...) + if err != nil { + return fmt.Errorf("terraform apply: %w", err) + } + + outputFileContents := clusterid.File{ + CloudProvider: opts.CSP, + InitSecret: []byte(tfOutput.Secret), + IP: tfOutput.IP, + UID: tfOutput.UID, + AttestationURL: tfOutput.AttestationURL, + } + + if err := fileHandler.RemoveAll(constants.TerraformWorkingDir); err != nil { + return fmt.Errorf("removing old terraform directory: %w", err) + } + + if err := fileHandler.CopyDir(filepath.Join(constants.UpgradeDir, constants.TerraformUpgradeWorkingDir), constants.TerraformWorkingDir); err != nil { + return fmt.Errorf("replacing old terraform directory with new one: %w", err) + } + + if err := fileHandler.RemoveAll(filepath.Join(constants.UpgradeDir, constants.TerraformUpgradeWorkingDir)); err != nil { + return fmt.Errorf("removing terraform upgrade directory: %w", err) + } + + if err := fileHandler.WriteJSON(opts.OutputFile, outputFileContents); err != nil { + return fmt.Errorf("writing terraform output to file: %w", err) + } + + return nil +} + +// a tfClient performs the Terraform interactions in an upgrade. +type tfClient interface { + PrepareUpgradeWorkspace(path, oldWorkingDir, newWorkingDir string, vars terraform.Variables) error + ShowPlan(ctx context.Context, logLevel terraform.LogLevel, planFilePath string, output io.Writer) error + Plan(ctx context.Context, logLevel terraform.LogLevel, planFile string, targets ...string) (bool, error) + CreateCluster(ctx context.Context, logLevel terraform.LogLevel, targets ...string) (terraform.CreateOutput, error) +} diff --git a/cli/internal/upgrade/terraform_test.go b/cli/internal/upgrade/terraform_test.go new file mode 100644 index 000000000..48aa414ea --- /dev/null +++ b/cli/internal/upgrade/terraform_test.go @@ -0,0 +1,313 @@ +/* +Copyright (c) Edgeless Systems GmbH + +SPDX-License-Identifier: AGPL-3.0-only +*/ + +package upgrade + +import ( + "bytes" + "context" + "io" + "path/filepath" + "testing" + + "github.com/edgelesssys/constellation/v2/cli/internal/terraform" + "github.com/edgelesssys/constellation/v2/internal/cloud/cloudprovider" + "github.com/edgelesssys/constellation/v2/internal/constants" + "github.com/edgelesssys/constellation/v2/internal/file" + "github.com/spf13/afero" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestCheckTerraformMigrations(t *testing.T) { + upgrader := func() *TerraformUpgrader { + u, err := NewTerraformUpgrader(&stubTerraformClient{}, bytes.NewBuffer(nil)) + require.NoError(t, err) + + return u + } + + workspace := func(existingFiles []string) file.Handler { + fs := afero.NewMemMapFs() + for _, f := range existingFiles { + require.NoError(t, afero.WriteFile(fs, f, []byte{}, 0o644)) + } + + return file.NewHandler(fs) + } + + testCases := map[string]struct { + workspace file.Handler + wantErr bool + }{ + "success": { + workspace: workspace(nil), + }, + "migration output file already exists": { + workspace: workspace([]string{constants.TerraformMigrationOutputFile}), + wantErr: true, + }, + "terraform backup dir already exists": { + workspace: workspace([]string{filepath.Join(constants.UpgradeDir, constants.TerraformUpgradeBackupDir)}), + wantErr: true, + }, + } + + for name, tc := range testCases { + t.Run(name, func(t *testing.T) { + u := upgrader() + err := u.CheckTerraformMigrations(tc.workspace) + if tc.wantErr { + require.Error(t, err) + return + } + + require.NoError(t, err) + }) + } +} + +func TestPlanTerraformMigrations(t *testing.T) { + upgrader := func(tf tfClient) *TerraformUpgrader { + u, err := NewTerraformUpgrader(tf, bytes.NewBuffer(nil)) + require.NoError(t, err) + + return u + } + + testCases := map[string]struct { + tf tfClient + want bool + wantErr bool + }{ + "success no diff": { + tf: &stubTerraformClient{}, + }, + "success diff": { + tf: &stubTerraformClient{ + hasDiff: true, + }, + want: true, + }, + "prepare workspace error": { + tf: &stubTerraformClient{ + prepareWorkspaceErr: assert.AnError, + }, + wantErr: true, + }, + "plan error": { + tf: &stubTerraformClient{ + planErr: assert.AnError, + }, + wantErr: true, + }, + "show plan error no diff": { + tf: &stubTerraformClient{ + showErr: assert.AnError, + }, + }, + "show plan error diff": { + tf: &stubTerraformClient{ + showErr: assert.AnError, + hasDiff: true, + }, + wantErr: true, + }, + } + + for name, tc := range testCases { + t.Run(name, func(t *testing.T) { + require := require.New(t) + + u := upgrader(tc.tf) + + opts := TerraformUpgradeOptions{ + LogLevel: terraform.LogLevelDebug, + CSP: cloudprovider.Unknown, + Vars: &terraform.QEMUVariables{}, + } + + diff, err := u.PlanTerraformMigrations(context.Background(), opts) + if tc.wantErr { + require.Error(err) + } else { + require.NoError(err) + require.Equal(tc.want, diff) + } + }) + } +} + +func TestApplyTerraformMigrations(t *testing.T) { + upgrader := func(tf tfClient) *TerraformUpgrader { + u, err := NewTerraformUpgrader(tf, bytes.NewBuffer(nil)) + require.NoError(t, err) + + return u + } + + fileHandler := func(existingFiles ...string) file.Handler { + fh := file.NewHandler(afero.NewMemMapFs()) + require.NoError(t, + fh.Write( + filepath.Join(constants.UpgradeDir, constants.TerraformUpgradeWorkingDir, "someFile"), + []byte("some content"), + )) + for _, f := range existingFiles { + require.NoError(t, fh.Write(f, []byte("some content"))) + } + return fh + } + + testCases := map[string]struct { + tf tfClient + fs file.Handler + outputFileName string + wantErr bool + }{ + "success": { + tf: &stubTerraformClient{}, + fs: fileHandler(), + outputFileName: "test.json", + }, + "create cluster error": { + tf: &stubTerraformClient{ + CreateClusterErr: assert.AnError, + }, + fs: fileHandler(), + outputFileName: "test.json", + wantErr: true, + }, + "empty file name": { + tf: &stubTerraformClient{}, + fs: fileHandler(), + outputFileName: "", + wantErr: true, + }, + "file already exists": { + tf: &stubTerraformClient{}, + fs: fileHandler("test.json"), + outputFileName: "test.json", + wantErr: true, + }, + } + + for name, tc := range testCases { + t.Run(name, func(t *testing.T) { + require := require.New(t) + + u := upgrader(tc.tf) + + opts := TerraformUpgradeOptions{ + LogLevel: terraform.LogLevelDebug, + CSP: cloudprovider.Unknown, + Vars: &terraform.QEMUVariables{}, + OutputFile: tc.outputFileName, + } + + err := u.ApplyTerraformMigrations(context.Background(), tc.fs, opts) + if tc.wantErr { + require.Error(err) + } else { + require.NoError(err) + } + }) + } +} + +func TestCleanUpTerraformMigrations(t *testing.T) { + upgrader := func() *TerraformUpgrader { + u, err := NewTerraformUpgrader(&stubTerraformClient{}, bytes.NewBuffer(nil)) + require.NoError(t, err) + + return u + } + + workspace := func(existingFiles []string) file.Handler { + fs := afero.NewMemMapFs() + for _, f := range existingFiles { + require.NoError(t, afero.WriteFile(fs, f, []byte{}, 0o644)) + } + + return file.NewHandler(fs) + } + + testCases := map[string]struct { + workspace file.Handler + wantFiles []string + wantErr bool + }{ + "no files": { + workspace: workspace(nil), + wantFiles: []string{}, + }, + "clean backup dir": { + workspace: workspace([]string{ + filepath.Join(constants.UpgradeDir, constants.TerraformUpgradeBackupDir), + }), + wantFiles: []string{}, + }, + "clean working dir": { + workspace: workspace([]string{ + filepath.Join(constants.UpgradeDir, constants.TerraformUpgradeWorkingDir), + }), + wantFiles: []string{}, + }, + "clean backup dir leave other files": { + workspace: workspace([]string{ + filepath.Join(constants.UpgradeDir, constants.TerraformUpgradeBackupDir), + filepath.Join(constants.UpgradeDir, "someFile"), + }), + wantFiles: []string{ + filepath.Join(constants.UpgradeDir, "someFile"), + }, + }, + } + + for name, tc := range testCases { + t.Run(name, func(t *testing.T) { + require := require.New(t) + + u := upgrader() + err := u.CleanUpTerraformMigrations(tc.workspace) + if tc.wantErr { + require.Error(err) + return + } + + require.NoError(err) + + for _, f := range tc.wantFiles { + _, err := tc.workspace.Stat(f) + require.NoError(err, "file %s should exist", f) + } + }) + } +} + +type stubTerraformClient struct { + hasDiff bool + prepareWorkspaceErr error + showErr error + planErr error + CreateClusterErr error +} + +func (u *stubTerraformClient) PrepareUpgradeWorkspace(string, string, string, terraform.Variables) error { + return u.prepareWorkspaceErr +} + +func (u *stubTerraformClient) ShowPlan(context.Context, terraform.LogLevel, string, io.Writer) error { + return u.showErr +} + +func (u *stubTerraformClient) Plan(context.Context, terraform.LogLevel, string, ...string) (bool, error) { + return u.hasDiff, u.planErr +} + +func (u *stubTerraformClient) CreateCluster(context.Context, terraform.LogLevel, ...string) (terraform.CreateOutput, error) { + return terraform.CreateOutput{}, u.CreateClusterErr +} diff --git a/cli/internal/upgrade/upgrade.go b/cli/internal/upgrade/upgrade.go new file mode 100644 index 000000000..1745684ad --- /dev/null +++ b/cli/internal/upgrade/upgrade.go @@ -0,0 +1,10 @@ +/* +Copyright (c) Edgeless Systems GmbH + +SPDX-License-Identifier: AGPL-3.0-only +*/ + +/* +Package upgrade provides functionality to upgrade the cluster and it's resources +*/ +package upgrade diff --git a/e2e/internal/upgrade/upgrade_test.go b/e2e/internal/upgrade/upgrade_test.go index 28998abf7..f79bf470e 100644 --- a/e2e/internal/upgrade/upgrade_test.go +++ b/e2e/internal/upgrade/upgrade_test.go @@ -52,7 +52,7 @@ var ( // setup checks that the prerequisites for the test are met: // - a workspace is set // - a CLI path is set -// - the constellation-upgrade folder does not exist. +// - the upgrade folder does not exist. func setup() error { workingDir, err := workingDir(*workspace) if err != nil { @@ -66,8 +66,8 @@ func setup() error { if _, err := getCLIPath(*cliPath); err != nil { return fmt.Errorf("getting CLI path: %w", err) } - if _, err := os.Stat("constellation-upgrade"); err == nil { - return errors.New("please remove the existing constellation-upgrade folder") + if _, err := os.Stat(constants.UpgradeDir); err == nil { + return fmt.Errorf("please remove the existing %s folder", constants.UpgradeDir) } return nil @@ -107,7 +107,16 @@ func TestUpgrade(t *testing.T) { log.Println(string(data)) log.Println("Triggering upgrade.") - cmd = exec.CommandContext(context.Background(), cli, "upgrade", "apply", "--force", "--debug", "-y") + + tfLogFlag := "" + cmd = exec.CommandContext(context.Background(), cli, "--help") + msg, err = cmd.CombinedOutput() + require.NoErrorf(err, "%s", string(msg)) + if strings.Contains(string(msg), "--tf-log") { + tfLogFlag = "--tf-log=DEBUG" + } + + cmd = exec.CommandContext(context.Background(), cli, "upgrade", "apply", "--force", "--debug", "--yes", tfLogFlag) msg, err = cmd.CombinedOutput() require.NoErrorf(err, "%s", string(msg)) require.NoError(containsUnexepectedMsg(string(msg))) diff --git a/internal/constants/constants.go b/internal/constants/constants.go index 4d41d5a2b..e4b0fb053 100644 --- a/internal/constants/constants.go +++ b/internal/constants/constants.go @@ -148,6 +148,16 @@ const ( MiniConstellationUID = "mini" // TerraformLogFile is the file name of the Terraform log file. TerraformLogFile = "terraform.log" + // TerraformUpgradePlanFile is the file name of the zipfile created by Terraform plan for Constellation upgrades. + TerraformUpgradePlanFile = "plan.zip" + // TerraformUpgradeWorkingDir is the directory name for the Terraform workspace being used in an upgrade. + TerraformUpgradeWorkingDir = "terraform" + // TerraformUpgradeBackupDir is the directory name being used to backup the pre-upgrade state in an upgrade. + TerraformUpgradeBackupDir = "terraform-backup" + // TerraformMigrationOutputFile is the file name of the output file created by a successful Terraform migration. + TerraformMigrationOutputFile = "terraform-migration-output.json" + // UpgradeDir is the name of the directory being used for cluster upgrades. + UpgradeDir = "constellation-upgrade" // // Kubernetes. diff --git a/internal/file/file.go b/internal/file/file.go index aad55a3e7..d11726e99 100644 --- a/internal/file/file.go +++ b/internal/file/file.go @@ -14,10 +14,13 @@ import ( "bytes" "encoding/json" "errors" + "fmt" "io" "io/fs" "os" "path" + "path/filepath" + "strings" "github.com/siderolabs/talos/pkg/machinery/config/encoder" "github.com/spf13/afero" @@ -175,3 +178,50 @@ func (h *Handler) Stat(name string) (fs.FileInfo, error) { func (h *Handler) MkdirAll(name string) error { return h.fs.MkdirAll(name, 0o700) } + +// CopyDir copies the src directory recursively into dst with the given options. OptMkdirAll +// is always set. CopyDir does not follow symlinks. +func (h *Handler) CopyDir(src, dst string, opts ...Option) error { + opts = append(opts, OptMkdirAll) + root := filepath.Join(src, string(filepath.Separator)) + + walkFunc := func(path string, info fs.FileInfo, err error) error { + if err != nil { + return err + } + + if info.IsDir() { + return nil + } + + pathWithoutRoot := strings.TrimPrefix(path, root) + return h.CopyFile(path, filepath.Join(dst, pathWithoutRoot), opts...) + } + + return h.fs.Walk(src, walkFunc) +} + +// CopyFile copies the file from src to dst with the given options, respecting file permissions. +func (h *Handler) CopyFile(src, dst string, opts ...Option) error { + srcInfo, err := h.fs.Stat(src) + if err != nil { + return fmt.Errorf("stat source file: %w", err) + } + + content, err := h.fs.ReadFile(src) + if err != nil { + return fmt.Errorf("read source file: %w", err) + } + + err = h.Write(dst, content, opts...) + if err != nil { + return fmt.Errorf("write destination file: %w", err) + } + + err = h.fs.Chmod(dst, srcInfo.Mode()) + if err != nil { + return fmt.Errorf("chmod destination file: %w", err) + } + + return nil +} diff --git a/internal/file/file_test.go b/internal/file/file_test.go index 614109071..e908c7fb9 100644 --- a/internal/file/file_test.go +++ b/internal/file/file_test.go @@ -8,6 +8,8 @@ package file import ( "encoding/json" + "io/fs" + "path/filepath" "testing" "github.com/edgelesssys/constellation/v2/internal/constants" @@ -350,3 +352,123 @@ func TestRemove(t *testing.T) { assert.Error(handler.Remove("d")) } + +func TestCopyFile(t *testing.T) { + perms := fs.FileMode(0o644) + + setupFs := func(existingFiles ...string) afero.Fs { + fs := afero.NewMemMapFs() + aferoHelper := afero.Afero{Fs: fs} + for _, file := range existingFiles { + require.NoError(t, aferoHelper.WriteFile(file, []byte{}, perms)) + } + return fs + } + + testCases := map[string]struct { + fs afero.Fs + copyFiles [][]string + checkFiles []string + opts []Option + wantErr bool + }{ + "successful copy": { + fs: setupFs("a"), + copyFiles: [][]string{{"a", "b"}}, + checkFiles: []string{"b"}, + }, + "copy to existing file overwrite": { + fs: setupFs("a", "b"), + copyFiles: [][]string{{"a", "b"}}, + checkFiles: []string{"b"}, + opts: []Option{OptOverwrite}, + }, + "copy to existing file no overwrite": { + fs: setupFs("a", "b"), + copyFiles: [][]string{{"a", "b"}}, + checkFiles: []string{"b"}, + wantErr: true, + }, + "file doesn't exist": { + fs: setupFs("a"), + copyFiles: [][]string{{"b", "c"}}, + checkFiles: []string{"a"}, + wantErr: true, + }, + } + + for name, tc := range testCases { + t.Run(name, func(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + + handler := NewHandler(tc.fs) + for _, files := range tc.copyFiles { + err := handler.CopyFile(files[0], files[1], tc.opts...) + if tc.wantErr { + assert.Error(err) + } else { + assert.NoError(err) + } + } + + for _, file := range tc.checkFiles { + info, err := handler.fs.Stat(file) + assert.Equal(perms, info.Mode()) + require.NoError(err) + } + }) + } +} + +func TestCopyDir(t *testing.T) { + setupHandler := func(existingFiles ...string) Handler { + fs := afero.NewMemMapFs() + handler := NewHandler(fs) + for _, file := range existingFiles { + err := handler.Write(file, []byte("some content"), OptMkdirAll) + require.NoError(t, err) + } + return handler + } + + testCases := map[string]struct { + handler Handler + copyFiles [][]string + checkFiles []string + opts []Option + }{ + "successful copy": { + handler: setupHandler(filepath.Join("someDir", "someFile"), filepath.Join("someDir", "someOtherDir", "someOtherFile")), + copyFiles: [][]string{{"someDir", "copiedDir"}}, + checkFiles: []string{filepath.Join("copiedDir", "someFile"), filepath.Join("copiedDir", "someOtherDir", "someOtherFile")}, + }, + "copy file": { + handler: setupHandler("someFile"), + copyFiles: [][]string{{"someFile", "copiedFile"}}, + checkFiles: []string{"copiedFile"}, + }, + "copy to existing dir overwrite": { + handler: setupHandler(filepath.Join("someDir", "someFile"), filepath.Join("someDir", "someOtherDir", "someOtherFile"), filepath.Join("copiedDir", "someExistingFile")), + copyFiles: [][]string{{"someDir", "copiedDir"}}, + checkFiles: []string{filepath.Join("copiedDir", "someFile"), filepath.Join("copiedDir", "someOtherDir", "someOtherFile"), filepath.Join("copiedDir", "someExistingFile")}, + opts: []Option{OptOverwrite}, + }, + } + + for name, tc := range testCases { + t.Run(name, func(t *testing.T) { + require := require.New(t) + + for _, files := range tc.copyFiles { + err := tc.handler.CopyDir(files[0], files[1], tc.opts...) + require.NoError(err) + } + + for _, file := range tc.checkFiles { + _, err := tc.handler.fs.Stat(file) + require.NoError(err) + } + }) + } +}