cli: Terraform migrations on upgrade (#1685)

* add terraform planning * overwrite terraform files in upgrade workspace * Revert "overwrite terraform files in upgrade workspace" This reverts commit 8bdacfb8bef23ef2cdbdb06bad0855b3bbc42df0. * prepare terraform workspace * test upgrade integration * print upgrade abort * rename plan file * write output to file * add show plan test * add upgrade tf workdir * fix workspace preparing * squash to 1 command * test * bazel build * plan test * register flag manually * bazel tidy * fix linter * remove MAA variable * fix workdir * accept tf variables * variable fetching * fix resource indices * accept Terraform targets * refactor upgrade command * Terraform migration apply unit test * pass down image fetcher to test * use new flags in e2e test * move file name to constant * update buildfiles * fix version constant * conditionally create MAA * move interface down * upgrade dir * update buildfiles * fix interface * fix createMAA check * fix imports * update buildfiles * wip: workspace backup * copy utils * backup upgrade workspace * remove debug print * replace old state after upgrade * check if flag exists * prepare test workspace * remove prefix Co-authored-by: Otto Bittner <cobittner@posteo.net> * respect file permissions * refactor tf upgrader * check workspace before upgrades * remove temp upgrade dir after completion * clean up workspace after abortion * fix upgrade apply test * fix linter --------- Co-authored-by: Otto Bittner <cobittner@posteo.net>
2025-05-02 22:34:56 -04:00 · 2023-05-22 13:31:20 +02:00 · 2023-05-22 13:31:20 +02:00 · c69e6777bd
commit c69e6777bd
parent 339e750c18
21 changed files with 1391 additions and 44 deletions
--- a/cli/internal/terraform/loader.go
+++ b/cli/internal/terraform/loader.go
@ -10,10 +10,12 @@ import (
 	"bytes"
 	"embed"
 	"errors"
+	"fmt"
 	"io/fs"
 	"path/filepath"
 	"strings"

+	"github.com/edgelesssys/constellation/v2/internal/constants"
 	"github.com/edgelesssys/constellation/v2/internal/file"
 	"github.com/spf13/afero"
 )
@ -27,8 +29,35 @@ var terraformFS embed.FS

 // prepareWorkspace loads the embedded Terraform files,
 // and writes them into the workspace.
-func prepareWorkspace(path string, fileHandler file.Handler, workingDir string) error {
-	rootDir := path
+func prepareWorkspace(rootDir string, fileHandler file.Handler, workingDir string) error {
+	return terraformCopier(fileHandler, rootDir, workingDir)
+}
+
+// prepareUpgradeWorkspace takes the Terraform state file from the old workspace and the
+// embedded Terraform files and writes them into the new workspace.
+func prepareUpgradeWorkspace(rootDir string, fileHandler file.Handler, oldWorkingDir, newWorkingDir string) error {
+	// backup old workspace
+	if err := fileHandler.CopyDir(
+		oldWorkingDir,
+		filepath.Join(constants.UpgradeDir, constants.TerraformUpgradeBackupDir),
+	); err != nil {
+		return fmt.Errorf("backing up old workspace: %w", err)
+	}
+
+	// copy state file
+	if err := fileHandler.CopyFile(
+		filepath.Join(oldWorkingDir, "terraform.tfstate"),
+		filepath.Join(newWorkingDir, "terraform.tfstate"),
+		file.OptMkdirAll,
+	); err != nil {
+		return fmt.Errorf("copying state file: %w", err)
+	}
+
+	return terraformCopier(fileHandler, rootDir, newWorkingDir)
+}
+
+// terraformCopier copies the embedded Terraform files into the workspace.
+func terraformCopier(fileHandler file.Handler, rootDir, workingDir string) error {
 	return fs.WalkDir(terraformFS, rootDir, func(path string, d fs.DirEntry, err error) error {
 		if err != nil {
 			return err
--- a/cli/internal/terraform/loader_test.go
+++ b/cli/internal/terraform/loader_test.go
@ -21,7 +21,7 @@ import (
 	"github.com/stretchr/testify/require"
 )

-func TestLoader(t *testing.T) {
+func TestPrepareWorkspace(t *testing.T) {
 	testCases := map[string]struct {
 		pathBase            string
 		provider            cloudprovider.Provider
@ -109,29 +109,114 @@ func TestLoader(t *testing.T) {
 			err := prepareWorkspace(path, file, constants.TerraformWorkingDir)

 			require.NoError(err)
-			checkFiles(t, file, func(err error) { assert.NoError(err) }, tc.fileList)
+			checkFiles(t, file, func(err error) { assert.NoError(err) }, constants.TerraformWorkingDir, tc.fileList)

 			if tc.testAlreadyUnpacked {
 				// Let's try the same again and check if we don't get a "file already exists" error.
 				require.NoError(file.Remove(filepath.Join(constants.TerraformWorkingDir, "variables.tf")))
 				err := prepareWorkspace(path, file, constants.TerraformWorkingDir)
 				assert.NoError(err)
-				checkFiles(t, file, func(err error) { assert.NoError(err) }, tc.fileList)
+				checkFiles(t, file, func(err error) { assert.NoError(err) }, constants.TerraformWorkingDir, tc.fileList)
 			}

 			err = cleanUpWorkspace(file, constants.TerraformWorkingDir)
 			require.NoError(err)

-			checkFiles(t, file, func(err error) { assert.ErrorIs(err, fs.ErrNotExist) }, tc.fileList)
+			checkFiles(t, file, func(err error) { assert.ErrorIs(err, fs.ErrNotExist) }, constants.TerraformWorkingDir, tc.fileList)
 		})
 	}
 }

-func checkFiles(t *testing.T, file file.Handler, assertion func(error), files []string) {
+func TestPrepareUpgradeWorkspace(t *testing.T) {
+	testCases := map[string]struct {
+		pathBase            string
+		provider            cloudprovider.Provider
+		oldWorkingDir       string
+		newWorkingDir       string
+		oldWorkspaceFiles   []string
+		newWorkspaceFiles   []string
+		expectedFiles       []string
+		testAlreadyUnpacked bool
+		wantErr             bool
+	}{
+		"works": {
+			pathBase:          "terraform",
+			provider:          cloudprovider.AWS,
+			oldWorkingDir:     "old",
+			newWorkingDir:     "new",
+			oldWorkspaceFiles: []string{"terraform.tfstate"},
+			expectedFiles: []string{
+				"main.tf",
+				"variables.tf",
+				"outputs.tf",
+				"modules",
+				"terraform.tfstate",
+			},
+		},
+		"state file does not exist": {
+			pathBase:          "terraform",
+			provider:          cloudprovider.AWS,
+			oldWorkingDir:     "old",
+			newWorkingDir:     "new",
+			oldWorkspaceFiles: []string{},
+			expectedFiles:     []string{},
+			wantErr:           true,
+		},
+		"terraform files already exist in new dir": {
+			pathBase:          "terraform",
+			provider:          cloudprovider.AWS,
+			oldWorkingDir:     "old",
+			newWorkingDir:     "new",
+			oldWorkspaceFiles: []string{"terraform.tfstate"},
+			newWorkspaceFiles: []string{"main.tf"},
+			wantErr:           true,
+		},
+	}
+
+	for name, tc := range testCases {
+		t.Run(name, func(t *testing.T) {
+			require := require.New(t)
+			assert := assert.New(t)
+
+			file := file.NewHandler(afero.NewMemMapFs())
+
+			path := path.Join(tc.pathBase, strings.ToLower(tc.provider.String()))
+
+			createFiles(t, file, tc.oldWorkspaceFiles, tc.oldWorkingDir)
+			createFiles(t, file, tc.newWorkspaceFiles, tc.newWorkingDir)
+
+			err := prepareUpgradeWorkspace(path, file, tc.oldWorkingDir, tc.newWorkingDir)
+
+			if tc.wantErr {
+				require.Error(err)
+			} else {
+				require.NoError(err)
+			}
+			checkFiles(t, file, func(err error) { assert.NoError(err) }, tc.newWorkingDir, tc.expectedFiles)
+			checkFiles(t, file, func(err error) { assert.NoError(err) },
+				filepath.Join(constants.UpgradeDir, constants.TerraformUpgradeBackupDir),
+				tc.oldWorkspaceFiles,
+			)
+		})
+	}
+}
+
+func checkFiles(t *testing.T, fileHandler file.Handler, assertion func(error), dir string, files []string) {
 	t.Helper()
 	for _, f := range files {
-		path := filepath.Join(constants.TerraformWorkingDir, f)
-		_, err := file.Stat(path)
+		path := filepath.Join(dir, f)
+		_, err := fileHandler.Stat(path)
 		assertion(err)
 	}
 }
+
+func createFiles(t *testing.T, fileHandler file.Handler, fileList []string, targetDir string) {
+	t.Helper()
+	require := require.New(t)
+
+	for _, f := range fileList {
+		path := filepath.Join(targetDir, f)
+		err := fileHandler.Write(path, []byte("1234"), file.OptOverwrite, file.OptMkdirAll)
+		require.NoError(err)
+	}
+}
--- a/cli/internal/terraform/terraform.go
+++ b/cli/internal/terraform/terraform.go
@ -18,6 +18,7 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"io"
 	"path/filepath"

 	"github.com/edgelesssys/constellation/v2/internal/cloud/cloudprovider"
@ -78,14 +79,24 @@ func (c *Client) Show(ctx context.Context) (*tfjson.State, error) {
 // PrepareWorkspace prepares a Terraform workspace for a Constellation cluster.
 func (c *Client) PrepareWorkspace(path string, vars Variables) error {
 	if err := prepareWorkspace(path, c.file, c.workingDir); err != nil {
-		return err
+		return fmt.Errorf("prepare workspace: %w", err)
+	}
+
+	return c.writeVars(vars)
+}
+
+// PrepareUpgradeWorkspace prepares a Terraform workspace for a Constellation version upgrade.
+// It copies the Terraform state from the old working dir and the embedded Terraform files into the new working dir.
+func (c *Client) PrepareUpgradeWorkspace(path, oldWorkingDir, newWorkingDir string, vars Variables) error {
+	if err := prepareUpgradeWorkspace(path, c.file, oldWorkingDir, newWorkingDir); err != nil {
+		return fmt.Errorf("prepare upgrade workspace: %w", err)
 	}

 	return c.writeVars(vars)
 }

 // CreateCluster creates a Constellation cluster using Terraform.
-func (c *Client) CreateCluster(ctx context.Context, logLevel LogLevel) (CreateOutput, error) {
+func (c *Client) CreateCluster(ctx context.Context, logLevel LogLevel, targets ...string) (CreateOutput, error) {
 	if err := c.setLogLevel(logLevel); err != nil {
 		return CreateOutput{}, fmt.Errorf("set terraform log level %s: %w", logLevel.String(), err)
 	}
@ -94,7 +105,12 @@ func (c *Client) CreateCluster(ctx context.Context, logLevel LogLevel) (CreateOu
 		return CreateOutput{}, fmt.Errorf("terraform init: %w", err)
 	}

-	if err := c.tf.Apply(ctx); err != nil {
+	opts := []tfexec.ApplyOption{}
+	for _, target := range targets {
+		opts = append(opts, tfexec.Target(target))
+	}
+
+	if err := c.tf.Apply(ctx, opts...); err != nil {
 		return CreateOutput{}, fmt.Errorf("terraform apply: %w", err)
 	}

@ -294,6 +310,45 @@ func (c *Client) CreateIAMConfig(ctx context.Context, provider cloudprovider.Pro
 	}
 }

+// Plan determines the diff that will be applied by Terraform. The plan output is written to the planFile.
+// If there is a diff, the returned bool is true. Otherwise, it is false.
+func (c *Client) Plan(ctx context.Context, logLevel LogLevel, planFile string, targets ...string) (bool, error) {
+	if err := c.setLogLevel(logLevel); err != nil {
+		return false, fmt.Errorf("set terraform log level %s: %w", logLevel.String(), err)
+	}
+
+	if err := c.tf.Init(ctx); err != nil {
+		return false, fmt.Errorf("terraform init: %w", err)
+	}
+
+	opts := []tfexec.PlanOption{
+		tfexec.Out(planFile),
+	}
+	for _, target := range targets {
+		opts = append(opts, tfexec.Target(target))
+	}
+	return c.tf.Plan(ctx, opts...)
+}
+
+// ShowPlan formats the diff in planFilePath and writes it to the specified output.
+func (c *Client) ShowPlan(ctx context.Context, logLevel LogLevel, planFilePath string, output io.Writer) error {
+	if err := c.setLogLevel(logLevel); err != nil {
+		return fmt.Errorf("set terraform log level %s: %w", logLevel.String(), err)
+	}
+
+	planResult, err := c.tf.ShowPlanFileRaw(ctx, planFilePath)
+	if err != nil {
+		return fmt.Errorf("terraform show plan: %w", err)
+	}
+
+	_, err = output.Write([]byte(planResult))
+	if err != nil {
+		return fmt.Errorf("write plan output: %w", err)
+	}
+
+	return nil
+}
+
 // Destroy destroys Terraform-created cloud resources.
 func (c *Client) Destroy(ctx context.Context, logLevel LogLevel) error {
 	if err := c.setLogLevel(logLevel); err != nil {
@ -386,6 +441,8 @@ type tfInterface interface {
 	Destroy(context.Context, ...tfexec.DestroyOption) error
 	Init(context.Context, ...tfexec.InitOption) error
 	Show(context.Context, ...tfexec.ShowOption) (*tfjson.State, error)
+	Plan(ctx context.Context, opts ...tfexec.PlanOption) (bool, error)
+	ShowPlanFileRaw(ctx context.Context, planPath string, opts ...tfexec.ShowOption) (string, error)
 	SetLog(level string) error
 	SetLogPath(path string) error
 }
--- a/cli/internal/terraform/terraform_test.go
+++ b/cli/internal/terraform/terraform_test.go
@ -7,6 +7,7 @@ SPDX-License-Identifier: AGPL-3.0-only
 package terraform

 import (
+	"bytes"
 	"context"
 	"errors"
 	"io/fs"
@ -934,14 +935,143 @@ func TestLogLevelString(t *testing.T) {
 	}
 }

+func TestPlan(t *testing.T) {
+	someError := errors.New("some error")
+
+	testCases := map[string]struct {
+		pathBase string
+		tf       *stubTerraform
+		fs       afero.Fs
+		wantErr  bool
+	}{
+		"plan succeeds": {
+			pathBase: "terraform",
+			tf:       &stubTerraform{},
+			fs:       afero.NewMemMapFs(),
+		},
+		"set log path fails": {
+			pathBase: "terraform",
+			tf: &stubTerraform{
+				setLogPathErr: someError,
+			},
+			fs:      afero.NewMemMapFs(),
+			wantErr: true,
+		},
+		"set log fails": {
+			pathBase: "terraform",
+			tf: &stubTerraform{
+				setLogErr: someError,
+			},
+			fs:      afero.NewMemMapFs(),
+			wantErr: true,
+		},
+		"plan fails": {
+			pathBase: "terraform",
+			tf: &stubTerraform{
+				planJSONErr: someError,
+			},
+			fs:      afero.NewMemMapFs(),
+			wantErr: true,
+		},
+		"init fails": {
+			pathBase: "terraform",
+			tf: &stubTerraform{
+				initErr: someError,
+			},
+			fs:      afero.NewMemMapFs(),
+			wantErr: true,
+		},
+	}
+
+	for name, tc := range testCases {
+		t.Run(name, func(t *testing.T) {
+			require := require.New(t)
+
+			c := &Client{
+				file:       file.NewHandler(tc.fs),
+				tf:         tc.tf,
+				workingDir: tc.pathBase,
+			}
+
+			_, err := c.Plan(context.Background(), LogLevelDebug, constants.TerraformUpgradePlanFile)
+			if tc.wantErr {
+				require.Error(err)
+			} else {
+				require.NoError(err)
+			}
+		})
+	}
+}
+
+func TestShowPlan(t *testing.T) {
+	someError := errors.New("some error")
+	testCases := map[string]struct {
+		pathBase string
+		tf       *stubTerraform
+		fs       afero.Fs
+		wantErr  bool
+	}{
+		"show plan succeeds": {
+			pathBase: "terraform",
+			tf:       &stubTerraform{},
+			fs:       afero.NewMemMapFs(),
+		},
+		"set log path fails": {
+			pathBase: "terraform",
+			tf: &stubTerraform{
+				setLogPathErr: someError,
+			},
+			fs:      afero.NewMemMapFs(),
+			wantErr: true,
+		},
+		"set log fails": {
+			pathBase: "terraform",
+			tf: &stubTerraform{
+				setLogErr: someError,
+			},
+			fs:      afero.NewMemMapFs(),
+			wantErr: true,
+		},
+		"show plan file fails": {
+			pathBase: "terraform",
+			tf: &stubTerraform{
+				showPlanFileErr: someError,
+			},
+			fs:      afero.NewMemMapFs(),
+			wantErr: true,
+		},
+	}
+
+	for name, tc := range testCases {
+		t.Run(name, func(t *testing.T) {
+			require := require.New(t)
+
+			c := &Client{
+				file:       file.NewHandler(tc.fs),
+				tf:         tc.tf,
+				workingDir: tc.pathBase,
+			}
+
+			err := c.ShowPlan(context.Background(), LogLevelDebug, "", bytes.NewBuffer(nil))
+			if tc.wantErr {
+				require.Error(err)
+			} else {
+				require.NoError(err)
+			}
+		})
+	}
+}
+
 type stubTerraform struct {
-	applyErr      error
-	destroyErr    error
-	initErr       error
-	showErr       error
-	setLogErr     error
-	setLogPathErr error
-	showState     *tfjson.State
+	applyErr        error
+	destroyErr      error
+	initErr         error
+	showErr         error
+	setLogErr       error
+	setLogPathErr   error
+	planJSONErr     error
+	showPlanFileErr error
+	showState       *tfjson.State
 }

 func (s *stubTerraform) Apply(context.Context, ...tfexec.ApplyOption) error {
@ -960,6 +1090,14 @@ func (s *stubTerraform) Show(context.Context, ...tfexec.ShowOption) (*tfjson.Sta
 	return s.showState, s.showErr
 }

+func (s *stubTerraform) Plan(context.Context, ...tfexec.PlanOption) (bool, error) {
+	return false, s.planJSONErr
+}
+
+func (s *stubTerraform) ShowPlanFileRaw(context.Context, string, ...tfexec.ShowOption) (string, error) {
+	return "", s.showPlanFileErr
+}
+
 func (s *stubTerraform) SetLog(_ string) error {
 	return s.setLogErr
 }