Git-Mirrors/constellation
1
0
Fork 0
mirror of https://github.com/edgelesssys/constellation.git synced 2025-01-22 21:31:14 -05:00
Code Issues Packages Projects Releases Wiki Activity

cli: Terraform migrations on upgrade (#1685)

Browse Source 
* add terraform planning

* overwrite terraform files in upgrade workspace

* Revert "overwrite terraform files in upgrade workspace"

This reverts commit 8bdacfb8bef23ef2cdbdb06bad0855b3bbc42df0.

* prepare terraform workspace

* test upgrade integration

* print upgrade abort

* rename plan file

* write output to file

* add show plan test

* add upgrade tf workdir

* fix workspace preparing

* squash to 1 command

* test

* bazel build

* plan test

* register flag manually

* bazel tidy

* fix linter

* remove MAA variable

* fix workdir

* accept tf variables

* variable fetching

* fix resource indices

* accept Terraform targets

* refactor upgrade command

* Terraform migration apply unit test

* pass down image fetcher to test

* use new flags in e2e test

* move file name to constant

* update buildfiles

* fix version constant

* conditionally create MAA

* move interface down

* upgrade dir

* update buildfiles

* fix interface

* fix createMAA check

* fix imports

* update buildfiles

* wip: workspace backup

* copy utils

* backup upgrade workspace

* remove debug print

* replace old state after upgrade

* check if flag exists

* prepare test workspace

* remove prefix

Co-authored-by: Otto Bittner <cobittner@posteo.net>

* respect file permissions

* refactor tf upgrader

* check workspace before upgrades

* remove temp upgrade dir after completion

* clean up workspace after abortion

* fix upgrade apply test

* fix linter

---------

Co-authored-by: Otto Bittner <cobittner@posteo.net>
This commit is contained in:
Moritz Sanft 2023-05-22 13:31:20 +02:00 committed by GitHub
parent 339e750c18
commit c69e6777bd
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
21 changed files with 1391 additions and 44 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
cli/internal/cloudcmd/clients.go
View File

@ -23,7 +23,7 @@ type imageFetcher interface {
type terraformClient interface {
PrepareWorkspace(path string, input terraform.Variables) error
CreateCluster(ctx context.Context, logLevel terraform.LogLevel) (terraform.CreateOutput, error)
CreateCluster(ctx context.Context, logLevel terraform.LogLevel, targets ...string) (terraform.CreateOutput, error)
CreateIAMConfig(ctx context.Context, provider cloudprovider.Provider, logLevel terraform.LogLevel) (terraform.IAMOutput, error)
Destroy(ctx context.Context, logLevel terraform.LogLevel) error
CleanUpWorkspace() error

2
cli/internal/cloudcmd/clients_test.go
View File

@ -45,7 +45,7 @@ type stubTerraformClient struct {
showErr error
}
func (c *stubTerraformClient) CreateCluster(_ context.Context, _ terraform.LogLevel) (terraform.CreateOutput, error) {
func (c *stubTerraformClient) CreateCluster(_ context.Context, _ terraform.LogLevel, _ ...string) (terraform.CreateOutput, error) {
return terraform.CreateOutput{
IP: c.ip,
Secret: c.initSecret,

3
cli/internal/cmd/BUILD.bazel
View File

@ -41,9 +41,11 @@ go_library(
"//cli/internal/clusterid",
"//cli/internal/helm",
"//cli/internal/iamid",
"//cli/internal/image",
"//cli/internal/kubernetes",
"//cli/internal/libvirt",
"//cli/internal/terraform",
"//cli/internal/upgrade",
"//disk-mapper/recoverproto",
"//internal/atls",
"//internal/attestation/measurements",
@ -122,6 +124,7 @@ go_test(
"//cli/internal/iamid",
"//cli/internal/kubernetes",
"//cli/internal/terraform",
"//cli/internal/upgrade",
"//disk-mapper/recoverproto",
"//internal/atls",
"//internal/attestation/measurements",

181
cli/internal/cmd/upgradeapply.go
View File

@ -10,11 +10,16 @@ import (
"context"
"errors"
"fmt"
"path/filepath"
"strings"
"time"
"github.com/edgelesssys/constellation/v2/cli/internal/clusterid"
"github.com/edgelesssys/constellation/v2/cli/internal/helm"
"github.com/edgelesssys/constellation/v2/cli/internal/image"
"github.com/edgelesssys/constellation/v2/cli/internal/kubernetes"
"github.com/edgelesssys/constellation/v2/cli/internal/terraform"
"github.com/edgelesssys/constellation/v2/cli/internal/upgrade"
"github.com/edgelesssys/constellation/v2/internal/cloud/cloudprovider"
"github.com/edgelesssys/constellation/v2/internal/compatibility"
"github.com/edgelesssys/constellation/v2/internal/config"
@ -55,17 +60,20 @@ func runUpgradeApply(cmd *cobra.Command, _ []string) error {
defer log.Sync()
fileHandler := file.NewHandler(afero.NewOsFs())
upgrader, err := kubernetes.NewUpgrader(cmd.OutOrStdout(), log)
upgrader, err := kubernetes.NewUpgrader(cmd.Context(), cmd.OutOrStdout(), log)
if err != nil {
return err
}
applyCmd := upgradeApplyCmd{upgrader: upgrader, log: log}
fetcher := image.New()
applyCmd := upgradeApplyCmd{upgrader: upgrader, log: log, fetcher: fetcher}
return applyCmd.upgradeApply(cmd, fileHandler)
}
type upgradeApplyCmd struct {
upgrader cloudUpgrader
fetcher imageFetcher
log debugLog
}
@ -94,6 +102,10 @@ func (u *upgradeApplyCmd) upgradeApply(cmd *cobra.Command, fileHandler file.Hand
return fmt.Errorf("upgrading measurements: %w", err)
}
if err := u.migrateTerraform(cmd, fileHandler, u.fetcher, conf, flags); err != nil {
return fmt.Errorf("performing Terraform migrations: %w", err)
}
if conf.GetProvider() == cloudprovider.Azure || conf.GetProvider() == cloudprovider.GCP || conf.GetProvider() == cloudprovider.AWS {
err = u.handleServiceUpgrade(cmd, conf, flags)
upgradeErr := &compatibility.InvalidUpgradeError{}
@ -120,6 +132,141 @@ func (u *upgradeApplyCmd) upgradeApply(cmd *cobra.Command, fileHandler file.Hand
return nil
}
// migrateTerraform checks if the Constellation version the cluster is being upgraded to requires a migration
// of cloud resources with Terraform. If so, the migration is performed.
func (u *upgradeApplyCmd) migrateTerraform(cmd *cobra.Command, file file.Handler, fetcher imageFetcher, conf *config.Config, flags upgradeApplyFlags) error {
u.log.Debugf("Planning Terraform migrations")
if err := u.upgrader.CheckTerraformMigrations(file); err != nil {
return fmt.Errorf("checking workspace: %w", err)
}
targets, vars, err := u.parseUpgradeVars(cmd, conf, fetcher)
if err != nil {
return fmt.Errorf("parsing upgrade variables: %w", err)
}
u.log.Debugf("Using migration targets:\n%v", targets)
u.log.Debugf("Using Terraform variables:\n%v", vars)
opts := upgrade.TerraformUpgradeOptions{
LogLevel: flags.terraformLogLevel,
CSP: conf.GetProvider(),
Vars: vars,
Targets: targets,
OutputFile: constants.TerraformMigrationOutputFile,
}
// Check if there are any Terraform migrations to apply
hasDiff, err := u.upgrader.PlanTerraformMigrations(cmd.Context(), opts)
if err != nil {
return fmt.Errorf("planning terraform migrations: %w", err)
}
if hasDiff {
// If there are any Terraform migrations to apply, ask for confirmation
if !flags.yes {
ok, err := askToConfirm(cmd, "Do you want to apply the Terraform migrations?")
if err != nil {
return fmt.Errorf("asking for confirmation: %w", err)
}
if !ok {
cmd.Println("Aborting upgrade.")
if err := u.upgrader.CleanUpTerraformMigrations(file); err != nil {
return fmt.Errorf("cleaning up workspace: %w", err)
}
return fmt.Errorf("aborted by user")
}
}
u.log.Debugf("Applying Terraform migrations")
err := u.upgrader.ApplyTerraformMigrations(cmd.Context(), file, opts)
if err != nil {
return fmt.Errorf("applying terraform migrations: %w", err)
}
cmd.Printf("Terraform migrations applied successfully and output written to: %s\n"+
"A backup of the pre-upgrade Terraform state has been written to: %s\n",
opts.OutputFile, filepath.Join(constants.UpgradeDir, constants.TerraformUpgradeBackupDir))
} else {
u.log.Debugf("No Terraform diff detected")
}
return nil
}
func (u *upgradeApplyCmd) parseUpgradeVars(cmd *cobra.Command, conf *config.Config, fetcher imageFetcher) ([]string, terraform.Variables, error) {
// Fetch variables to execute Terraform script with
imageRef, err := fetcher.FetchReference(cmd.Context(), conf)
if err != nil {
return nil, nil, fmt.Errorf("fetching image reference: %w", err)
}
commonVariables := terraform.CommonVariables{
Name: conf.Name,
StateDiskSizeGB: conf.StateDiskSizeGB,
// Ignore node count as their values are only being respected for creation
// See here: https://developer.hashicorp.com/terraform/language/meta-arguments/lifecycle#ignore_changes
}
switch conf.GetProvider() {
case cloudprovider.AWS:
targets := []string{}
vars := &terraform.AWSClusterVariables{
CommonVariables: commonVariables,
StateDiskType: conf.Provider.AWS.StateDiskType,
Region: conf.Provider.AWS.Region,
Zone: conf.Provider.AWS.Zone,
InstanceType: conf.Provider.AWS.InstanceType,
AMIImageID: imageRef,
IAMProfileControlPlane: conf.Provider.AWS.IAMProfileControlPlane,
IAMProfileWorkerNodes: conf.Provider.AWS.IAMProfileWorkerNodes,
Debug: conf.IsDebugCluster(),
}
return targets, vars, nil
case cloudprovider.Azure:
targets := []string{"azurerm_attestation_provider.attestation_provider"}
// Azure Terraform provider is very strict about it's casing
imageRef = strings.Replace(imageRef, "CommunityGalleries", "communityGalleries", 1)
imageRef = strings.Replace(imageRef, "Images", "images", 1)
imageRef = strings.Replace(imageRef, "Versions", "versions", 1)
vars := &terraform.AzureClusterVariables{
CommonVariables: commonVariables,
Location: conf.Provider.Azure.Location,
ResourceGroup: conf.Provider.Azure.ResourceGroup,
UserAssignedIdentity: conf.Provider.Azure.UserAssignedIdentity,
InstanceType: conf.Provider.Azure.InstanceType,
StateDiskType: conf.Provider.Azure.StateDiskType,
ImageID: imageRef,
SecureBoot: *conf.Provider.Azure.SecureBoot,
CreateMAA: conf.GetAttestationConfig().GetVariant().Equal(variant.AzureSEVSNP{}),
Debug: conf.IsDebugCluster(),
}
return targets, vars, nil
case cloudprovider.GCP:
targets := []string{}
vars := &terraform.GCPClusterVariables{
CommonVariables: commonVariables,
Project: conf.Provider.GCP.Project,
Region: conf.Provider.GCP.Region,
Zone: conf.Provider.GCP.Zone,
CredentialsFile: conf.Provider.GCP.ServiceAccountKeyPath,
InstanceType: conf.Provider.GCP.InstanceType,
StateDiskType: conf.Provider.GCP.StateDiskType,
ImageID: imageRef,
Debug: conf.IsDebugCluster(),
}
return targets, vars, nil
default:
return nil, nil, fmt.Errorf("unsupported provider: %s", conf.GetProvider())
}
}
type imageFetcher interface {
FetchReference(ctx context.Context, conf *config.Config) (string, error)
}
// upgradeAttestConfigIfDiff checks if the locally configured measurements are different from the cluster's measurements.
// If so the function will ask the user to confirm (if --yes is not set) and upgrade the measurements only.
func (u *upgradeApplyCmd) upgradeAttestConfigIfDiff(cmd *cobra.Command, newConfig config.AttestationCfg, flags upgradeApplyFlags) error {
@ -193,14 +340,30 @@ func parseUpgradeApplyFlags(cmd *cobra.Command) (upgradeApplyFlags, error) {
return upgradeApplyFlags{}, fmt.Errorf("parsing force argument: %w", err)
}
return upgradeApplyFlags{configPath: configPath, yes: yes, upgradeTimeout: timeout, force: force}, nil
logLevelString, err := cmd.Flags().GetString("tf-log")
if err != nil {
return upgradeApplyFlags{}, fmt.Errorf("parsing tf-log string: %w", err)
}
logLevel, err := terraform.ParseLogLevel(logLevelString)
if err != nil {
return upgradeApplyFlags{}, fmt.Errorf("parsing Terraform log level %s: %w", logLevelString, err)
}
return upgradeApplyFlags{
configPath: configPath,
yes: yes,
upgradeTimeout: timeout,
force: force,
terraformLogLevel: logLevel,
}, nil
}
type upgradeApplyFlags struct {
configPath string
yes bool
upgradeTimeout time.Duration
force bool
configPath string
yes bool
upgradeTimeout time.Duration
force bool
terraformLogLevel terraform.LogLevel
}
type cloudUpgrader interface {
@ -208,4 +371,8 @@ type cloudUpgrader interface {
UpgradeHelmServices(ctx context.Context, config *config.Config, timeout time.Duration, allowDestructive bool) error
UpdateAttestationConfig(ctx context.Context, newConfig config.AttestationCfg) error
GetClusterAttestationConfig(ctx context.Context, variant variant.Variant) (config.AttestationCfg, *corev1.ConfigMap, error)
PlanTerraformMigrations(ctx context.Context, opts upgrade.TerraformUpgradeOptions) (bool, error)
ApplyTerraformMigrations(ctx context.Context, fileHandler file.Handler, opts upgrade.TerraformUpgradeOptions) error
CheckTerraformMigrations(fileHandler file.Handler) error
CleanUpTerraformMigrations(fileHandler file.Handler) error
}

112
cli/internal/cmd/upgradeapply_test.go
View File

@ -7,6 +7,7 @@ SPDX-License-Identifier: AGPL-3.0-only
package cmd
import (
"bytes"
"context"
"errors"
"testing"
@ -14,6 +15,7 @@ import (
"github.com/edgelesssys/constellation/v2/cli/internal/clusterid"
"github.com/edgelesssys/constellation/v2/cli/internal/kubernetes"
"github.com/edgelesssys/constellation/v2/cli/internal/upgrade"
"github.com/edgelesssys/constellation/v2/internal/cloud/cloudprovider"
"github.com/edgelesssys/constellation/v2/internal/config"
"github.com/edgelesssys/constellation/v2/internal/constants"
@ -30,10 +32,14 @@ func TestUpgradeApply(t *testing.T) {
someErr := errors.New("some error")
testCases := map[string]struct {
upgrader stubUpgrader
fetcher stubImageFetcher
wantErr bool
yesFlag bool
stdin string
}{
"success": {
upgrader: stubUpgrader{currentConfig: config.DefaultForAzureSEVSNP()},
yesFlag: true,
},
"nodeVersion some error": {
upgrader: stubUpgrader{
@ -41,12 +47,14 @@ func TestUpgradeApply(t *testing.T) {
nodeVersionErr: someErr,
},
wantErr: true,
yesFlag: true,
},
"nodeVersion in progress error": {
upgrader: stubUpgrader{
currentConfig: config.DefaultForAzureSEVSNP(),
nodeVersionErr: kubernetes.ErrInProgress,
},
yesFlag: true,
},
"helm other error": {
upgrader: stubUpgrader{
@ -54,6 +62,63 @@ func TestUpgradeApply(t *testing.T) {
helmErr: someErr,
},
wantErr: true,
fetcher: stubImageFetcher{},
yesFlag: true,
},
"check terraform error": {
upgrader: stubUpgrader{
currentConfig: config.DefaultForAzureSEVSNP(),
checkTerraformErr: someErr,
},
fetcher: stubImageFetcher{},
wantErr: true,
yesFlag: true,
},
"abort": {
upgrader: stubUpgrader{
currentConfig: config.DefaultForAzureSEVSNP(),
terraformDiff: true,
},
fetcher: stubImageFetcher{},
wantErr: true,
stdin: "no\n",
},
"clean terraform error": {
upgrader: stubUpgrader{
currentConfig: config.DefaultForAzureSEVSNP(),
cleanTerraformErr: someErr,
terraformDiff: true,
},
fetcher: stubImageFetcher{},
wantErr: true,
stdin: "no\n",
},
"plan terraform error": {
upgrader: stubUpgrader{
currentConfig: config.DefaultForAzureSEVSNP(),
planTerraformErr: someErr,
},
fetcher: stubImageFetcher{},
wantErr: true,
yesFlag: true,
},
"apply terraform error": {
upgrader: stubUpgrader{
currentConfig: config.DefaultForAzureSEVSNP(),
applyTerraformErr: someErr,
terraformDiff: true,
},
fetcher: stubImageFetcher{},
wantErr: true,
yesFlag: true,
},
"fetch reference error": {
upgrader: stubUpgrader{
currentConfig: config.DefaultForAzureSEVSNP(),
},
fetcher: stubImageFetcher{fetchReferenceErr: someErr},
wantErr: true,
yesFlag: true,
},
}
@ -62,19 +127,23 @@ func TestUpgradeApply(t *testing.T) {
assert := assert.New(t)
require := require.New(t)
cmd := newUpgradeApplyCmd()
cmd.SetIn(bytes.NewBufferString(tc.stdin))
cmd.Flags().String("config", constants.ConfigFilename, "") // register persistent flag manually
cmd.Flags().Bool("force", true, "") // register persistent flag manually
cmd.Flags().String("tf-log", "DEBUG", "") // register persistent flag manually
err := cmd.Flags().Set("yes", "true")
require.NoError(err)
if tc.yesFlag {
err := cmd.Flags().Set("yes", "true")
require.NoError(err)
}
handler := file.NewHandler(afero.NewMemMapFs())
cfg := defaultConfigWithExpectedMeasurements(t, config.Default(), cloudprovider.Azure)
require.NoError(handler.WriteYAML(constants.ConfigFilename, cfg))
require.NoError(handler.WriteJSON(constants.ClusterIDsFileName, clusterid.File{}))
upgrader := upgradeApplyCmd{upgrader: tc.upgrader, log: logger.NewTest(t)}
err = upgrader.upgradeApply(cmd, handler)
upgrader := upgradeApplyCmd{upgrader: tc.upgrader, log: logger.NewTest(t), fetcher: tc.fetcher}
err := upgrader.upgradeApply(cmd, handler)
if tc.wantErr {
assert.Error(err)
} else {
@ -85,9 +154,14 @@ func TestUpgradeApply(t *testing.T) {
}
type stubUpgrader struct {
currentConfig config.AttestationCfg
nodeVersionErr error
helmErr error
currentConfig config.AttestationCfg
nodeVersionErr error
helmErr error
terraformDiff bool
planTerraformErr error
checkTerraformErr error
applyTerraformErr error
cleanTerraformErr error
}
func (u stubUpgrader) UpgradeNodeVersion(_ context.Context, _ *config.Config) error {
@ -105,3 +179,27 @@ func (u stubUpgrader) UpdateAttestationConfig(_ context.Context, _ config.Attest
func (u stubUpgrader) GetClusterAttestationConfig(_ context.Context, _ variant.Variant) (config.AttestationCfg, *corev1.ConfigMap, error) {
return u.currentConfig, &corev1.ConfigMap{}, nil
}
func (u stubUpgrader) CheckTerraformMigrations(file.Handler) error {
return u.checkTerraformErr
}
func (u stubUpgrader) CleanUpTerraformMigrations(file.Handler) error {
return u.cleanTerraformErr
}
func (u stubUpgrader) PlanTerraformMigrations(context.Context, upgrade.TerraformUpgradeOptions) (bool, error) {
return u.terraformDiff, u.planTerraformErr
}
func (u stubUpgrader) ApplyTerraformMigrations(context.Context, file.Handler, upgrade.TerraformUpgradeOptions) error {
return u.applyTerraformErr
}
type stubImageFetcher struct {
fetchReferenceErr error
}
func (s stubImageFetcher) FetchReference(context.Context, *config.Config) (string, error) {
return "", s.fetchReferenceErr
}

2
cli/internal/cmd/upgradecheck.go
View File

@ -62,7 +62,7 @@ func runUpgradeCheck(cmd *cobra.Command, _ []string) error {
if err != nil {
return err
}
checker, err := kubernetes.NewUpgrader(cmd.OutOrStdout(), log)
checker, err := kubernetes.NewUpgrader(cmd.Context(), cmd.OutOrStdout(), log)
if err != nil {
return err
}

7
cli/internal/helm/backup.go
View File

@ -11,14 +11,15 @@ import (
"fmt"
"path/filepath"
"github.com/edgelesssys/constellation/v2/internal/constants"
apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
"k8s.io/apimachinery/pkg/runtime/schema"
"sigs.k8s.io/yaml"
)
const (
crdBackupFolder = "constellation-upgrade/backups/crds/"
backupFolder = "constellation-upgrade/backups/"
var (
backupFolder = filepath.Join(constants.UpgradeDir, "backups") + string(filepath.Separator)
crdBackupFolder = filepath.Join(backupFolder, "crds") + string(filepath.Separator)
)
func (c *Client) backupCRDs(ctx context.Context) ([]apiextensionsv1.CustomResourceDefinition, error) {

3
cli/internal/kubernetes/BUILD.bazel
View File

@ -13,10 +13,13 @@ go_library(
deps = [
"//cli/internal/helm",
"//cli/internal/image",
"//cli/internal/terraform",
"//cli/internal/upgrade",
"//internal/attestation/measurements",
"//internal/compatibility",
"//internal/config",
"//internal/constants",
"//internal/file",
"//internal/kubernetes",
"//internal/kubernetes/kubectl",
"//internal/variant",

45
cli/internal/kubernetes/upgrade.go
View File

@ -12,14 +12,18 @@ import (
"errors"
"fmt"
"io"
"path/filepath"
"time"
"github.com/edgelesssys/constellation/v2/cli/internal/helm"
"github.com/edgelesssys/constellation/v2/cli/internal/image"
"github.com/edgelesssys/constellation/v2/cli/internal/terraform"
"github.com/edgelesssys/constellation/v2/cli/internal/upgrade"
"github.com/edgelesssys/constellation/v2/internal/attestation/measurements"
"github.com/edgelesssys/constellation/v2/internal/compatibility"
"github.com/edgelesssys/constellation/v2/internal/config"
"github.com/edgelesssys/constellation/v2/internal/constants"
"github.com/edgelesssys/constellation/v2/internal/file"
internalk8s "github.com/edgelesssys/constellation/v2/internal/kubernetes"
"github.com/edgelesssys/constellation/v2/internal/kubernetes/kubectl"
"github.com/edgelesssys/constellation/v2/internal/variant"
@ -77,11 +81,12 @@ type Upgrader struct {
helmClient helmInterface
imageFetcher imageFetcher
outWriter io.Writer
tfUpgrader *upgrade.TerraformUpgrader
log debugLog
}
// NewUpgrader returns a new Upgrader.
func NewUpgrader(outWriter io.Writer, log debugLog) (*Upgrader, error) {
func NewUpgrader(ctx context.Context, outWriter io.Writer, log debugLog) (*Upgrader, error) {
kubeConfig, err := clientcmd.BuildConfigFromFlags("", constants.AdminConfFilename)
if err != nil {
return nil, fmt.Errorf("building kubernetes config: %w", err)
@ -103,16 +108,54 @@ func NewUpgrader(outWriter io.Writer, log debugLog) (*Upgrader, error) {
return nil, fmt.Errorf("setting up helm client: %w", err)
}
tfClient, err := terraform.New(ctx, filepath.Join(constants.UpgradeDir, constants.TerraformUpgradeWorkingDir))
if err != nil {
return nil, fmt.Errorf("setting up terraform client: %w", err)
}
tfUpgrader, err := upgrade.NewTerraformUpgrader(tfClient, outWriter)
if err != nil {
return nil, fmt.Errorf("setting up terraform upgrader: %w", err)
}
return &Upgrader{
stableInterface: &stableClient{client: kubeClient},
dynamicInterface: &NodeVersionClient{client: unstructuredClient},
helmClient: helmClient,
imageFetcher: image.New(),
outWriter: outWriter,
tfUpgrader: tfUpgrader,
log: log,
}, nil
}
// CheckTerraformMigrations checks whether Terraform migrations are possible in the current workspace.
// If the files that will be written during the upgrade already exist, it returns an error.
func (u *Upgrader) CheckTerraformMigrations(fileHandler file.Handler) error {
return u.tfUpgrader.CheckTerraformMigrations(fileHandler)
}
// CleanUpTerraformMigrations cleans up the Terraform migration workspace, for example when an upgrade is
// aborted by the user.
func (u *Upgrader) CleanUpTerraformMigrations(fileHandler file.Handler) error {
return u.tfUpgrader.CleanUpTerraformMigrations(fileHandler)
}
// PlanTerraformMigrations prepares the upgrade workspace and plans the Terraform migrations for the Constellation upgrade.
// If a diff exists, it's being written to the upgrader's output writer. It also returns
// a bool indicating whether a diff exists.
func (u *Upgrader) PlanTerraformMigrations(ctx context.Context, opts upgrade.TerraformUpgradeOptions) (bool, error) {
return u.tfUpgrader.PlanTerraformMigrations(ctx, opts)
}
// ApplyTerraformMigrations applies the migerations planned by PlanTerraformMigrations.
// If PlanTerraformMigrations has not been executed before, it will return an error.
// In case of a successful upgrade, the output will be written to the specified file and the old Terraform directory is replaced
// By the new one.
func (u *Upgrader) ApplyTerraformMigrations(ctx context.Context, fileHandler file.Handler, opts upgrade.TerraformUpgradeOptions) error {
return u.tfUpgrader.ApplyTerraformMigrations(ctx, fileHandler, opts)
}
// UpgradeHelmServices upgrade helm services.
func (u *Upgrader) UpgradeHelmServices(ctx context.Context, config *config.Config, timeout time.Duration, allowDestructive bool) error {
return u.helmClient.Upgrade(ctx, config, timeout, allowDestructive)

33
cli/internal/terraform/loader.go
View File

@ -10,10 +10,12 @@ import (
"bytes"
"embed"
"errors"
"fmt"
"io/fs"
"path/filepath"
"strings"
"github.com/edgelesssys/constellation/v2/internal/constants"
"github.com/edgelesssys/constellation/v2/internal/file"
"github.com/spf13/afero"
)
@ -27,8 +29,35 @@ var terraformFS embed.FS
// prepareWorkspace loads the embedded Terraform files,
// and writes them into the workspace.
func prepareWorkspace(path string, fileHandler file.Handler, workingDir string) error {
rootDir := path
func prepareWorkspace(rootDir string, fileHandler file.Handler, workingDir string) error {
return terraformCopier(fileHandler, rootDir, workingDir)
}
// prepareUpgradeWorkspace takes the Terraform state file from the old workspace and the
// embedded Terraform files and writes them into the new workspace.
func prepareUpgradeWorkspace(rootDir string, fileHandler file.Handler, oldWorkingDir, newWorkingDir string) error {
// backup old workspace
if err := fileHandler.CopyDir(
oldWorkingDir,
filepath.Join(constants.UpgradeDir, constants.TerraformUpgradeBackupDir),
); err != nil {
return fmt.Errorf("backing up old workspace: %w", err)
}
// copy state file
if err := fileHandler.CopyFile(
filepath.Join(oldWorkingDir, "terraform.tfstate"),
filepath.Join(newWorkingDir, "terraform.tfstate"),
file.OptMkdirAll,
); err != nil {
return fmt.Errorf("copying state file: %w", err)
}
return terraformCopier(fileHandler, rootDir, newWorkingDir)
}
// terraformCopier copies the embedded Terraform files into the workspace.
func terraformCopier(fileHandler file.Handler, rootDir, workingDir string) error {
return fs.WalkDir(terraformFS, rootDir, func(path string, d fs.DirEntry, err error) error {
if err != nil {
return err

99
cli/internal/terraform/loader_test.go
View File

@ -21,7 +21,7 @@ import (
"github.com/stretchr/testify/require"
)
func TestLoader(t *testing.T) {
func TestPrepareWorkspace(t *testing.T) {
testCases := map[string]struct {
pathBase string
provider cloudprovider.Provider
@ -109,29 +109,114 @@ func TestLoader(t *testing.T) {
err := prepareWorkspace(path, file, constants.TerraformWorkingDir)
require.NoError(err)
checkFiles(t, file, func(err error) { assert.NoError(err) }, tc.fileList)
checkFiles(t, file, func(err error) { assert.NoError(err) }, constants.TerraformWorkingDir, tc.fileList)
if tc.testAlreadyUnpacked {
// Let's try the same again and check if we don't get a "file already exists" error.
require.NoError(file.Remove(filepath.Join(constants.TerraformWorkingDir, "variables.tf")))
err := prepareWorkspace(path, file, constants.TerraformWorkingDir)
assert.NoError(err)
checkFiles(t, file, func(err error) { assert.NoError(err) }, tc.fileList)
checkFiles(t, file, func(err error) { assert.NoError(err) }, constants.TerraformWorkingDir, tc.fileList)
}
err = cleanUpWorkspace(file, constants.TerraformWorkingDir)
require.NoError(err)
checkFiles(t, file, func(err error) { assert.ErrorIs(err, fs.ErrNotExist) }, tc.fileList)
checkFiles(t, file, func(err error) { assert.ErrorIs(err, fs.ErrNotExist) }, constants.TerraformWorkingDir, tc.fileList)
})
}
}
func checkFiles(t *testing.T, file file.Handler, assertion func(error), files []string) {
func TestPrepareUpgradeWorkspace(t *testing.T) {
testCases := map[string]struct {
pathBase string
provider cloudprovider.Provider
oldWorkingDir string
newWorkingDir string
oldWorkspaceFiles []string
newWorkspaceFiles []string
expectedFiles []string
testAlreadyUnpacked bool
wantErr bool
}{
"works": {
pathBase: "terraform",
provider: cloudprovider.AWS,
oldWorkingDir: "old",
newWorkingDir: "new",
oldWorkspaceFiles: []string{"terraform.tfstate"},
expectedFiles: []string{
"main.tf",
"variables.tf",
"outputs.tf",
"modules",
"terraform.tfstate",
},
},
"state file does not exist": {
pathBase: "terraform",
provider: cloudprovider.AWS,
oldWorkingDir: "old",
newWorkingDir: "new",
oldWorkspaceFiles: []string{},
expectedFiles: []string{},
wantErr: true,
},
"terraform files already exist in new dir": {
pathBase: "terraform",
provider: cloudprovider.AWS,
oldWorkingDir: "old",
newWorkingDir: "new",
oldWorkspaceFiles: []string{"terraform.tfstate"},
newWorkspaceFiles: []string{"main.tf"},
wantErr: true,
},
}
for name, tc := range testCases {
t.Run(name, func(t *testing.T) {
require := require.New(t)
assert := assert.New(t)
file := file.NewHandler(afero.NewMemMapFs())
path := path.Join(tc.pathBase, strings.ToLower(tc.provider.String()))
createFiles(t, file, tc.oldWorkspaceFiles, tc.oldWorkingDir)
createFiles(t, file, tc.newWorkspaceFiles, tc.newWorkingDir)
err := prepareUpgradeWorkspace(path, file, tc.oldWorkingDir, tc.newWorkingDir)
if tc.wantErr {
require.Error(err)
} else {
require.NoError(err)
}
checkFiles(t, file, func(err error) { assert.NoError(err) }, tc.newWorkingDir, tc.expectedFiles)
checkFiles(t, file, func(err error) { assert.NoError(err) },
filepath.Join(constants.UpgradeDir, constants.TerraformUpgradeBackupDir),
tc.oldWorkspaceFiles,
)
})
}
}
func checkFiles(t *testing.T, fileHandler file.Handler, assertion func(error), dir string, files []string) {
t.Helper()
for _, f := range files {
path := filepath.Join(constants.TerraformWorkingDir, f)
_, err := file.Stat(path)
path := filepath.Join(dir, f)
_, err := fileHandler.Stat(path)
assertion(err)
}
}
func createFiles(t *testing.T, fileHandler file.Handler, fileList []string, targetDir string) {
t.Helper()
require := require.New(t)
for _, f := range fileList {
path := filepath.Join(targetDir, f)
err := fileHandler.Write(path, []byte("1234"), file.OptOverwrite, file.OptMkdirAll)
require.NoError(err)
}
}

63
cli/internal/terraform/terraform.go
View File

@ -18,6 +18,7 @@ import (
"context"
"errors"
"fmt"
"io"
"path/filepath"
"github.com/edgelesssys/constellation/v2/internal/cloud/cloudprovider"
@ -78,14 +79,24 @@ func (c *Client) Show(ctx context.Context) (*tfjson.State, error) {
// PrepareWorkspace prepares a Terraform workspace for a Constellation cluster.
func (c *Client) PrepareWorkspace(path string, vars Variables) error {
if err := prepareWorkspace(path, c.file, c.workingDir); err != nil {
return err
return fmt.Errorf("prepare workspace: %w", err)
}
return c.writeVars(vars)
}
// PrepareUpgradeWorkspace prepares a Terraform workspace for a Constellation version upgrade.
// It copies the Terraform state from the old working dir and the embedded Terraform files into the new working dir.
func (c *Client) PrepareUpgradeWorkspace(path, oldWorkingDir, newWorkingDir string, vars Variables) error {
if err := prepareUpgradeWorkspace(path, c.file, oldWorkingDir, newWorkingDir); err != nil {
return fmt.Errorf("prepare upgrade workspace: %w", err)
}
return c.writeVars(vars)
}
// CreateCluster creates a Constellation cluster using Terraform.
func (c *Client) CreateCluster(ctx context.Context, logLevel LogLevel) (CreateOutput, error) {
func (c *Client) CreateCluster(ctx context.Context, logLevel LogLevel, targets ...string) (CreateOutput, error) {
if err := c.setLogLevel(logLevel); err != nil {
return CreateOutput{}, fmt.Errorf("set terraform log level %s: %w", logLevel.String(), err)
}
@ -94,7 +105,12 @@ func (c *Client) CreateCluster(ctx context.Context, logLevel LogLevel) (CreateOu
return CreateOutput{}, fmt.Errorf("terraform init: %w", err)
}
if err := c.tf.Apply(ctx); err != nil {
opts := []tfexec.ApplyOption{}
for _, target := range targets {
opts = append(opts, tfexec.Target(target))
}
if err := c.tf.Apply(ctx, opts...); err != nil {
return CreateOutput{}, fmt.Errorf("terraform apply: %w", err)
}
@ -294,6 +310,45 @@ func (c *Client) CreateIAMConfig(ctx context.Context, provider cloudprovider.Pro
}
}
// Plan determines the diff that will be applied by Terraform. The plan output is written to the planFile.
// If there is a diff, the returned bool is true. Otherwise, it is false.
func (c *Client) Plan(ctx context.Context, logLevel LogLevel, planFile string, targets ...string) (bool, error) {
if err := c.setLogLevel(logLevel); err != nil {
return false, fmt.Errorf("set terraform log level %s: %w", logLevel.String(), err)
}
if err := c.tf.Init(ctx); err != nil {
return false, fmt.Errorf("terraform init: %w", err)
}
opts := []tfexec.PlanOption{
tfexec.Out(planFile),
}
for _, target := range targets {
opts = append(opts, tfexec.Target(target))
}
return c.tf.Plan(ctx, opts...)
}
// ShowPlan formats the diff in planFilePath and writes it to the specified output.
func (c *Client) ShowPlan(ctx context.Context, logLevel LogLevel, planFilePath string, output io.Writer) error {
if err := c.setLogLevel(logLevel); err != nil {
return fmt.Errorf("set terraform log level %s: %w", logLevel.String(), err)
}
planResult, err := c.tf.ShowPlanFileRaw(ctx, planFilePath)
if err != nil {
return fmt.Errorf("terraform show plan: %w", err)
}
_, err = output.Write([]byte(planResult))
if err != nil {
return fmt.Errorf("write plan output: %w", err)
}
return nil
}
// Destroy destroys Terraform-created cloud resources.
func (c *Client) Destroy(ctx context.Context, logLevel LogLevel) error {
if err := c.setLogLevel(logLevel); err != nil {
@ -386,6 +441,8 @@ type tfInterface interface {
Destroy(context.Context, ...tfexec.DestroyOption) error
Init(context.Context, ...tfexec.InitOption) error
Show(context.Context, ...tfexec.ShowOption) (*tfjson.State, error)
Plan(ctx context.Context, opts ...tfexec.PlanOption) (bool, error)
ShowPlanFileRaw(ctx context.Context, planPath string, opts ...tfexec.ShowOption) (string, error)
SetLog(level string) error
SetLogPath(path string) error
}

152
cli/internal/terraform/terraform_test.go
View File

@ -7,6 +7,7 @@ SPDX-License-Identifier: AGPL-3.0-only
package terraform
import (
"bytes"
"context"
"errors"
"io/fs"
@ -934,14 +935,143 @@ func TestLogLevelString(t *testing.T) {
}
}
func TestPlan(t *testing.T) {
someError := errors.New("some error")
testCases := map[string]struct {
pathBase string
tf *stubTerraform
fs afero.Fs
wantErr bool
}{
"plan succeeds": {
pathBase: "terraform",
tf: &stubTerraform{},
fs: afero.NewMemMapFs(),
},
"set log path fails": {
pathBase: "terraform",
tf: &stubTerraform{
setLogPathErr: someError,
},
fs: afero.NewMemMapFs(),
wantErr: true,
},
"set log fails": {
pathBase: "terraform",
tf: &stubTerraform{
setLogErr: someError,
},
fs: afero.NewMemMapFs(),
wantErr: true,
},
"plan fails": {
pathBase: "terraform",
tf: &stubTerraform{
planJSONErr: someError,
},
fs: afero.NewMemMapFs(),
wantErr: true,
},
"init fails": {
pathBase: "terraform",
tf: &stubTerraform{
initErr: someError,
},
fs: afero.NewMemMapFs(),
wantErr: true,
},
}
for name, tc := range testCases {
t.Run(name, func(t *testing.T) {
require := require.New(t)
c := &Client{
file: file.NewHandler(tc.fs),
tf: tc.tf,
workingDir: tc.pathBase,
}
_, err := c.Plan(context.Background(), LogLevelDebug, constants.TerraformUpgradePlanFile)
if tc.wantErr {
require.Error(err)
} else {
require.NoError(err)
}
})
}
}
func TestShowPlan(t *testing.T) {
someError := errors.New("some error")
testCases := map[string]struct {
pathBase string
tf *stubTerraform
fs afero.Fs
wantErr bool
}{
"show plan succeeds": {
pathBase: "terraform",
tf: &stubTerraform{},
fs: afero.NewMemMapFs(),
},
"set log path fails": {
pathBase: "terraform",
tf: &stubTerraform{
setLogPathErr: someError,
},
fs: afero.NewMemMapFs(),
wantErr: true,
},
"set log fails": {
pathBase: "terraform",
tf: &stubTerraform{
setLogErr: someError,
},
fs: afero.NewMemMapFs(),
wantErr: true,
},
"show plan file fails": {
pathBase: "terraform",
tf: &stubTerraform{
showPlanFileErr: someError,
},
fs: afero.NewMemMapFs(),
wantErr: true,
},
}
for name, tc := range testCases {
t.Run(name, func(t *testing.T) {
require := require.New(t)
c := &Client{
file: file.NewHandler(tc.fs),
tf: tc.tf,
workingDir: tc.pathBase,
}
err := c.ShowPlan(context.Background(), LogLevelDebug, "", bytes.NewBuffer(nil))
if tc.wantErr {
require.Error(err)
} else {
require.NoError(err)
}
})
}
}
type stubTerraform struct {
applyErr error
destroyErr error
initErr error
showErr error
setLogErr error
setLogPathErr error
showState *tfjson.State
applyErr error
destroyErr error
initErr error
showErr error
setLogErr error
setLogPathErr error
planJSONErr error
showPlanFileErr error
showState *tfjson.State
}
func (s *stubTerraform) Apply(context.Context, ...tfexec.ApplyOption) error {
@ -960,6 +1090,14 @@ func (s *stubTerraform) Show(context.Context, ...tfexec.ShowOption) (*tfjson.Sta
return s.showState, s.showErr
}
func (s *stubTerraform) Plan(context.Context, ...tfexec.PlanOption) (bool, error) {
return false, s.planJSONErr
}
func (s *stubTerraform) ShowPlanFileRaw(context.Context, string, ...tfexec.ShowOption) (string, error) {
return "", s.showPlanFileErr
}
func (s *stubTerraform) SetLog(_ string) error {
return s.setLogErr
}

34
cli/internal/upgrade/BUILD.bazel Normal file
View File

@ -0,0 +1,34 @@
load("@io_bazel_rules_go//go:def.bzl", "go_library")
load("//bazel/go:go_test.bzl", "go_test")
go_library(
name = "upgrade",
srcs = [
"terraform.go",
"upgrade.go",
],
importpath = "github.com/edgelesssys/constellation/v2/cli/internal/upgrade",
visibility = ["//cli:__subpackages__"],
deps = [
"//cli/internal/clusterid",
"//cli/internal/terraform",
"//internal/cloud/cloudprovider",
"//internal/constants",
"//internal/file",
],
)
go_test(
name = "upgrade_test",
srcs = ["terraform_test.go"],
embed = [":upgrade"],
deps = [
"//cli/internal/terraform",
"//internal/cloud/cloudprovider",
"//internal/constants",
"//internal/file",
"@com_github_spf13_afero//:afero",
"@com_github_stretchr_testify//assert",
"@com_github_stretchr_testify//require",
],
)

175
cli/internal/upgrade/terraform.go Normal file
View File

@ -0,0 +1,175 @@
/*
Copyright (c) Edgeless Systems GmbH
SPDX-License-Identifier: AGPL-3.0-only
*/
package upgrade
import (
"context"
"fmt"
"io"
"os"
"path/filepath"
"strings"
"github.com/edgelesssys/constellation/v2/cli/internal/clusterid"
"github.com/edgelesssys/constellation/v2/cli/internal/terraform"
"github.com/edgelesssys/constellation/v2/internal/cloud/cloudprovider"
"github.com/edgelesssys/constellation/v2/internal/constants"
"github.com/edgelesssys/constellation/v2/internal/file"
)
// NewTerraformUpgrader returns a new TerraformUpgrader.
func NewTerraformUpgrader(tfClient tfClient, outWriter io.Writer) (*TerraformUpgrader, error) {
return &TerraformUpgrader{
tf: tfClient,
outWriter: outWriter,
}, nil
}
// TerraformUpgrader is responsible for performing Terraform migrations on cluster upgrades.
type TerraformUpgrader struct {
tf tfClient
outWriter io.Writer
}
// TerraformUpgradeOptions are the options used for the Terraform upgrade.
type TerraformUpgradeOptions struct {
// LogLevel is the log level used for Terraform.
LogLevel terraform.LogLevel
// CSP is the cloud provider to perform the upgrade on.
CSP cloudprovider.Provider
// Vars are the Terraform variables used for the upgrade.
Vars terraform.Variables
// Targets are the Terraform targets used for the upgrade.
Targets []string
// OutputFile is the file to write the Terraform output to.
OutputFile string
}
// CheckTerraformMigrations checks whether Terraform migrations are possible in the current workspace.
// If the files that will be written during the upgrade already exist, it returns an error.
func (u *TerraformUpgrader) CheckTerraformMigrations(fileHandler file.Handler) error {
var existingFiles []string
filesToCheck := []string{
constants.TerraformMigrationOutputFile,
filepath.Join(constants.UpgradeDir, constants.TerraformUpgradeBackupDir),
}
for _, f := range filesToCheck {
if err := checkFileExists(fileHandler, &existingFiles, f); err != nil {
return fmt.Errorf("checking terraform migrations: %w", err)
}
}
if len(existingFiles) > 0 {
return fmt.Errorf("file(s) %s already exist", strings.Join(existingFiles, ", "))
}
return nil
}
// checkFileExists checks whether a file exists and adds it to the existingFiles slice if it does.
func checkFileExists(fileHandler file.Handler, existingFiles *[]string, filename string) error {
_, err := fileHandler.Stat(filename)
if err != nil {
if !os.IsNotExist(err) {
return fmt.Errorf("checking %s: %w", filename, err)
}
return nil
}
*existingFiles = append(*existingFiles, filename)
return nil
}
// PlanTerraformMigrations prepares the upgrade workspace and plans the Terraform migrations for the Constellation upgrade.
// If a diff exists, it's being written to the upgrader's output writer. It also returns
// a bool indicating whether a diff exists.
func (u *TerraformUpgrader) PlanTerraformMigrations(ctx context.Context, opts TerraformUpgradeOptions) (bool, error) {
err := u.tf.PrepareUpgradeWorkspace(
filepath.Join("terraform", strings.ToLower(opts.CSP.String())),
constants.TerraformWorkingDir,
filepath.Join(constants.UpgradeDir, constants.TerraformUpgradeWorkingDir),
opts.Vars,
)
if err != nil {
return false, fmt.Errorf("preparing terraform workspace: %w", err)
}
hasDiff, err := u.tf.Plan(ctx, opts.LogLevel, constants.TerraformUpgradePlanFile, opts.Targets...)
if err != nil {
return false, fmt.Errorf("terraform plan: %w", err)
}
if hasDiff {
if err := u.tf.ShowPlan(ctx, opts.LogLevel, constants.TerraformUpgradePlanFile, u.outWriter); err != nil {
return false, fmt.Errorf("terraform show plan: %w", err)
}
}
return hasDiff, nil
}
// CleanUpTerraformMigrations cleans up the Terraform migration workspace, for example when an upgrade is
// aborted by the user.
func (u *TerraformUpgrader) CleanUpTerraformMigrations(fileHandler file.Handler) error {
cleanupFiles := []string{
filepath.Join(constants.UpgradeDir, constants.TerraformUpgradeBackupDir),
filepath.Join(constants.UpgradeDir, constants.TerraformUpgradeWorkingDir),
}
for _, f := range cleanupFiles {
if err := fileHandler.RemoveAll(f); err != nil {
return fmt.Errorf("cleaning up file %s: %w", f, err)
}
}
return nil
}
// ApplyTerraformMigrations applies the migerations planned by PlanTerraformMigrations.
// If PlanTerraformMigrations has not been executed before, it will return an error.
// In case of a successful upgrade, the output will be written to the specified file and the old Terraform directory is replaced
// By the new one.
func (u *TerraformUpgrader) ApplyTerraformMigrations(ctx context.Context, fileHandler file.Handler, opts TerraformUpgradeOptions) error {
tfOutput, err := u.tf.CreateCluster(ctx, opts.LogLevel, opts.Targets...)
if err != nil {
return fmt.Errorf("terraform apply: %w", err)
}
outputFileContents := clusterid.File{
CloudProvider: opts.CSP,
InitSecret: []byte(tfOutput.Secret),
IP: tfOutput.IP,
UID: tfOutput.UID,
AttestationURL: tfOutput.AttestationURL,
}
if err := fileHandler.RemoveAll(constants.TerraformWorkingDir); err != nil {
return fmt.Errorf("removing old terraform directory: %w", err)
}
if err := fileHandler.CopyDir(filepath.Join(constants.UpgradeDir, constants.TerraformUpgradeWorkingDir), constants.TerraformWorkingDir); err != nil {
return fmt.Errorf("replacing old terraform directory with new one: %w", err)
}
if err := fileHandler.RemoveAll(filepath.Join(constants.UpgradeDir, constants.TerraformUpgradeWorkingDir)); err != nil {
return fmt.Errorf("removing terraform upgrade directory: %w", err)
}
if err := fileHandler.WriteJSON(opts.OutputFile, outputFileContents); err != nil {
return fmt.Errorf("writing terraform output to file: %w", err)
}
return nil
}
// a tfClient performs the Terraform interactions in an upgrade.
type tfClient interface {
PrepareUpgradeWorkspace(path, oldWorkingDir, newWorkingDir string, vars terraform.Variables) error
ShowPlan(ctx context.Context, logLevel terraform.LogLevel, planFilePath string, output io.Writer) error
Plan(ctx context.Context, logLevel terraform.LogLevel, planFile string, targets ...string) (bool, error)
CreateCluster(ctx context.Context, logLevel terraform.LogLevel, targets ...string) (terraform.CreateOutput, error)
}

313
cli/internal/upgrade/terraform_test.go Normal file
View File

@ -0,0 +1,313 @@
/*
Copyright (c) Edgeless Systems GmbH
SPDX-License-Identifier: AGPL-3.0-only
*/
package upgrade
import (
"bytes"
"context"
"io"
"path/filepath"
"testing"
"github.com/edgelesssys/constellation/v2/cli/internal/terraform"
"github.com/edgelesssys/constellation/v2/internal/cloud/cloudprovider"
"github.com/edgelesssys/constellation/v2/internal/constants"
"github.com/edgelesssys/constellation/v2/internal/file"
"github.com/spf13/afero"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestCheckTerraformMigrations(t *testing.T) {
upgrader := func() *TerraformUpgrader {
u, err := NewTerraformUpgrader(&stubTerraformClient{}, bytes.NewBuffer(nil))
require.NoError(t, err)
return u
}
workspace := func(existingFiles []string) file.Handler {
fs := afero.NewMemMapFs()
for _, f := range existingFiles {
require.NoError(t, afero.WriteFile(fs, f, []byte{}, 0o644))
}
return file.NewHandler(fs)
}
testCases := map[string]struct {
workspace file.Handler
wantErr bool
}{
"success": {
workspace: workspace(nil),
},
"migration output file already exists": {
workspace: workspace([]string{constants.TerraformMigrationOutputFile}),
wantErr: true,
},
"terraform backup dir already exists": {
workspace: workspace([]string{filepath.Join(constants.UpgradeDir, constants.TerraformUpgradeBackupDir)}),
wantErr: true,
},
}
for name, tc := range testCases {
t.Run(name, func(t *testing.T) {
u := upgrader()
err := u.CheckTerraformMigrations(tc.workspace)
if tc.wantErr {
require.Error(t, err)
return
}
require.NoError(t, err)
})
}
}
func TestPlanTerraformMigrations(t *testing.T) {
upgrader := func(tf tfClient) *TerraformUpgrader {
u, err := NewTerraformUpgrader(tf, bytes.NewBuffer(nil))
require.NoError(t, err)
return u
}
testCases := map[string]struct {
tf tfClient
want bool
wantErr bool
}{
"success no diff": {
tf: &stubTerraformClient{},
},
"success diff": {
tf: &stubTerraformClient{
hasDiff: true,
},
want: true,
},
"prepare workspace error": {
tf: &stubTerraformClient{
prepareWorkspaceErr: assert.AnError,
},
wantErr: true,
},
"plan error": {
tf: &stubTerraformClient{
planErr: assert.AnError,
},
wantErr: true,
},
"show plan error no diff": {
tf: &stubTerraformClient{
showErr: assert.AnError,
},
},
"show plan error diff": {
tf: &stubTerraformClient{
showErr: assert.AnError,
hasDiff: true,
},
wantErr: true,
},
}
for name, tc := range testCases {
t.Run(name, func(t *testing.T) {
require := require.New(t)
u := upgrader(tc.tf)
opts := TerraformUpgradeOptions{
LogLevel: terraform.LogLevelDebug,
CSP: cloudprovider.Unknown,
Vars: &terraform.QEMUVariables{},
}
diff, err := u.PlanTerraformMigrations(context.Background(), opts)
if tc.wantErr {
require.Error(err)
} else {
require.NoError(err)
require.Equal(tc.want, diff)
}
})
}
}
func TestApplyTerraformMigrations(t *testing.T) {
upgrader := func(tf tfClient) *TerraformUpgrader {
u, err := NewTerraformUpgrader(tf, bytes.NewBuffer(nil))
require.NoError(t, err)
return u
}
fileHandler := func(existingFiles ...string) file.Handler {
fh := file.NewHandler(afero.NewMemMapFs())
require.NoError(t,
fh.Write(
filepath.Join(constants.UpgradeDir, constants.TerraformUpgradeWorkingDir, "someFile"),
[]byte("some content"),
))
for _, f := range existingFiles {
require.NoError(t, fh.Write(f, []byte("some content")))
}
return fh
}
testCases := map[string]struct {
tf tfClient
fs file.Handler
outputFileName string
wantErr bool
}{
"success": {
tf: &stubTerraformClient{},
fs: fileHandler(),
outputFileName: "test.json",
},
"create cluster error": {
tf: &stubTerraformClient{
CreateClusterErr: assert.AnError,
},
fs: fileHandler(),
outputFileName: "test.json",
wantErr: true,
},
"empty file name": {
tf: &stubTerraformClient{},
fs: fileHandler(),
outputFileName: "",
wantErr: true,
},
"file already exists": {
tf: &stubTerraformClient{},
fs: fileHandler("test.json"),
outputFileName: "test.json",
wantErr: true,
},
}
for name, tc := range testCases {
t.Run(name, func(t *testing.T) {
require := require.New(t)
u := upgrader(tc.tf)
opts := TerraformUpgradeOptions{
LogLevel: terraform.LogLevelDebug,
CSP: cloudprovider.Unknown,
Vars: &terraform.QEMUVariables{},
OutputFile: tc.outputFileName,
}
err := u.ApplyTerraformMigrations(context.Background(), tc.fs, opts)
if tc.wantErr {
require.Error(err)
} else {
require.NoError(err)
}
})
}
}
func TestCleanUpTerraformMigrations(t *testing.T) {
upgrader := func() *TerraformUpgrader {
u, err := NewTerraformUpgrader(&stubTerraformClient{}, bytes.NewBuffer(nil))
require.NoError(t, err)
return u
}
workspace := func(existingFiles []string) file.Handler {
fs := afero.NewMemMapFs()
for _, f := range existingFiles {
require.NoError(t, afero.WriteFile(fs, f, []byte{}, 0o644))
}
return file.NewHandler(fs)
}
testCases := map[string]struct {
workspace file.Handler
wantFiles []string
wantErr bool
}{
"no files": {
workspace: workspace(nil),
wantFiles: []string{},
},
"clean backup dir": {
workspace: workspace([]string{
filepath.Join(constants.UpgradeDir, constants.TerraformUpgradeBackupDir),
}),
wantFiles: []string{},
},
"clean working dir": {
workspace: workspace([]string{
filepath.Join(constants.UpgradeDir, constants.TerraformUpgradeWorkingDir),
}),
wantFiles: []string{},
},
"clean backup dir leave other files": {
workspace: workspace([]string{
filepath.Join(constants.UpgradeDir, constants.TerraformUpgradeBackupDir),
filepath.Join(constants.UpgradeDir, "someFile"),
}),
wantFiles: []string{
filepath.Join(constants.UpgradeDir, "someFile"),
},
},
}
for name, tc := range testCases {
t.Run(name, func(t *testing.T) {
require := require.New(t)
u := upgrader()
err := u.CleanUpTerraformMigrations(tc.workspace)
if tc.wantErr {
require.Error(err)
return
}
require.NoError(err)
for _, f := range tc.wantFiles {
_, err := tc.workspace.Stat(f)
require.NoError(err, "file %s should exist", f)
}
})
}
}
type stubTerraformClient struct {
hasDiff bool
prepareWorkspaceErr error
showErr error
planErr error
CreateClusterErr error
}
func (u *stubTerraformClient) PrepareUpgradeWorkspace(string, string, string, terraform.Variables) error {
return u.prepareWorkspaceErr
}
func (u *stubTerraformClient) ShowPlan(context.Context, terraform.LogLevel, string, io.Writer) error {
return u.showErr
}
func (u *stubTerraformClient) Plan(context.Context, terraform.LogLevel, string, ...string) (bool, error) {
return u.hasDiff, u.planErr
}
func (u *stubTerraformClient) CreateCluster(context.Context, terraform.LogLevel, ...string) (terraform.CreateOutput, error) {
return terraform.CreateOutput{}, u.CreateClusterErr
}

10
cli/internal/upgrade/upgrade.go Normal file
View File

@ -0,0 +1,10 @@
/*
Copyright (c) Edgeless Systems GmbH
SPDX-License-Identifier: AGPL-3.0-only
*/
/*
Package upgrade provides functionality to upgrade the cluster and it's resources
*/
package upgrade

17
e2e/internal/upgrade/upgrade_test.go
View File

@ -52,7 +52,7 @@ var (
// setup checks that the prerequisites for the test are met:
// - a workspace is set
// - a CLI path is set
// - the constellation-upgrade folder does not exist.
// - the upgrade folder does not exist.
func setup() error {
workingDir, err := workingDir(*workspace)
if err != nil {
@ -66,8 +66,8 @@ func setup() error {
if _, err := getCLIPath(*cliPath); err != nil {
return fmt.Errorf("getting CLI path: %w", err)
}
if _, err := os.Stat("constellation-upgrade"); err == nil {
return errors.New("please remove the existing constellation-upgrade folder")
if _, err := os.Stat(constants.UpgradeDir); err == nil {
return fmt.Errorf("please remove the existing %s folder", constants.UpgradeDir)
}
return nil
@ -107,7 +107,16 @@ func TestUpgrade(t *testing.T) {
log.Println(string(data))
log.Println("Triggering upgrade.")
cmd = exec.CommandContext(context.Background(), cli, "upgrade", "apply", "--force", "--debug", "-y")
tfLogFlag := ""
cmd = exec.CommandContext(context.Background(), cli, "--help")
msg, err = cmd.CombinedOutput()
require.NoErrorf(err, "%s", string(msg))
if strings.Contains(string(msg), "--tf-log") {
tfLogFlag = "--tf-log=DEBUG"
}
cmd = exec.CommandContext(context.Background(), cli, "upgrade", "apply", "--force", "--debug", "--yes", tfLogFlag)
msg, err = cmd.CombinedOutput()
require.NoErrorf(err, "%s", string(msg))
require.NoError(containsUnexepectedMsg(string(msg)))

10
internal/constants/constants.go
View File

@ -148,6 +148,16 @@ const (
MiniConstellationUID = "mini"
// TerraformLogFile is the file name of the Terraform log file.
TerraformLogFile = "terraform.log"
// TerraformUpgradePlanFile is the file name of the zipfile created by Terraform plan for Constellation upgrades.
TerraformUpgradePlanFile = "plan.zip"
// TerraformUpgradeWorkingDir is the directory name for the Terraform workspace being used in an upgrade.
TerraformUpgradeWorkingDir = "terraform"
// TerraformUpgradeBackupDir is the directory name being used to backup the pre-upgrade state in an upgrade.
TerraformUpgradeBackupDir = "terraform-backup"
// TerraformMigrationOutputFile is the file name of the output file created by a successful Terraform migration.
TerraformMigrationOutputFile = "terraform-migration-output.json"
// UpgradeDir is the name of the directory being used for cluster upgrades.
UpgradeDir = "constellation-upgrade"
//
// Kubernetes.

50
internal/file/file.go
View File

@ -14,10 +14,13 @@ import (
"bytes"
"encoding/json"
"errors"
"fmt"
"io"
"io/fs"
"os"
"path"
"path/filepath"
"strings"
"github.com/siderolabs/talos/pkg/machinery/config/encoder"
"github.com/spf13/afero"
@ -175,3 +178,50 @@ func (h *Handler) Stat(name string) (fs.FileInfo, error) {
func (h *Handler) MkdirAll(name string) error {
return h.fs.MkdirAll(name, 0o700)
}
// CopyDir copies the src directory recursively into dst with the given options. OptMkdirAll
// is always set. CopyDir does not follow symlinks.
func (h *Handler) CopyDir(src, dst string, opts ...Option) error {
opts = append(opts, OptMkdirAll)
root := filepath.Join(src, string(filepath.Separator))
walkFunc := func(path string, info fs.FileInfo, err error) error {
if err != nil {
return err
}
if info.IsDir() {
return nil
}
pathWithoutRoot := strings.TrimPrefix(path, root)
return h.CopyFile(path, filepath.Join(dst, pathWithoutRoot), opts...)
}
return h.fs.Walk(src, walkFunc)
}
// CopyFile copies the file from src to dst with the given options, respecting file permissions.
func (h *Handler) CopyFile(src, dst string, opts ...Option) error {
srcInfo, err := h.fs.Stat(src)
if err != nil {
return fmt.Errorf("stat source file: %w", err)
}
content, err := h.fs.ReadFile(src)
if err != nil {
return fmt.Errorf("read source file: %w", err)
}
err = h.Write(dst, content, opts...)
if err != nil {
return fmt.Errorf("write destination file: %w", err)
}
err = h.fs.Chmod(dst, srcInfo.Mode())
if err != nil {
return fmt.Errorf("chmod destination file: %w", err)
}
return nil
}

122
internal/file/file_test.go
View File

@ -8,6 +8,8 @@ package file
import (
"encoding/json"
"io/fs"
"path/filepath"
"testing"
"github.com/edgelesssys/constellation/v2/internal/constants"
@ -350,3 +352,123 @@ func TestRemove(t *testing.T) {
assert.Error(handler.Remove("d"))
}
func TestCopyFile(t *testing.T) {
perms := fs.FileMode(0o644)
setupFs := func(existingFiles ...string) afero.Fs {
fs := afero.NewMemMapFs()
aferoHelper := afero.Afero{Fs: fs}
for _, file := range existingFiles {
require.NoError(t, aferoHelper.WriteFile(file, []byte{}, perms))
}
return fs
}
testCases := map[string]struct {
fs afero.Fs
copyFiles [][]string
checkFiles []string
opts []Option
wantErr bool
}{
"successful copy": {
fs: setupFs("a"),
copyFiles: [][]string{{"a", "b"}},
checkFiles: []string{"b"},
},
"copy to existing file overwrite": {
fs: setupFs("a", "b"),
copyFiles: [][]string{{"a", "b"}},
checkFiles: []string{"b"},
opts: []Option{OptOverwrite},
},
"copy to existing file no overwrite": {
fs: setupFs("a", "b"),
copyFiles: [][]string{{"a", "b"}},
checkFiles: []string{"b"},
wantErr: true,
},
"file doesn't exist": {
fs: setupFs("a"),
copyFiles: [][]string{{"b", "c"}},
checkFiles: []string{"a"},
wantErr: true,
},
}
for name, tc := range testCases {
t.Run(name, func(t *testing.T) {
assert := assert.New(t)
require := require.New(t)
handler := NewHandler(tc.fs)
for _, files := range tc.copyFiles {
err := handler.CopyFile(files[0], files[1], tc.opts...)
if tc.wantErr {
assert.Error(err)
} else {
assert.NoError(err)
}
}
for _, file := range tc.checkFiles {
info, err := handler.fs.Stat(file)
assert.Equal(perms, info.Mode())
require.NoError(err)
}
})
}
}
func TestCopyDir(t *testing.T) {
setupHandler := func(existingFiles ...string) Handler {
fs := afero.NewMemMapFs()
handler := NewHandler(fs)
for _, file := range existingFiles {
err := handler.Write(file, []byte("some content"), OptMkdirAll)
require.NoError(t, err)
}
return handler
}
testCases := map[string]struct {
handler Handler
copyFiles [][]string
checkFiles []string
opts []Option
}{
"successful copy": {
handler: setupHandler(filepath.Join("someDir", "someFile"), filepath.Join("someDir", "someOtherDir", "someOtherFile")),
copyFiles: [][]string{{"someDir", "copiedDir"}},
checkFiles: []string{filepath.Join("copiedDir", "someFile"), filepath.Join("copiedDir", "someOtherDir", "someOtherFile")},
},
"copy file": {
handler: setupHandler("someFile"),
copyFiles: [][]string{{"someFile", "copiedFile"}},
checkFiles: []string{"copiedFile"},
},
"copy to existing dir overwrite": {
handler: setupHandler(filepath.Join("someDir", "someFile"), filepath.Join("someDir", "someOtherDir", "someOtherFile"), filepath.Join("copiedDir", "someExistingFile")),
copyFiles: [][]string{{"someDir", "copiedDir"}},
checkFiles: []string{filepath.Join("copiedDir", "someFile"), filepath.Join("copiedDir", "someOtherDir", "someOtherFile"), filepath.Join("copiedDir", "someExistingFile")},
opts: []Option{OptOverwrite},
},
}
for name, tc := range testCases {
t.Run(name, func(t *testing.T) {
require := require.New(t)
for _, files := range tc.copyFiles {
err := tc.handler.CopyDir(files[0], files[1], tc.opts...)
require.NoError(err)
}
for _, file := range tc.checkFiles {
_, err := tc.handler.fs.Stat(file)
require.NoError(err)
}
})
}
}