mirror of
https://github.com/edgelesssys/constellation.git
synced 2025-08-02 20:16:15 -04:00
aws: use new LB controller to fix SecurityGroup cleanup on K8s service deletion (#2090)
* add current chart add current helm chart * disable service controller for aws ccm * add new iam roles * doc AWS internet LB + add to LB test * pass clusterName to helm for AWS LB * fix update-aws-lb chart to also include .helmignore * move chart outside services * working state * add subnet tags for AWS subnet discovery * fix .helmignore load rule with file in subdirectory * upgrade iam profile * revert new loader impl since cilium is not correctly loaded * install chart if not already present during `upgrade apply` * cleanup PR + fix build + add todos cleanup PR + add todos * shared helm pkg for cli install and bootstrapper * add link to eks docs * refactor iamMigrationCmd * delete unused helm.symwallk * move iammigrate to upgrade pkg * fixup! delete unused helm.symwallk * add to upgradecheck * remove nodeSelector from go code (Otto) * update iam docs and sort permission + remove duplicate roles * fix bug in `upgrade check` * better upgrade check output when svc version upgrade not possible * pr feedback * remove force flag in upgrade_test * use upgrader.GetUpgradeID instead of extra type * remove todos + fix check * update doc lb (leo) * remove bootstrapper helm package * Update cli/internal/cmd/upgradecheck.go Co-authored-by: Daniel Weiße <66256922+daniel-weisse@users.noreply.github.com> * final nits * add docs for e2e upgrade test setup * Apply suggestions from code review Co-authored-by: Daniel Weiße <66256922+daniel-weisse@users.noreply.github.com> * Update cli/internal/helm/loader.go Co-authored-by: Daniel Weiße <66256922+daniel-weisse@users.noreply.github.com> * Update cli/internal/cmd/tfmigrationclient.go Co-authored-by: Daniel Weiße <66256922+daniel-weisse@users.noreply.github.com> * fix daniel review * link to the iam permissions instead of manually updating them (agreed with leo) * disable iam upgrade in upgrade apply --------- Co-authored-by: Daniel Weiße <66256922+daniel-weisse@users.noreply.github.com> Co-authored-by: Malte Poll
This commit is contained in:
Adrian Stobbe
GitHub
parent
8da6a23aa5
commit
a87b7894db
67 changed files with 3018 additions and 451 deletions
|
|
@ -4,6 +4,7 @@ load("//bazel/go:go_test.bzl", "go_test")
|
|||
go_library(
|
||||
name = "upgrade",
|
||||
srcs = [
|
||||
"iammigrate.go",
|
||||
"terraform.go",
|
||||
"upgrade.go",
|
||||
],
|
||||
|
|
@ -21,7 +22,10 @@ go_library(
|
|||
|
||||
go_test(
|
||||
name = "upgrade_test",
|
||||
srcs = ["terraform_test.go"],
|
||||
srcs = [
|
||||
"iammigrate_test.go",
|
||||
"terraform_test.go",
|
||||
],
|
||||
embed = [":upgrade"],
|
||||
deps = [
|
||||
"//cli/internal/terraform",
|
||||
|
|
|
|||
112
cli/internal/upgrade/iammigrate.go
Normal file
112
cli/internal/upgrade/iammigrate.go
Normal file
|
|
@ -0,0 +1,112 @@
|
|||
/*
|
||||
Copyright (c) Edgeless Systems GmbH
|
||||
|
||||
SPDX-License-Identifier: AGPL-3.0-only
|
||||
*/
|
||||
|
||||
package upgrade
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"github.com/edgelesssys/constellation/v2/cli/internal/terraform"
|
||||
"github.com/edgelesssys/constellation/v2/internal/cloud/cloudprovider"
|
||||
"github.com/edgelesssys/constellation/v2/internal/constants"
|
||||
"github.com/edgelesssys/constellation/v2/internal/file"
|
||||
)
|
||||
|
||||
// TfMigrationCmd is an interface for all terraform upgrade / migration commands.
|
||||
type TfMigrationCmd interface {
|
||||
CheckTerraformMigrations(file file.Handler) error
|
||||
Plan(ctx context.Context, file file.Handler, outWriter io.Writer) (bool, error)
|
||||
Apply(ctx context.Context, fileHandler file.Handler) error
|
||||
String() string
|
||||
UpgradeID() string
|
||||
}
|
||||
|
||||
// IAMMigrateCmd is a terraform migration command for IAM.
|
||||
type IAMMigrateCmd struct {
|
||||
tf tfIAMClient
|
||||
upgradeID string
|
||||
csp cloudprovider.Provider
|
||||
logLevel terraform.LogLevel
|
||||
}
|
||||
|
||||
// NewIAMMigrateCmd creates a new IAMMigrateCmd.
|
||||
func NewIAMMigrateCmd(ctx context.Context, upgradeID string, csp cloudprovider.Provider, logLevel terraform.LogLevel) (*IAMMigrateCmd, error) {
|
||||
tfClient, err := terraform.New(ctx, filepath.Join(constants.UpgradeDir, upgradeID, constants.TerraformIAMUpgradeWorkingDir))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("setting up terraform client: %w", err)
|
||||
}
|
||||
return &IAMMigrateCmd{
|
||||
tf: tfClient,
|
||||
upgradeID: upgradeID,
|
||||
csp: csp,
|
||||
logLevel: logLevel,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// String returns the name of the command.
|
||||
func (c *IAMMigrateCmd) String() string {
|
||||
return "iam migration"
|
||||
}
|
||||
|
||||
// UpgradeID returns the upgrade ID.
|
||||
func (c *IAMMigrateCmd) UpgradeID() string {
|
||||
return c.upgradeID
|
||||
}
|
||||
|
||||
// CheckTerraformMigrations checks whether Terraform migrations are possible in the current workspace.
|
||||
func (c *IAMMigrateCmd) CheckTerraformMigrations(file file.Handler) error {
|
||||
return checkTerraformMigrations(file, c.upgradeID, constants.TerraformIAMUpgradeBackupDir)
|
||||
}
|
||||
|
||||
// Plan prepares the upgrade workspace and plans the Terraform migrations for the Constellation upgrade, writing the plan to the outWriter.
|
||||
func (c *IAMMigrateCmd) Plan(ctx context.Context, file file.Handler, outWriter io.Writer) (bool, error) {
|
||||
templateDir := filepath.Join("terraform", "iam", strings.ToLower(c.csp.String()))
|
||||
if err := terraform.PrepareIAMUpgradeWorkspace(file,
|
||||
templateDir,
|
||||
constants.TerraformIAMWorkingDir,
|
||||
filepath.Join(constants.UpgradeDir, c.upgradeID, constants.TerraformIAMUpgradeWorkingDir),
|
||||
filepath.Join(constants.UpgradeDir, c.upgradeID, constants.TerraformIAMUpgradeBackupDir),
|
||||
); err != nil {
|
||||
return false, fmt.Errorf("preparing terraform workspace: %w", err)
|
||||
}
|
||||
|
||||
hasDiff, err := c.tf.Plan(ctx, c.logLevel, constants.TerraformUpgradePlanFile)
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("terraform plan: %w", err)
|
||||
}
|
||||
|
||||
if hasDiff {
|
||||
if err := c.tf.ShowPlan(ctx, c.logLevel, constants.TerraformUpgradePlanFile, outWriter); err != nil {
|
||||
return false, fmt.Errorf("terraform show plan: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
return hasDiff, nil
|
||||
}
|
||||
|
||||
// Apply applies the Terraform IAM migrations for the Constellation upgrade.
|
||||
func (c *IAMMigrateCmd) Apply(ctx context.Context, fileHandler file.Handler) error {
|
||||
if _, err := c.tf.ApplyIAMConfig(ctx, c.csp, c.logLevel); err != nil {
|
||||
return fmt.Errorf("terraform apply: %w", err)
|
||||
}
|
||||
|
||||
if err := fileHandler.RemoveAll(constants.TerraformIAMWorkingDir); err != nil {
|
||||
return fmt.Errorf("removing old terraform directory: %w", err)
|
||||
}
|
||||
if err := fileHandler.CopyDir(filepath.Join(constants.UpgradeDir, c.upgradeID, constants.TerraformIAMUpgradeWorkingDir), constants.TerraformIAMWorkingDir); err != nil {
|
||||
return fmt.Errorf("replacing old terraform directory with new one: %w", err)
|
||||
}
|
||||
|
||||
if err := fileHandler.RemoveAll(filepath.Join(constants.UpgradeDir, c.upgradeID, constants.TerraformIAMUpgradeWorkingDir)); err != nil {
|
||||
return fmt.Errorf("removing terraform upgrade directory: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
112
cli/internal/upgrade/iammigrate_test.go
Normal file
112
cli/internal/upgrade/iammigrate_test.go
Normal file
|
|
@ -0,0 +1,112 @@
|
|||
/*
|
||||
Copyright (c) Edgeless Systems GmbH
|
||||
|
||||
SPDX-License-Identifier: AGPL-3.0-only
|
||||
*/
|
||||
|
||||
package upgrade
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"io"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/edgelesssys/constellation/v2/cli/internal/terraform"
|
||||
"github.com/edgelesssys/constellation/v2/internal/cloud/cloudprovider"
|
||||
"github.com/edgelesssys/constellation/v2/internal/constants"
|
||||
"github.com/edgelesssys/constellation/v2/internal/file"
|
||||
"github.com/spf13/afero"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestIAMMigrate(t *testing.T) {
|
||||
upgradeID := "test-upgrade"
|
||||
upgradeDir := filepath.Join(constants.UpgradeDir, upgradeID, constants.TerraformIAMUpgradeWorkingDir)
|
||||
fs, file := setupMemFSAndFileHandler(t, []string{"terraform.tfvars", "terraform.tfstate"}, []byte("OLD"))
|
||||
// act
|
||||
fakeTfClient := &tfClientStub{upgradeID, file}
|
||||
sut := &IAMMigrateCmd{fakeTfClient, upgradeID, cloudprovider.AWS, terraform.LogLevelDebug}
|
||||
hasDiff, err := sut.Plan(context.Background(), file, bytes.NewBuffer(nil))
|
||||
// assert
|
||||
assert.NoError(t, err)
|
||||
assert.False(t, hasDiff)
|
||||
assertFileExists(fs, filepath.Join(upgradeDir, "terraform.tfvars"), t)
|
||||
assertFileExists(fs, filepath.Join(upgradeDir, "terraform.tfstate"), t)
|
||||
|
||||
// act
|
||||
err = sut.Apply(context.Background(), file)
|
||||
assert.NoError(t, err)
|
||||
// assert
|
||||
assertFileReadsContent(file, filepath.Join(constants.TerraformIAMWorkingDir, "terraform.tfvars"), "NEW", t)
|
||||
assertFileReadsContent(file, filepath.Join(constants.TerraformIAMWorkingDir, "terraform.tfstate"), "NEW", t)
|
||||
assertFileDoesntExist(fs, filepath.Join(upgradeDir), t)
|
||||
}
|
||||
|
||||
func assertFileReadsContent(file file.Handler, path string, expectedContent string, t *testing.T) {
|
||||
bt, err := file.Read(path)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, expectedContent, string(bt))
|
||||
}
|
||||
|
||||
func assertFileExists(fs afero.Fs, path string, t *testing.T) {
|
||||
res, err := fs.Stat(path)
|
||||
assert.NoError(t, err)
|
||||
assert.NotNil(t, res)
|
||||
}
|
||||
|
||||
func assertFileDoesntExist(fs afero.Fs, path string, t *testing.T) {
|
||||
res, err := fs.Stat(path)
|
||||
assert.Error(t, err)
|
||||
assert.Nil(t, res)
|
||||
}
|
||||
|
||||
// setupMemFSAndFileHandler sets up a file handler with a memory file system and writes the given files with the given content.
|
||||
func setupMemFSAndFileHandler(t *testing.T, files []string, content []byte) (afero.Fs, file.Handler) {
|
||||
fs := afero.NewMemMapFs()
|
||||
file := file.NewHandler(fs)
|
||||
err := file.MkdirAll(constants.TerraformIAMWorkingDir)
|
||||
require.NoError(t, err)
|
||||
|
||||
for _, f := range files {
|
||||
err := file.Write(filepath.Join(constants.TerraformIAMWorkingDir, f), content)
|
||||
require.NoError(t, err)
|
||||
}
|
||||
return fs, file
|
||||
}
|
||||
|
||||
type tfClientStub struct {
|
||||
upgradeID string
|
||||
file file.Handler
|
||||
}
|
||||
|
||||
func (t *tfClientStub) Plan(_ context.Context, _ terraform.LogLevel, _ string) (bool, error) {
|
||||
return false, nil
|
||||
}
|
||||
|
||||
func (t *tfClientStub) ShowPlan(_ context.Context, _ terraform.LogLevel, _ string, _ io.Writer) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (t *tfClientStub) ApplyIAMConfig(_ context.Context, _ cloudprovider.Provider, _ terraform.LogLevel) (terraform.IAMOutput, error) {
|
||||
upgradeDir := filepath.Join(constants.UpgradeDir, t.upgradeID, constants.TerraformIAMUpgradeWorkingDir)
|
||||
err := t.file.Remove(filepath.Join(upgradeDir, "terraform.tfvars"))
|
||||
if err != nil {
|
||||
return terraform.IAMOutput{}, err
|
||||
}
|
||||
err = t.file.Write(filepath.Join(upgradeDir, "terraform.tfvars"), []byte("NEW"))
|
||||
if err != nil {
|
||||
return terraform.IAMOutput{}, err
|
||||
}
|
||||
err = t.file.Remove(filepath.Join(upgradeDir, "terraform.tfstate"))
|
||||
if err != nil {
|
||||
return terraform.IAMOutput{}, err
|
||||
}
|
||||
err = t.file.Write(filepath.Join(upgradeDir, "terraform.tfstate"), []byte("NEW"))
|
||||
if err != nil {
|
||||
return terraform.IAMOutput{}, err
|
||||
}
|
||||
return terraform.IAMOutput{}, nil
|
||||
}
|
||||
|
|
@ -23,7 +23,7 @@ import (
|
|||
)
|
||||
|
||||
// NewTerraformUpgrader returns a new TerraformUpgrader.
|
||||
func NewTerraformUpgrader(tfClient tfClient, outWriter io.Writer, fileHandler file.Handler) (*TerraformUpgrader, error) {
|
||||
func NewTerraformUpgrader(tfClient tfResourceClient, outWriter io.Writer, fileHandler file.Handler) (*TerraformUpgrader, error) {
|
||||
return &TerraformUpgrader{
|
||||
tf: tfClient,
|
||||
policyPatcher: cloudcmd.NewAzurePolicyPatcher(),
|
||||
|
|
@ -34,7 +34,7 @@ func NewTerraformUpgrader(tfClient tfClient, outWriter io.Writer, fileHandler fi
|
|||
|
||||
// TerraformUpgrader is responsible for performing Terraform migrations on cluster upgrades.
|
||||
type TerraformUpgrader struct {
|
||||
tf tfClient
|
||||
tf tfResourceClient
|
||||
policyPatcher policyPatcher
|
||||
outWriter io.Writer
|
||||
fileHandler file.Handler
|
||||
|
|
@ -51,15 +51,14 @@ type TerraformUpgradeOptions struct {
|
|||
}
|
||||
|
||||
// CheckTerraformMigrations checks whether Terraform migrations are possible in the current workspace.
|
||||
// If the files that will be written during the upgrade already exist, it returns an error.
|
||||
func (u *TerraformUpgrader) CheckTerraformMigrations(upgradeID string) error {
|
||||
func checkTerraformMigrations(file file.Handler, upgradeID, upgradeSubDir string) error {
|
||||
var existingFiles []string
|
||||
filesToCheck := []string{
|
||||
filepath.Join(constants.UpgradeDir, upgradeID, constants.TerraformUpgradeBackupDir),
|
||||
filepath.Join(constants.UpgradeDir, upgradeID, upgradeSubDir),
|
||||
}
|
||||
|
||||
for _, f := range filesToCheck {
|
||||
if err := checkFileExists(u.fileHandler, &existingFiles, f); err != nil {
|
||||
if err := checkFileExists(file, &existingFiles, f); err != nil {
|
||||
return fmt.Errorf("checking terraform migrations: %w", err)
|
||||
}
|
||||
}
|
||||
|
|
@ -70,6 +69,12 @@ func (u *TerraformUpgrader) CheckTerraformMigrations(upgradeID string) error {
|
|||
return nil
|
||||
}
|
||||
|
||||
// CheckTerraformMigrations checks whether Terraform migrations are possible in the current workspace.
|
||||
// If the files that will be written during the upgrade already exist, it returns an error.
|
||||
func (u *TerraformUpgrader) CheckTerraformMigrations(upgradeID, upgradeSubDir string) error {
|
||||
return checkTerraformMigrations(u.fileHandler, upgradeID, upgradeSubDir)
|
||||
}
|
||||
|
||||
// checkFileExists checks whether a file exists and adds it to the existingFiles slice if it does.
|
||||
func checkFileExists(fileHandler file.Handler, existingFiles *[]string, filename string) error {
|
||||
_, err := fileHandler.Stat(filename)
|
||||
|
|
@ -125,12 +130,17 @@ func (u *TerraformUpgrader) PlanTerraformMigrations(ctx context.Context, opts Te
|
|||
// CleanUpTerraformMigrations cleans up the Terraform migration workspace, for example when an upgrade is
|
||||
// aborted by the user.
|
||||
func (u *TerraformUpgrader) CleanUpTerraformMigrations(upgradeID string) error {
|
||||
return CleanUpTerraformMigrations(upgradeID, u.fileHandler)
|
||||
}
|
||||
|
||||
// CleanUpTerraformMigrations cleans up the Terraform upgrade directory.
|
||||
func CleanUpTerraformMigrations(upgradeID string, fileHandler file.Handler) error {
|
||||
cleanupFiles := []string{
|
||||
filepath.Join(constants.UpgradeDir, upgradeID),
|
||||
}
|
||||
|
||||
for _, f := range cleanupFiles {
|
||||
if err := u.fileHandler.RemoveAll(f); err != nil {
|
||||
if err := fileHandler.RemoveAll(f); err != nil {
|
||||
return fmt.Errorf("cleaning up file %s: %w", f, err)
|
||||
}
|
||||
}
|
||||
|
|
@ -138,7 +148,7 @@ func (u *TerraformUpgrader) CleanUpTerraformMigrations(upgradeID string) error {
|
|||
return nil
|
||||
}
|
||||
|
||||
// ApplyTerraformMigrations applies the migerations planned by PlanTerraformMigrations.
|
||||
// ApplyTerraformMigrations applies the migrations planned by PlanTerraformMigrations.
|
||||
// If PlanTerraformMigrations has not been executed before, it will return an error.
|
||||
// In case of a successful upgrade, the output will be written to the specified file and the old Terraform directory is replaced
|
||||
// By the new one.
|
||||
|
|
@ -197,12 +207,22 @@ func (u *TerraformUpgrader) mergeClusterIDFile(migrationOutput clusterid.File) e
|
|||
return nil
|
||||
}
|
||||
|
||||
// a tfClient performs the Terraform interactions in an upgrade.
|
||||
type tfClient interface {
|
||||
PrepareUpgradeWorkspace(path, oldWorkingDir, newWorkingDir, upgradeID string, vars terraform.Variables) error
|
||||
type tfClientCommon interface {
|
||||
ShowPlan(ctx context.Context, logLevel terraform.LogLevel, planFilePath string, output io.Writer) error
|
||||
Plan(ctx context.Context, logLevel terraform.LogLevel, planFile string) (bool, error)
|
||||
}
|
||||
|
||||
// tfResourceClient is a Terraform client for managing cluster resources.
|
||||
type tfResourceClient interface {
|
||||
PrepareUpgradeWorkspace(path, oldWorkingDir, newWorkingDir, backupDir string, vars terraform.Variables) error
|
||||
CreateCluster(ctx context.Context, logLevel terraform.LogLevel) (terraform.ApplyOutput, error)
|
||||
tfClientCommon
|
||||
}
|
||||
|
||||
// tfIAMClient is a Terraform client for managing IAM resources.
|
||||
type tfIAMClient interface {
|
||||
ApplyIAMConfig(ctx context.Context, csp cloudprovider.Provider, logLevel terraform.LogLevel) (terraform.IAMOutput, error)
|
||||
tfClientCommon
|
||||
}
|
||||
|
||||
// policyPatcher interacts with the CSP (currently only applies for Azure) to update the attestation policy.
|
||||
|
|
|
|||
|
|
@ -58,7 +58,7 @@ func TestCheckTerraformMigrations(t *testing.T) {
|
|||
for name, tc := range testCases {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
u := upgrader(tc.workspace)
|
||||
err := u.CheckTerraformMigrations(tc.upgradeID)
|
||||
err := u.CheckTerraformMigrations(tc.upgradeID, constants.TerraformUpgradeBackupDir)
|
||||
if tc.wantErr {
|
||||
require.Error(t, err)
|
||||
return
|
||||
|
|
@ -70,7 +70,7 @@ func TestCheckTerraformMigrations(t *testing.T) {
|
|||
}
|
||||
|
||||
func TestPlanTerraformMigrations(t *testing.T) {
|
||||
upgrader := func(tf tfClient, fileHandler file.Handler) *TerraformUpgrader {
|
||||
upgrader := func(tf tfResourceClient, fileHandler file.Handler) *TerraformUpgrader {
|
||||
u, err := NewTerraformUpgrader(tf, bytes.NewBuffer(nil), fileHandler)
|
||||
require.NoError(t, err)
|
||||
|
||||
|
|
@ -87,7 +87,7 @@ func TestPlanTerraformMigrations(t *testing.T) {
|
|||
|
||||
testCases := map[string]struct {
|
||||
upgradeID string
|
||||
tf tfClient
|
||||
tf tfResourceClient
|
||||
workspace file.Handler
|
||||
want bool
|
||||
wantErr bool
|
||||
|
|
@ -174,7 +174,7 @@ func TestPlanTerraformMigrations(t *testing.T) {
|
|||
}
|
||||
|
||||
func TestApplyTerraformMigrations(t *testing.T) {
|
||||
upgrader := func(tf tfClient, fileHandler file.Handler) *TerraformUpgrader {
|
||||
upgrader := func(tf tfResourceClient, fileHandler file.Handler) *TerraformUpgrader {
|
||||
u, err := NewTerraformUpgrader(tf, bytes.NewBuffer(nil), fileHandler)
|
||||
require.NoError(t, err)
|
||||
|
||||
|
|
@ -197,7 +197,7 @@ func TestApplyTerraformMigrations(t *testing.T) {
|
|||
|
||||
testCases := map[string]struct {
|
||||
upgradeID string
|
||||
tf tfClient
|
||||
tf tfResourceClient
|
||||
policyPatcher stubPolicyPatcher
|
||||
fs file.Handler
|
||||
skipIDFileCreation bool // if true, do not create the constellation-id.json file
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue