cli: add iam upgrade apply (#2132)

* add new iam upgrade apply

* remove iam tf plan from upgrade apply check

* add iam migration warning to upgrade apply

* update release process

* document migration

* Apply suggestions from code review

Co-authored-by: Otto Bittner <cobittner@posteo.net>

* add iam upgrade

* remove upgrade dir check in test

* ask only without --yes

* make iam upgrade provider specific

* test without seperate logins

* remove csi and only add conditionally

* Revert "test without seperate logins"

This reverts commit 05a12e59c9.

* fix msising cred

* support iam migration for all csps

* add iam upgrade label

---------

Co-authored-by: Otto Bittner <cobittner@posteo.net>

.github/actions/constellation_iam_upgrade/action.yml

@@ -0,0 +1,9 @@
+name: Constellation IAM upgrade
+description: Upgrade IAM configuration for a Constellation cluster.
+runs:
+  using: "composite"
+  steps:
+    - name: Constellation iam upgrade aws
+      shell: bash
+      run: |
+        constellation iam upgrade apply --yes --debug

.github/release.yml

@@ -7,6 +7,7 @@ changelog:
     - title: 🛠 Breaking changes
       labels:
         - breaking change
+        - iam upgrade
     - title: 🎁 New features
       labels:
         - feature

.github/workflows/e2e-upgrade.yml

@@ -163,6 +163,54 @@ jobs:
           outputPath: "build/constellation"
           push: true
 
+      - name: Login to GCP (IAM service account)
+        if: inputs.cloudProvider == 'gcp'
+        uses: ./.github/actions/login_gcp
+        with:
+          service_account: "constellation-iam-e2e@constellation-331613.iam.gserviceaccount.com"
+
+      - name: Login to AWS (IAM role)
+        if: inputs.cloudProvider == 'aws'
+        uses: aws-actions/configure-aws-credentials@5fd3084fc36e372ff1fff382a39b10d03659f355 # v2.2.0
+        with:
+          role-to-assume: arn:aws:iam::795746500882:role/GithubActionsE2EIAM
+          aws-region: eu-central-1
+          # extend token expiry to 6 hours to ensure constellation can terminate
+          role-duration-seconds: 21600
+
+      - name: Login to Azure (IAM service principal)
+        if: inputs.cloudProvider == 'azure'
+        uses: ./.github/actions/login_azure
+        with:
+          azure_credentials: ${{ secrets.AZURE_E2E_IAM_CREDENTIALS }}
+
+
+          ## IAM upgrade
+      - name: Upgrade IAM configuration
+        id: constellation-iam-upgrade
+        uses: ./.github/actions/constellation_iam_upgrade
+
+      - name: Login to GCP (Cluster service account)
+        if: inputs.cloudProvider == 'gcp'
+        uses: ./.github/actions/login_gcp
+        with:
+          service_account: "constellation-e2e-cluster@constellation-331613.iam.gserviceaccount.com"
+
+      - name: Login to AWS (Cluster role)
+        if: inputs.cloudProvider == 'aws'
+        uses: aws-actions/configure-aws-credentials@5fd3084fc36e372ff1fff382a39b10d03659f355 # v2.2.0
+        with:
+          role-to-assume: arn:aws:iam::795746500882:role/GithubActionsE2ECluster
+          aws-region: eu-central-1
+          # extend token expiry to 6 hours to ensure constellation can terminate
+          role-duration-seconds: 21600
+
+      - name: Login to Azure (Cluster service principal)
+        if: inputs.cloudProvider == 'azure'
+        uses: ./.github/actions/login_azure
+        with:
+          azure_credentials: ${{ secrets.AZURE_E2E_CLUSTER_CREDENTIALS }}
+
       - name: Run upgrade test
         env:
           KUBECONFIG: ${{ steps.e2e_test.outputs.kubeconfig }}

cli/internal/cmd/BUILD.bazel

@@ -15,6 +15,7 @@ go_library(
         "create.go",
         "iamcreate.go",
         "iamdestroy.go",
+        "iamupgradeapply.go",
         "init.go",
         "log.go",
         "manualtfstatemigration.go",
@@ -82,6 +83,7 @@ go_library(
         "//internal/versions",
         "//operators/constellation-node-operator/api/v1alpha1",
         "//verify/verifyproto",
+        "@com_github_google_uuid//:uuid",
         "@com_github_mattn_go_isatty//:go-isatty",
         "@com_github_siderolabs_talos_pkg_machinery//config/encoder",
         "@com_github_spf13_afero//:afero",

cli/internal/cmd/iamcreate.go

@@ -44,7 +44,7 @@ func NewIAMCmd() *cobra.Command {
 
 	cmd.AddCommand(newIAMCreateCmd())
 	cmd.AddCommand(newIAMDestroyCmd())
-
+	cmd.AddCommand(newIAMUpgradeCmd())
 	return cmd
 }
 

cli/internal/cmd/iamupgradeapply.go

@@ -0,0 +1,99 @@
+/*
+Copyright (c) Edgeless Systems GmbH
+SPDX-License-Identifier: AGPL-3.0-only
+*/
+
+package cmd
+
+import (
+	"errors"
+	"fmt"
+	"strings"
+	"time"
+
+	"github.com/edgelesssys/constellation/v2/cli/internal/terraform"
+	"github.com/edgelesssys/constellation/v2/cli/internal/upgrade"
+	"github.com/edgelesssys/constellation/v2/internal/api/attestationconfigapi"
+	"github.com/edgelesssys/constellation/v2/internal/cloud/cloudprovider"
+	"github.com/edgelesssys/constellation/v2/internal/config"
+	"github.com/edgelesssys/constellation/v2/internal/file"
+	"github.com/google/uuid"
+	"github.com/spf13/afero"
+	"github.com/spf13/cobra"
+)
+
+func upgradeRequiresIAMMigration(provider cloudprovider.Provider) bool {
+	switch provider {
+	case cloudprovider.AWS:
+		return true // needs to be set on every release. Can we automate this?
+	default:
+		return false
+	}
+}
+
+func newIAMUpgradeCmd() *cobra.Command {
+	cmd := &cobra.Command{
+		Use:   "upgrade",
+		Short: "Find and apply upgrades to your IAM profile",
+		Long:  "Find and apply upgrades to your IAM profile.",
+		Args:  cobra.ExactArgs(0),
+	}
+	cmd.AddCommand(newIAMUpgradeApplyCmd())
+	return cmd
+}
+
+func newIAMUpgradeApplyCmd() *cobra.Command {
+	cmd := &cobra.Command{
+		Use:   "apply",
+		Short: "Apply an upgrade to an IAM profile",
+		Long:  "Apply an upgrade to an IAM profile.",
+		Args:  cobra.NoArgs,
+		RunE:  runIAMUpgradeApply,
+	}
+	cmd.Flags().BoolP("yes", "y", false, "run upgrades without further confirmation\n")
+	return cmd
+}
+
+func runIAMUpgradeApply(cmd *cobra.Command, _ []string) error {
+	configPath, err := cmd.Flags().GetString("config")
+	if err != nil {
+		return err
+	}
+
+	force, err := cmd.Flags().GetBool("force")
+	if err != nil {
+		return fmt.Errorf("parsing force argument: %w", err)
+	}
+	fileHandler := file.NewHandler(afero.NewOsFs())
+	configFetcher := attestationconfigapi.NewFetcher()
+	conf, err := config.New(fileHandler, configPath, configFetcher, force)
+	var configValidationErr *config.ValidationError
+	if errors.As(err, &configValidationErr) {
+		cmd.PrintErrln(configValidationErr.LongMessage())
+	}
+	if err != nil {
+		return err
+	}
+	upgradeID := "iam-" + time.Now().Format("20060102150405") + "-" + strings.Split(uuid.New().String(), "-")[0]
+	iamMigrateCmd, err := upgrade.NewIAMMigrateCmd(cmd.Context(), upgradeID, conf.GetProvider(), terraform.LogLevelDebug)
+	if err != nil {
+		return fmt.Errorf("setting up IAM migration command: %w", err)
+	}
+
+	log, err := newCLILogger(cmd)
+	if err != nil {
+		return fmt.Errorf("setting up logger: %w", err)
+	}
+	migrator := &tfMigrationClient{log}
+
+	yes, err := cmd.Flags().GetBool("yes")
+	if err != nil {
+		return err
+	}
+	err = migrator.applyMigration(cmd, file.NewHandler(afero.NewOsFs()), iamMigrateCmd, yes)
+	if err != nil {
+		return fmt.Errorf("applying IAM migration: %w", err)
+	}
+	cmd.Println("IAM profile successfully applied.")
+	return nil
+}

cli/internal/cmd/tfmigrationclient.go

@@ -35,15 +35,15 @@ func (u *tfMigrationClient) planMigration(cmd *cobra.Command, file file.Handler,
 
 // applyMigration plans and then applies the Terraform migration. The user is asked for confirmation if there are any changes.
 // adapted from migrateTerraform().
-func (u *tfMigrationClient) applyMigration(cmd *cobra.Command, file file.Handler, migrateCmd upgrade.TfMigrationCmd, flags upgradeApplyFlags) error {
+func (u *tfMigrationClient) applyMigration(cmd *cobra.Command, file file.Handler, migrateCmd upgrade.TfMigrationCmd, yesFlag bool) error {
 	hasDiff, err := u.planMigration(cmd, file, migrateCmd)
 	if err != nil {
 		return fmt.Errorf("planning terraform migrations: %w", err)
 	}
 	if hasDiff {
 		// If there are any Terraform migrations to apply, ask for confirmation
-		fmt.Fprintf(cmd.OutOrStdout(), "The %s upgrade requires a migration of Constellation cloud resources by applying an updated Terraform template. Please manually review the suggested changes below.\n", migrateCmd.String())
-		if !flags.yes {
+		fmt.Fprintf(cmd.OutOrStdout(), "The %s upgrade requires a migration by applying an updated Terraform template. Please manually review the suggested changes below.\n", migrateCmd.String())
+		if !yesFlag {
 			ok, err := askToConfirm(cmd, fmt.Sprintf("Do you want to apply the %s?", migrateCmd.String()))
 			if err != nil {
 				return fmt.Errorf("asking for confirmation: %w", err)

cli/internal/cmd/upgradeapply.go

@@ -60,7 +60,6 @@ func runUpgradeApply(cmd *cobra.Command, _ []string) error {
 		return fmt.Errorf("creating logger: %w", err)
 	}
 	defer log.Sync()
-
 	fileHandler := file.NewHandler(afero.NewOsFs())
 	upgrader, err := kubernetes.NewUpgrader(cmd.Context(), cmd.OutOrStdout(), fileHandler, log, kubernetes.UpgradeCmdKindApply)
 	if err != nil {
@@ -70,17 +69,15 @@ func runUpgradeApply(cmd *cobra.Command, _ []string) error {
 	imagefetcher := imagefetcher.New()
 	configFetcher := attestationconfigapi.NewFetcher()
 
-	applyCmd := upgradeApplyCmd{upgrader: upgrader, log: log, imageFetcher: imagefetcher, configFetcher: configFetcher, migrationExecutor: &tfMigrationClient{log}}
+	applyCmd := upgradeApplyCmd{upgrader: upgrader, log: log, imageFetcher: imagefetcher, configFetcher: configFetcher}
 	return applyCmd.upgradeApply(cmd, fileHandler)
 }
 
 type upgradeApplyCmd struct {
-	upgrader          cloudUpgrader
-	imageFetcher      imageFetcher
-	configFetcher     attestationconfigapi.Fetcher
-	log               debugLog
-	migrationExecutor tfMigrationApplier
-	migrationCmds     []upgrade.TfMigrationCmd
+	upgrader      cloudUpgrader
+	imageFetcher  imageFetcher
+	configFetcher attestationconfigapi.Fetcher
+	log           debugLog
 }
 
 func (u *upgradeApplyCmd) upgradeApply(cmd *cobra.Command, fileHandler file.Handler) error {
@@ -88,6 +85,7 @@ func (u *upgradeApplyCmd) upgradeApply(cmd *cobra.Command, fileHandler file.Hand
 	if err != nil {
 		return fmt.Errorf("parsing flags: %w", err)
 	}
+
 	conf, err := config.New(fileHandler, flags.configPath, u.configFetcher, flags.force)
 	var configValidationErr *config.ValidationError
 	if errors.As(err, &configValidationErr) {
@@ -96,6 +94,19 @@ func (u *upgradeApplyCmd) upgradeApply(cmd *cobra.Command, fileHandler file.Hand
 	if err != nil {
 		return err
 	}
+	if upgradeRequiresIAMMigration(conf.GetProvider()) {
+		cmd.Println("WARNING: This upgrade requires an IAM migration. Please make sure you have applied the IAM migration using `iam upgrade apply` before continuing.")
+		if !flags.yes {
+			yes, err := askToConfirm(cmd, "Did you upgrade the IAM resources?")
+			if err != nil {
+				return fmt.Errorf("asking for confirmation: %w", err)
+			}
+			if !yes {
+				cmd.Println("Skipping upgrade.")
+				return nil
+			}
+		}
+	}
 
 	if err := handleInvalidK8sPatchVersion(cmd, conf.KubernetesVersion, flags.yes); err != nil {
 		return err
@@ -111,11 +122,6 @@ func (u *upgradeApplyCmd) upgradeApply(cmd *cobra.Command, fileHandler file.Hand
 	if err := u.upgradeAttestConfigIfDiff(cmd, conf.GetAttestationConfig(), flags); err != nil {
 		return fmt.Errorf("upgrading measurements: %w", err)
 	}
-	for _, migrationCmd := range u.migrationCmds {
-		if err := u.migrationExecutor.applyMigration(cmd, fileHandler, migrationCmd, flags); err != nil {
-			return fmt.Errorf("executing %s migration: %w", migrationCmd.String(), err)
-		}
-	}
 	// not moving existing Terraform migrator because of planned apply refactor
 	if err := u.migrateTerraform(cmd, u.imageFetcher, conf, flags); err != nil {
 		return fmt.Errorf("performing Terraform migrations: %w", err)
@@ -379,7 +385,3 @@ type cloudUpgrader interface {
 	CleanUpTerraformMigrations() error
 	AddManualStateMigration(migration terraform.StateMigration)
 }
-
-type tfMigrationApplier interface {
-	applyMigration(cmd *cobra.Command, file file.Handler, migrateCmd upgrade.TfMigrationCmd, flags upgradeApplyFlags) error
-}

cli/internal/cmd/upgradeapply_test.go

@@ -24,7 +24,6 @@ import (
 	"github.com/edgelesssys/constellation/v2/internal/file"
 	"github.com/edgelesssys/constellation/v2/internal/logger"
 	"github.com/spf13/afero"
-	"github.com/spf13/cobra"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 	corev1 "k8s.io/api/core/v1"
@@ -144,7 +143,7 @@ func TestUpgradeApply(t *testing.T) {
 			require.NoError(handler.WriteYAML(constants.ConfigFilename, cfg))
 			require.NoError(handler.WriteJSON(constants.ClusterIDsFileName, clusterid.File{}))
 
-			upgrader := upgradeApplyCmd{upgrader: tc.upgrader, log: logger.NewTest(t), imageFetcher: tc.fetcher, configFetcher: stubAttestationFetcher{}, migrationExecutor: &migrationExecutorPlaceholder{}}
+			upgrader := upgradeApplyCmd{upgrader: tc.upgrader, log: logger.NewTest(t), imageFetcher: tc.fetcher, configFetcher: stubAttestationFetcher{}}
 
 			err := upgrader.upgradeApply(cmd, handler)
 			if tc.wantErr {
@@ -156,12 +155,6 @@ func TestUpgradeApply(t *testing.T) {
 	}
 }
 
-type migrationExecutorPlaceholder struct{}
-
-func (d *migrationExecutorPlaceholder) applyMigration(_ *cobra.Command, _ file.Handler, _ upgrade.TfMigrationCmd, _ upgradeApplyFlags) error {
-	return nil
-}
-
 type stubUpgrader struct {
 	currentConfig     config.AttestationCfg
 	nodeVersionErr    error

cli/internal/cmd/upgradecheck.go

@@ -78,10 +78,6 @@ func runUpgradeCheck(cmd *cobra.Command, _ []string) error {
 	if err != nil {
 		return fmt.Errorf("constructing Rekor client: %w", err)
 	}
-	iamMigrateCmd, err := upgrade.NewIAMMigrateCmd(cmd.Context(), checker.GetUpgradeID(), cloudprovider.AWS, terraform.LogLevelDebug)
-	if err != nil {
-		return fmt.Errorf("setting up IAM migration command: %w", err)
-	}
 	up := &upgradeCheckCmd{
 		canUpgradeCheck: featureset.CanUpgradeCheck,
 		collect: &versionCollector{
@@ -97,11 +93,9 @@ func runUpgradeCheck(cmd *cobra.Command, _ []string) error {
 			log:            log,
 			versionsapi:    versionfetcher,
 		},
-		checker:       checker,
-		imagefetcher:  imagefetcher.New(),
-		log:           log,
-		iamMigrateCmd: iamMigrateCmd,
-		planExecutor:  &tfMigrationClient{log},
+		checker:      checker,
+		imagefetcher: imagefetcher.New(),
+		log:          log,
 	}
 
 	return up.upgradeCheck(cmd, fileHandler, attestationconfigapi.NewFetcher(), flags)
@@ -148,18 +142,12 @@ func parseUpgradeCheckFlags(cmd *cobra.Command) (upgradeCheckFlags, error) {
 	}, nil
 }
 
-type tfPlanner interface {
-	planMigration(cmd *cobra.Command, file file.Handler, migrateCmd upgrade.TfMigrationCmd) (hasDiff bool, err error)
-}
-
 type upgradeCheckCmd struct {
 	canUpgradeCheck bool
 	collect         collector
 	checker         upgradeChecker
 	imagefetcher    imageFetcher
 	log             debugLog
-	iamMigrateCmd   upgrade.TfMigrationCmd
-	planExecutor    tfPlanner
 }
 
 // upgradePlan plans an upgrade of a Constellation cluster.
@@ -216,18 +204,6 @@ func (u *upgradeCheckCmd) upgradeCheck(cmd *cobra.Command, fileHandler file.Hand
 		return err
 	}
 
-	cmd.Println("The following IAM migrations are available with this CLI:")
-	u.log.Debugf("Planning IAM migrations")
-	if u.iamMigrateCmd != nil {
-		hasIAMDiff, err := u.planExecutor.planMigration(cmd, fileHandler, u.iamMigrateCmd)
-		if err != nil {
-			return fmt.Errorf("planning IAM migration: %w", err)
-		}
-		if !hasIAMDiff {
-			cmd.Println("  No IAM migrations are available.")
-		}
-	}
-
 	u.log.Debugf("Planning Terraform migrations")
 	if err := u.checker.CheckTerraformMigrations(); err != nil {
 		return fmt.Errorf("checking workspace: %w", err)

cli/internal/upgrade/iammigrate.go

@@ -28,7 +28,7 @@ type TfMigrationCmd interface {
 	UpgradeID() string
 }
 
-// IAMMigrateCmd is a terraform migration command for IAM.
+// IAMMigrateCmd is a terraform migration command for IAM. Which is used for the tfMigrationClient.
 type IAMMigrateCmd struct {
 	tf        tfIAMClient
 	upgradeID string

dev-docs/workflows/release.md

@@ -6,6 +6,8 @@ This checklist will prepare `v1.3.0` from `v1.2.0` (minor release) or `v1.3.1` f
 
 1. Search the code for TODOs and FIXMEs that should be resolved before releasing.
 2. [Update titles and labels for all PRs relevant for this release](/dev-docs/conventions.md#pr-conventions) to aid in the [changelog generation](/.github/release.yml).
+3. Check PRs for the label `iam upgrade`. If there is any, update `upgradeRequiresIAMMigration`  in `iamupgradeapply.go`. This ensures the CLI issues a warning on `upgrade apply` to run `iam upgrade apply` before upgrading the cluster.
+
 
 
 ## Automated release

docs/docs/reference/cli.md

@@ -35,6 +35,8 @@ Commands:
     * [azure](#constellation-iam-create-azure): Create IAM configuration on Microsoft Azure for your Constellation cluster
     * [gcp](#constellation-iam-create-gcp): Create IAM configuration on GCP for your Constellation cluster
   * [destroy](#constellation-iam-destroy): Destroy an IAM configuration and delete local Terraform files
+  * [upgrade](#constellation-iam-upgrade): Find and apply upgrades to your IAM profile
+    * [apply](#constellation-iam-upgrade-apply): Apply an upgrade to an IAM profile
 * [version](#constellation-version): Display version of this CLI
 
 ## constellation config
@@ -725,6 +727,58 @@ constellation iam destroy [flags]
       --tf-log string   Terraform log level (default "NONE")
 ```
 
+## constellation iam upgrade
+
+Find and apply upgrades to your IAM profile
+
+### Synopsis
+
+Find and apply upgrades to your IAM profile.
+
+### Options
+
+```
+  -h, --help   help for upgrade
+```
+
+### Options inherited from parent commands
+
+```
+      --config string   path to the configuration file (default "constellation-conf.yaml")
+      --debug           enable debug logging
+      --force           disable version compatibility checks - might result in corrupted clusters
+      --tf-log string   Terraform log level (default "NONE")
+```
+
+## constellation iam upgrade apply
+
+Apply an upgrade to an IAM profile
+
+### Synopsis
+
+Apply an upgrade to an IAM profile.
+
+```
+constellation iam upgrade apply [flags]
+```
+
+### Options
+
+```
+  -h, --help   help for apply
+  -y, --yes    run upgrades without further confirmation
+               
+```
+
+### Options inherited from parent commands
+
+```
+      --config string   path to the configuration file (default "constellation-conf.yaml")
+      --debug           enable debug logging
+      --force           disable version compatibility checks - might result in corrupted clusters
+      --tf-log string   Terraform log level (default "NONE")
+```
+
 ## constellation version
 
 Display version of this CLI

docs/docs/reference/migration.md

@@ -13,6 +13,11 @@ Use [`constellation config migrate`](./cli.md#constellation-config-migrate) to a
   2. Set `useManagedIdentityExtension` to `true`  and use the `userAssignedIdentity` from the Constellation config for the value of `userAssignedIdentityID`.
   3. Restart the CSI driver, cloud controller manager, cluster autoscaler, and Constellation operator pods.
 
+
+## Migrating from CLI versions before 2.10
+
+- AWS cluster upgrades require additional IAM permissions for the newly introduced `aws-load-balancer-controller`. Please upgrade your IAM roles using `iam upgrade apply`. This will show necessary changes and apply them, if desired.
+
 ## Migrating from CLI versions before 2.9
 
 - The `provider.azure.appClientID` and `provider.azure.clientSecretValue` fields were removed to enforce migration to managed identity authentication

e2e/internal/upgrade/upgrade_test.go

@@ -128,10 +128,6 @@ func setup() error {
 	if _, err := getCLIPath(*cliPath); err != nil {
 		return fmt.Errorf("getting CLI path: %w", err)
 	}
-	if _, err := os.Stat(constants.UpgradeDir); err == nil {
-		return fmt.Errorf("please remove the existing %s folder", constants.UpgradeDir)
-	}
-
 	return nil
 }