mirror of
https://github.com/edgelesssys/constellation.git
synced 2025-08-06 05:54:28 -04:00
cli: use state file on init and upgrade (#2395)
* [wip] use state file in CLI Signed-off-by: Moritz Sanft <58110325+msanft@users.noreply.github.com> tidy Signed-off-by: Moritz Sanft <58110325+msanft@users.noreply.github.com> * use state file in CLI Signed-off-by: Moritz Sanft <58110325+msanft@users.noreply.github.com> take clusterConfig from IDFile for compat Signed-off-by: Moritz Sanft <58110325+msanft@users.noreply.github.com> various fixes Signed-off-by: Moritz Sanft <58110325+msanft@users.noreply.github.com> wip Signed-off-by: Moritz Sanft <58110325+msanft@users.noreply.github.com> * add GCP-specific values in Helm loader test Signed-off-by: Moritz Sanft <58110325+msanft@users.noreply.github.com> * remove unnecessary pointer Signed-off-by: Moritz Sanft <58110325+msanft@users.noreply.github.com> * write ClusterValues in one step Signed-off-by: Moritz Sanft <58110325+msanft@users.noreply.github.com> * move stub to test file Signed-off-by: Moritz Sanft <58110325+msanft@users.noreply.github.com> * remove mention of id-file Signed-off-by: Moritz Sanft <58110325+msanft@users.noreply.github.com> * move output to `migrateTerraform` Signed-off-by: Moritz Sanft <58110325+msanft@users.noreply.github.com> * unconditional assignments converting from idFile Signed-off-by: Moritz Sanft <58110325+msanft@users.noreply.github.com> * move require block in go modules file Signed-off-by: Moritz Sanft <58110325+msanft@users.noreply.github.com> * fall back to id file on upgrade Signed-off-by: Moritz Sanft <58110325+msanft@users.noreply.github.com> * tidy Signed-off-by: Moritz Sanft <58110325+msanft@users.noreply.github.com> * fix linter check Signed-off-by: Moritz Sanft <58110325+msanft@users.noreply.github.com> * add notice to remove Terraform state check on manual migration Signed-off-by: Moritz Sanft <58110325+msanft@users.noreply.github.com> * add `name` field Signed-off-by: Moritz Sanft <58110325+msanft@users.noreply.github.com> fix name tests Signed-off-by: Moritz Sanft <58110325+msanft@users.noreply.github.com> * return early if no Terraform diff Signed-off-by: Moritz Sanft <58110325+msanft@users.noreply.github.com> * tidy Signed-off-by: Moritz Sanft <58110325+msanft@users.noreply.github.com> * return infrastructure state even if no diff exists Signed-off-by: Moritz Sanft <58110325+msanft@users.noreply.github.com> * add TODO to remove comment Signed-off-by: Moritz Sanft <58110325+msanft@users.noreply.github.com> * use state-file in miniconstellation Signed-off-by: Moritz Sanft <58110325+msanft@users.noreply.github.com> * cli: remove id-file (#2402) * remove id-file from `constellation create` Signed-off-by: Moritz Sanft <58110325+msanft@users.noreply.github.com> * add file renaming to handler * rename id-file after upgrade * use idFile on `constellation init` Signed-off-by: Moritz Sanft <58110325+msanft@users.noreply.github.com> * remove id-file from `constellation verify` Signed-off-by: Moritz Sanft <58110325+msanft@users.noreply.github.com> * linter fixes Signed-off-by: Moritz Sanft <58110325+msanft@users.noreply.github.com> * remove id-file from `constellation mini` * remove id-file from `constellation recover` * linter fixes * remove id-file from `constellation terminate` * fix initSecret type * fix recover argument precedence * fix terminate test * generate * add TODO to remove id-file removal * Update cli/internal/cmd/init.go Co-authored-by: Adrian Stobbe <stobbe.adrian@gmail.com> * fix verify arg parse logic Signed-off-by: Moritz Sanft <58110325+msanft@users.noreply.github.com> * add version test Signed-off-by: Moritz Sanft <58110325+msanft@users.noreply.github.com> * remove id-file from docs * add file not found log * use state-file in miniconstellation Signed-off-by: Moritz Sanft <58110325+msanft@users.noreply.github.com> * remove id-file from `constellation iam destroy` Signed-off-by: Moritz Sanft <58110325+msanft@users.noreply.github.com> * remove id-file from `cdbg deploy` Signed-off-by: Moritz Sanft <58110325+msanft@users.noreply.github.com> --------- Signed-off-by: Moritz Sanft <58110325+msanft@users.noreply.github.com> Co-authored-by: Adrian Stobbe <stobbe.adrian@gmail.com> * use state-file in CI Signed-off-by: Moritz Sanft <58110325+msanft@users.noreply.github.com> * update orchestration docs --------- Signed-off-by: Moritz Sanft <58110325+msanft@users.noreply.github.com> Co-authored-by: Adrian Stobbe <stobbe.adrian@gmail.com>
This commit is contained in:
Moritz Sanft
GitHub
parent
dbf40d185c
commit
005e865a13
51 changed files with 1189 additions and 497 deletions
|
|
@ -11,6 +11,7 @@ import (
|
|||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/fs"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
|
@ -145,6 +146,10 @@ type upgradeApplyCmd struct {
|
|||
log debugLog
|
||||
}
|
||||
|
||||
type infrastructureShower interface {
|
||||
ShowInfrastructure(ctx context.Context, provider cloudprovider.Provider) (state.Infrastructure, error)
|
||||
}
|
||||
|
||||
func (u *upgradeApplyCmd) upgradeApply(cmd *cobra.Command, upgradeDir string, flags upgradeApplyFlags) error {
|
||||
conf, err := config.New(u.fileHandler, constants.ConfigFilename, u.configFetcher, flags.force)
|
||||
var configValidationErr *config.ValidationError
|
||||
|
|
@ -172,11 +177,24 @@ func (u *upgradeApplyCmd) upgradeApply(cmd *cobra.Command, upgradeDir string, fl
|
|||
return err
|
||||
}
|
||||
|
||||
var idFile clusterid.File
|
||||
if err := u.fileHandler.ReadJSON(constants.ClusterIDsFilename, &idFile); err != nil {
|
||||
return fmt.Errorf("reading cluster ID file: %w", err)
|
||||
stateFile, err := state.ReadFromFile(u.fileHandler, constants.StateFilename)
|
||||
// TODO(msanft): Remove reading from idFile once v2.12.0 is released and read from state file directly.
|
||||
// For now, this is only here to ensure upgradability from an id-file to a state file version.
|
||||
if errors.Is(err, fs.ErrNotExist) {
|
||||
u.log.Debugf("%s does not exist in current directory, falling back to reading from %s",
|
||||
constants.StateFilename, constants.ClusterIDsFilename)
|
||||
var idFile clusterid.File
|
||||
if err := u.fileHandler.ReadJSON(constants.ClusterIDsFilename, &idFile); err != nil {
|
||||
return fmt.Errorf("reading cluster ID file: %w", err)
|
||||
}
|
||||
// Convert id-file to state file
|
||||
stateFile = state.NewFromIDFile(idFile, conf)
|
||||
if stateFile.Infrastructure.Azure != nil {
|
||||
conf.UpdateMAAURL(stateFile.Infrastructure.Azure.AttestationURL)
|
||||
}
|
||||
} else if err != nil {
|
||||
return fmt.Errorf("reading state file: %w", err)
|
||||
}
|
||||
conf.UpdateMAAURL(idFile.AttestationURL)
|
||||
|
||||
// Apply migrations necessary for the upgrade
|
||||
if err := migrateFrom2_10(cmd.Context(), u.kubeUpgrader); err != nil {
|
||||
|
|
@ -186,37 +204,55 @@ func (u *upgradeApplyCmd) upgradeApply(cmd *cobra.Command, upgradeDir string, fl
|
|||
return fmt.Errorf("applying migration for upgrading from v2.11: %w", err)
|
||||
}
|
||||
|
||||
if err := u.confirmAndUpgradeAttestationConfig(cmd, conf.GetAttestationConfig(), idFile.MeasurementSalt, flags); err != nil {
|
||||
if err := u.confirmAndUpgradeAttestationConfig(cmd, conf.GetAttestationConfig(), stateFile.ClusterValues.MeasurementSalt, flags); err != nil {
|
||||
return fmt.Errorf("upgrading measurements: %w", err)
|
||||
}
|
||||
|
||||
var infraState state.Infrastructure
|
||||
// If infrastructure phase is skipped, we expect the new infrastructure
|
||||
// to be in the Terraform configuration already. Otherwise, perform
|
||||
// the Terraform migrations.
|
||||
var postMigrationInfraState state.Infrastructure
|
||||
if flags.skipPhases.contains(skipInfrastructurePhase) {
|
||||
infraState, err = u.clusterShower.ShowInfrastructure(cmd.Context(), conf.GetProvider())
|
||||
// TODO(msanft): Once v2.12.0 is released, this should be removed and the state should be read
|
||||
// from the state file instead, as it will be the only source of truth for the cluster's infrastructure.
|
||||
postMigrationInfraState, err = u.clusterShower.ShowInfrastructure(cmd.Context(), conf.GetProvider())
|
||||
if err != nil {
|
||||
return fmt.Errorf("getting infra state: %w", err)
|
||||
return fmt.Errorf("getting Terraform state: %w", err)
|
||||
}
|
||||
} else {
|
||||
infraState, err = u.migrateTerraform(cmd, conf, upgradeDir, flags)
|
||||
postMigrationInfraState, err = u.migrateTerraform(cmd, conf, upgradeDir, flags)
|
||||
if err != nil {
|
||||
return fmt.Errorf("performing Terraform migrations: %w", err)
|
||||
}
|
||||
}
|
||||
// reload idFile after terraform migration
|
||||
// it might have been updated by the migration
|
||||
if err := u.fileHandler.ReadJSON(constants.ClusterIDsFilename, &idFile); err != nil {
|
||||
return fmt.Errorf("reading updated cluster ID file: %w", err)
|
||||
|
||||
// Merge the pre-upgrade state with the post-migration infrastructure values
|
||||
if _, err := stateFile.Merge(
|
||||
// temporary state with post-migration infrastructure values
|
||||
state.New().SetInfrastructure(postMigrationInfraState),
|
||||
); err != nil {
|
||||
return fmt.Errorf("merging pre-upgrade state with post-migration infrastructure values: %w", err)
|
||||
}
|
||||
state := state.NewState(infraState)
|
||||
// TODO(elchead): AB#3424 move this to updateClusterIDFile and correctly handle existing state when writing state
|
||||
if err := u.fileHandler.WriteYAML(constants.StateFilename, state, file.OptOverwrite); err != nil {
|
||||
|
||||
// Write the post-migration state to disk
|
||||
if err := stateFile.WriteToFile(u.fileHandler, constants.StateFilename); err != nil {
|
||||
return fmt.Errorf("writing state file: %w", err)
|
||||
}
|
||||
|
||||
// TODO(msanft): Remove this after v2.12.0 is released, as we do not support
|
||||
// the id-file starting from v2.13.0.
|
||||
err = u.fileHandler.RenameFile(constants.ClusterIDsFilename, constants.ClusterIDsFilename+".old")
|
||||
if !errors.Is(err, fs.ErrNotExist) && err != nil {
|
||||
return fmt.Errorf("removing cluster ID file: %w", err)
|
||||
}
|
||||
|
||||
// extend the clusterConfig cert SANs with any of the supported endpoints:
|
||||
// - (legacy) public IP
|
||||
// - fallback endpoint
|
||||
// - custom (user-provided) endpoint
|
||||
sans := append([]string{idFile.IP, conf.CustomEndpoint}, idFile.APIServerCertSANs...)
|
||||
// TODO(msanft): Remove the comment below once v2.12.0 is released.
|
||||
// At this point, state file and id-file should have been merged, so we can use the state file.
|
||||
sans := append([]string{stateFile.Infrastructure.ClusterEndpoint, conf.CustomEndpoint}, stateFile.Infrastructure.APIServerCertSANs...)
|
||||
if err := u.kubeUpgrader.ExtendClusterConfigCertSANs(cmd.Context(), sans); err != nil {
|
||||
return fmt.Errorf("extending cert SANs: %w", err)
|
||||
}
|
||||
|
|
@ -228,7 +264,7 @@ func (u *upgradeApplyCmd) upgradeApply(cmd *cobra.Command, upgradeDir string, fl
|
|||
|
||||
var upgradeErr *compatibility.InvalidUpgradeError
|
||||
if !flags.skipPhases.contains(skipHelmPhase) {
|
||||
err = u.handleServiceUpgrade(cmd, conf, idFile, infraState, upgradeDir, flags)
|
||||
err = u.handleServiceUpgrade(cmd, conf, stateFile, upgradeDir, flags)
|
||||
switch {
|
||||
case errors.As(err, &upgradeErr):
|
||||
cmd.PrintErrln(err)
|
||||
|
|
@ -269,14 +305,16 @@ func diffAttestationCfg(currentAttestationCfg config.AttestationCfg, newAttestat
|
|||
}
|
||||
|
||||
// migrateTerraform checks if the Constellation version the cluster is being upgraded to requires a migration
|
||||
// of cloud resources with Terraform. If so, the migration is performed.
|
||||
func (u *upgradeApplyCmd) migrateTerraform(cmd *cobra.Command, conf *config.Config, upgradeDir string, flags upgradeApplyFlags,
|
||||
) (res state.Infrastructure, err error) {
|
||||
// of cloud resources with Terraform. If so, the migration is performed and the post-migration infrastructure state is returned.
|
||||
// If no migration is required, the current (pre-upgrade) infrastructure state is returned.
|
||||
func (u *upgradeApplyCmd) migrateTerraform(
|
||||
cmd *cobra.Command, conf *config.Config, upgradeDir string, flags upgradeApplyFlags,
|
||||
) (state.Infrastructure, error) {
|
||||
u.log.Debugf("Planning Terraform migrations")
|
||||
|
||||
vars, err := cloudcmd.TerraformUpgradeVars(conf)
|
||||
if err != nil {
|
||||
return res, fmt.Errorf("parsing upgrade variables: %w", err)
|
||||
return state.Infrastructure{}, fmt.Errorf("parsing upgrade variables: %w", err)
|
||||
}
|
||||
u.log.Debugf("Using Terraform variables:\n%v", vars)
|
||||
|
||||
|
|
@ -292,60 +330,46 @@ func (u *upgradeApplyCmd) migrateTerraform(cmd *cobra.Command, conf *config.Conf
|
|||
|
||||
hasDiff, err := u.clusterUpgrader.PlanClusterUpgrade(cmd.Context(), cmd.OutOrStdout(), vars, conf.GetProvider())
|
||||
if err != nil {
|
||||
return res, fmt.Errorf("planning terraform migrations: %w", err)
|
||||
return state.Infrastructure{}, fmt.Errorf("planning terraform migrations: %w", err)
|
||||
}
|
||||
|
||||
if hasDiff {
|
||||
// If there are any Terraform migrations to apply, ask for confirmation
|
||||
fmt.Fprintln(cmd.OutOrStdout(), "The upgrade requires a migration of Constellation cloud resources by applying an updated Terraform template. Please manually review the suggested changes below.")
|
||||
if !flags.yes {
|
||||
ok, err := askToConfirm(cmd, "Do you want to apply the Terraform migrations?")
|
||||
if err != nil {
|
||||
return res, fmt.Errorf("asking for confirmation: %w", err)
|
||||
}
|
||||
if !ok {
|
||||
cmd.Println("Aborting upgrade.")
|
||||
// User doesn't expect to see any changes in his workspace after aborting an "upgrade apply",
|
||||
// therefore, roll back to the backed up state.
|
||||
if err := u.clusterUpgrader.RestoreClusterWorkspace(); err != nil {
|
||||
return res, fmt.Errorf(
|
||||
"restoring Terraform workspace: %w, restore the Terraform workspace manually from %s ",
|
||||
err,
|
||||
filepath.Join(upgradeDir, constants.TerraformUpgradeBackupDir),
|
||||
)
|
||||
}
|
||||
return res, fmt.Errorf("cluster upgrade aborted by user")
|
||||
}
|
||||
}
|
||||
u.log.Debugf("Applying Terraform migrations")
|
||||
infraState, err := u.clusterUpgrader.ApplyClusterUpgrade(cmd.Context(), conf.GetProvider())
|
||||
if err != nil {
|
||||
return infraState, fmt.Errorf("applying terraform migrations: %w", err)
|
||||
}
|
||||
|
||||
// Apply possible updates to cluster ID file
|
||||
if err := updateClusterIDFile(infraState, u.fileHandler); err != nil {
|
||||
return infraState, fmt.Errorf("merging cluster ID files: %w", err)
|
||||
}
|
||||
|
||||
cmd.Printf("Terraform migrations applied successfully and output written to: %s\n"+
|
||||
"A backup of the pre-upgrade state has been written to: %s\n",
|
||||
flags.pf.PrefixPrintablePath(constants.ClusterIDsFilename),
|
||||
flags.pf.PrefixPrintablePath(filepath.Join(upgradeDir, constants.TerraformUpgradeBackupDir)),
|
||||
)
|
||||
} else {
|
||||
if !hasDiff {
|
||||
u.log.Debugf("No Terraform diff detected")
|
||||
return u.clusterShower.ShowInfrastructure(cmd.Context(), conf.GetProvider())
|
||||
}
|
||||
u.log.Debugf("No Terraform diff detected")
|
||||
infraState, err := u.clusterShower.ShowInfrastructure(cmd.Context(), conf.GetProvider())
|
||||
|
||||
// If there are any Terraform migrations to apply, ask for confirmation
|
||||
fmt.Fprintln(cmd.OutOrStdout(), "The upgrade requires a migration of Constellation cloud resources by applying an updated Terraform template. Please manually review the suggested changes below.")
|
||||
if !flags.yes {
|
||||
ok, err := askToConfirm(cmd, "Do you want to apply the Terraform migrations?")
|
||||
if err != nil {
|
||||
return state.Infrastructure{}, fmt.Errorf("asking for confirmation: %w", err)
|
||||
}
|
||||
if !ok {
|
||||
cmd.Println("Aborting upgrade.")
|
||||
// User doesn't expect to see any changes in his workspace after aborting an "upgrade apply",
|
||||
// therefore, roll back to the backed up state.
|
||||
if err := u.clusterUpgrader.RestoreClusterWorkspace(); err != nil {
|
||||
return state.Infrastructure{}, fmt.Errorf(
|
||||
"restoring Terraform workspace: %w, restore the Terraform workspace manually from %s ",
|
||||
err,
|
||||
filepath.Join(upgradeDir, constants.TerraformUpgradeBackupDir),
|
||||
)
|
||||
}
|
||||
return state.Infrastructure{}, fmt.Errorf("cluster upgrade aborted by user")
|
||||
}
|
||||
}
|
||||
u.log.Debugf("Applying Terraform migrations")
|
||||
|
||||
infraState, err := u.clusterUpgrader.ApplyClusterUpgrade(cmd.Context(), conf.GetProvider())
|
||||
if err != nil {
|
||||
return infraState, fmt.Errorf("getting Terraform output: %w", err)
|
||||
}
|
||||
state := state.NewState(infraState)
|
||||
// TODO(elchead): AB#3424 move this to updateClusterIDFile and correctly handle existing state when writing state
|
||||
if err := u.fileHandler.WriteYAML(constants.StateFilename, state, file.OptOverwrite); err != nil {
|
||||
return infraState, fmt.Errorf("writing state file: %w", err)
|
||||
return state.Infrastructure{}, fmt.Errorf("applying terraform migrations: %w", err)
|
||||
}
|
||||
|
||||
cmd.Printf("Infrastructure migrations applied successfully and output written to: %s\n"+
|
||||
"A backup of the pre-upgrade state has been written to: %s\n",
|
||||
flags.pf.PrefixPrintablePath(constants.StateFilename),
|
||||
flags.pf.PrefixPrintablePath(filepath.Join(upgradeDir, constants.TerraformUpgradeBackupDir)),
|
||||
)
|
||||
return infraState, nil
|
||||
}
|
||||
|
||||
|
|
@ -408,12 +432,12 @@ func (u *upgradeApplyCmd) confirmAndUpgradeAttestationConfig(
|
|||
if err := u.kubeUpgrader.ApplyJoinConfig(cmd.Context(), newConfig, measurementSalt); err != nil {
|
||||
return fmt.Errorf("updating attestation config: %w", err)
|
||||
}
|
||||
cmd.Println("Successfully update the cluster's attestation config")
|
||||
cmd.Println("Successfully updated the cluster's attestation config")
|
||||
return nil
|
||||
}
|
||||
|
||||
func (u *upgradeApplyCmd) handleServiceUpgrade(
|
||||
cmd *cobra.Command, conf *config.Config, idFile clusterid.File, infra state.Infrastructure,
|
||||
cmd *cobra.Command, conf *config.Config, stateFile *state.State,
|
||||
upgradeDir string, flags upgradeApplyFlags,
|
||||
) error {
|
||||
var secret uri.MasterSecret
|
||||
|
|
@ -432,8 +456,7 @@ func (u *upgradeApplyCmd) handleServiceUpgrade(
|
|||
|
||||
prepareApply := func(allowDestructive bool) (helm.Applier, bool, error) {
|
||||
options.AllowDestructive = allowDestructive
|
||||
executor, includesUpgrades, err := u.helmApplier.PrepareApply(conf, idFile, options,
|
||||
infra, serviceAccURI, secret)
|
||||
executor, includesUpgrades, err := u.helmApplier.PrepareApply(conf, stateFile, options, serviceAccURI, secret)
|
||||
var upgradeErr *compatibility.InvalidUpgradeError
|
||||
switch {
|
||||
case errors.As(err, &upgradeErr):
|
||||
|
|
@ -587,29 +610,6 @@ func parseUpgradeApplyFlags(cmd *cobra.Command) (upgradeApplyFlags, error) {
|
|||
}, nil
|
||||
}
|
||||
|
||||
func updateClusterIDFile(infraState state.Infrastructure, fileHandler file.Handler) error {
|
||||
newIDFile := clusterid.File{
|
||||
InitSecret: []byte(infraState.InitSecret),
|
||||
IP: infraState.ClusterEndpoint,
|
||||
APIServerCertSANs: infraState.APIServerCertSANs,
|
||||
UID: infraState.UID,
|
||||
}
|
||||
if infraState.Azure != nil {
|
||||
newIDFile.AttestationURL = infraState.Azure.AttestationURL
|
||||
}
|
||||
|
||||
idFile := &clusterid.File{}
|
||||
if err := fileHandler.ReadJSON(constants.ClusterIDsFilename, idFile); err != nil {
|
||||
return fmt.Errorf("reading %s: %w", constants.ClusterIDsFilename, err)
|
||||
}
|
||||
|
||||
if err := fileHandler.WriteJSON(constants.ClusterIDsFilename, idFile.Merge(newIDFile), file.OptOverwrite); err != nil {
|
||||
return fmt.Errorf("writing %s: %w", constants.ClusterIDsFilename, err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
type upgradeApplyFlags struct {
|
||||
pf pathprefix.PathPrefixer
|
||||
yes bool
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue