Git-Mirrors/constellation
1
0
Fork 0
mirror of https://github.com/edgelesssys/constellation.git synced 2024-10-01 01:36:09 -04:00
Code Issues Packages Projects Releases Wiki Activity

cli: write infrastructure to new state file (#2321)

Browse Source 
Co-authored-by: 3u13r <lc@edgeless.systems>
This commit is contained in:
Adrian Stobbe 2023-09-25 16:19:43 +02:00 committed by GitHub
parent 8f5a2867b4
commit 322c4aad10
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
26 changed files with 263 additions and 109 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
CODEOWNERS
View File

@ -14,6 +14,7 @@
/cli/internal/libvirt @daniel-weisse
/cli/internal/terraform @elchead
/cli/internal/upgrade @elchead
/cli/internal/state @elchead
/csi @daniel-weisse
/debugd @malt3
/disk-mapper @daniel-weisse
@ -23,7 +24,6 @@
/hack/bazel-deps-mirror @malt3
/hack/cli-k8s-compatibility @derpsteb
/hack/clidocgen @thomasten
/hack/configapi @elchead
/hack/fetch-broken-e2e @katexochen
/hack/gocoverage @katexochen
/hack/oci-pin @malt3

3
cli/internal/cloudcmd/BUILD.bazel
View File

@ -21,9 +21,9 @@ go_library(
importpath = "github.com/edgelesssys/constellation/v2/cli/internal/cloudcmd",
visibility = ["//cli:__subpackages__"],
deps = [
"//cli/internal/clusterid",
"//cli/internal/cmd/pathprefix",
"//cli/internal/libvirt",
"//cli/internal/state",
"//cli/internal/terraform",
"//internal/atls",
"//internal/attestation/choose",
@ -60,6 +60,7 @@ go_test(
],
embed = [":cloudcmd"],
deps = [
"//cli/internal/state",
"//cli/internal/terraform",
"//internal/attestation/measurements",
"//internal/attestation/variant",

3
cli/internal/cloudcmd/clients.go
View File

@ -10,6 +10,7 @@ import (
"context"
"io"
"github.com/edgelesssys/constellation/v2/cli/internal/state"
"github.com/edgelesssys/constellation/v2/cli/internal/terraform"
"github.com/edgelesssys/constellation/v2/internal/attestation/variant"
"github.com/edgelesssys/constellation/v2/internal/cloud/cloudprovider"
@ -33,7 +34,7 @@ type tfCommonClient interface {
type tfResourceClient interface {
tfCommonClient
ApplyCluster(ctx context.Context, provider cloudprovider.Provider, logLevel terraform.LogLevel) (terraform.ApplyOutput, error)
ShowCluster(ctx context.Context, provider cloudprovider.Provider) (terraform.ApplyOutput, error)
ShowInfrastructure(ctx context.Context, provider cloudprovider.Provider) (state.Infrastructure, error)
}
type tfIAMClient interface {

7
cli/internal/cloudcmd/clients_test.go
View File

@ -11,6 +11,7 @@ import (
"io"
"testing"
"github.com/edgelesssys/constellation/v2/cli/internal/state"
"github.com/edgelesssys/constellation/v2/cli/internal/terraform"
"github.com/edgelesssys/constellation/v2/internal/attestation/variant"
"github.com/edgelesssys/constellation/v2/internal/cloud/cloudprovider"
@ -31,7 +32,7 @@ type stubTerraformClient struct {
iamOutput terraform.IAMOutput
uid string
attestationURL string
applyOutput terraform.ApplyOutput
infraState state.Infrastructure
cleanUpWorkspaceCalled bool
removeInstallerCalled bool
destroyCalled bool
@ -77,9 +78,9 @@ func (c *stubTerraformClient) RemoveInstaller() {
c.removeInstallerCalled = true
}
func (c *stubTerraformClient) ShowCluster(_ context.Context, _ cloudprovider.Provider) (terraform.ApplyOutput, error) {
func (c *stubTerraformClient) ShowInfrastructure(_ context.Context, _ cloudprovider.Provider) (state.Infrastructure, error) {
c.showCalled = true
return c.applyOutput, c.showErr
return c.infraState, c.showErr
}
func (c *stubTerraformClient) ShowIAM(_ context.Context, _ cloudprovider.Provider) (terraform.IAMOutput, error) {

26
cli/internal/cloudcmd/create.go
View File

@ -18,8 +18,8 @@ import (
"runtime"
"strings"
"github.com/edgelesssys/constellation/v2/cli/internal/clusterid"
"github.com/edgelesssys/constellation/v2/cli/internal/libvirt"
"github.com/edgelesssys/constellation/v2/cli/internal/state"
"github.com/edgelesssys/constellation/v2/cli/internal/terraform"
"github.com/edgelesssys/constellation/v2/internal/cloud/cloudprovider"
"github.com/edgelesssys/constellation/v2/internal/config"
@ -64,19 +64,19 @@ type CreateOptions struct {
}
// Create creates the handed amount of instances and all the needed resources.
func (c *Creator) Create(ctx context.Context, opts CreateOptions) (clusterid.File, error) {
func (c *Creator) Create(ctx context.Context, opts CreateOptions) (state.Infrastructure, error) {
provider := opts.Config.GetProvider()
attestationVariant := opts.Config.GetAttestationConfig().GetVariant()
region := opts.Config.GetRegion()
image, err := c.image.FetchReference(ctx, provider, attestationVariant, opts.Config.Image, region)
if err != nil {
return clusterid.File{}, fmt.Errorf("fetching image reference: %w", err)
return state.Infrastructure{}, fmt.Errorf("fetching image reference: %w", err)
}
opts.image = image
cl, err := c.newTerraformClient(ctx, opts.TFWorkspace)
if err != nil {
return clusterid.File{}, err
return state.Infrastructure{}, err
}
defer cl.RemoveInstaller()
@ -96,7 +96,7 @@ func (c *Creator) Create(ctx context.Context, opts CreateOptions) (clusterid.Fil
tfOutput, err = c.createOpenStack(ctx, cl, opts)
case cloudprovider.QEMU:
if runtime.GOARCH != "amd64" || runtime.GOOS != "linux" {
return clusterid.File{}, fmt.Errorf("creation of a QEMU based Constellation is not supported for %s/%s", runtime.GOOS, runtime.GOARCH)
return state.Infrastructure{}, fmt.Errorf("creation of a QEMU based Constellation is not supported for %s/%s", runtime.GOOS, runtime.GOARCH)
}
lv := c.newLibvirtRunner()
qemuOpts := qemuCreateOptions{
@ -106,23 +106,13 @@ func (c *Creator) Create(ctx context.Context, opts CreateOptions) (clusterid.Fil
tfOutput, err = c.createQEMU(ctx, cl, lv, qemuOpts)
default:
return clusterid.File{}, fmt.Errorf("unsupported cloud provider: %s", opts.Provider)
return state.Infrastructure{}, fmt.Errorf("unsupported cloud provider: %s", opts.Provider)
}
if err != nil {
return clusterid.File{}, fmt.Errorf("creating cluster: %w", err)
return state.Infrastructure{}, fmt.Errorf("creating cluster: %w", err)
}
res := clusterid.File{
CloudProvider: opts.Provider,
IP: tfOutput.IP,
APIServerCertSANs: tfOutput.APIServerCertSANs,
InitSecret: []byte(tfOutput.Secret),
UID: tfOutput.UID,
}
if tfOutput.Azure != nil {
res.AttestationURL = tfOutput.Azure.AttestationURL
}
return res, nil
return terraform.ConvertToInfrastructure(tfOutput), nil
}
func (c *Creator) createAWS(ctx context.Context, cl tfResourceClient, opts CreateOptions) (tfOutput terraform.ApplyOutput, retErr error) {

3
cli/internal/cloudcmd/create_test.go
View File

@ -237,8 +237,7 @@ func TestCreator(t *testing.T) {
}
} else {
assert.NoError(err)
assert.Equal(tc.provider, idFile.CloudProvider)
assert.Equal(ip, idFile.IP)
assert.Equal(ip, idFile.ClusterEndpoint)
}
})
}

2
cli/internal/cmd/BUILD.bazel
View File

@ -46,6 +46,7 @@ go_library(
"//cli/internal/helm",
"//cli/internal/kubecmd",
"//cli/internal/libvirt",
"//cli/internal/state",
"//cli/internal/terraform",
"//disk-mapper/recoverproto",
"//internal/api/attestationconfigapi",
@ -137,6 +138,7 @@ go_test(
"//cli/internal/cmd/pathprefix",
"//cli/internal/helm",
"//cli/internal/kubecmd",
"//cli/internal/state",
"//cli/internal/terraform",
"//disk-mapper/recoverproto",
"//internal/api/attestationconfigapi",

4
cli/internal/cmd/cloud.go
View File

@ -10,7 +10,7 @@ import (
"context"
"github.com/edgelesssys/constellation/v2/cli/internal/cloudcmd"
"github.com/edgelesssys/constellation/v2/cli/internal/clusterid"
"github.com/edgelesssys/constellation/v2/cli/internal/state"
"github.com/edgelesssys/constellation/v2/cli/internal/terraform"
"github.com/edgelesssys/constellation/v2/internal/cloud/cloudprovider"
"github.com/edgelesssys/constellation/v2/internal/cloud/gcpshared"
@ -20,7 +20,7 @@ type cloudCreator interface {
Create(
ctx context.Context,
opts cloudcmd.CreateOptions,
) (clusterid.File, error)
) (state.Infrastructure, error)
}
type cloudIAMCreator interface {

11
cli/internal/cmd/cloud_test.go
View File

@ -11,7 +11,7 @@ import (
"testing"
"github.com/edgelesssys/constellation/v2/cli/internal/cloudcmd"
"github.com/edgelesssys/constellation/v2/cli/internal/clusterid"
"github.com/edgelesssys/constellation/v2/cli/internal/state"
"github.com/edgelesssys/constellation/v2/cli/internal/terraform"
"github.com/edgelesssys/constellation/v2/internal/cloud/cloudprovider"
"github.com/edgelesssys/constellation/v2/internal/cloud/gcpshared"
@ -27,17 +27,16 @@ func TestMain(m *testing.M) {
type stubCloudCreator struct {
createCalled bool
id clusterid.File
state state.Infrastructure
createErr error
}
func (c *stubCloudCreator) Create(
_ context.Context,
opts cloudcmd.CreateOptions,
) (clusterid.File, error) {
_ cloudcmd.CreateOptions,
) (state.Infrastructure, error) {
c.createCalled = true
c.id.CloudProvider = opts.Provider
return c.id, c.createErr
return c.state, c.createErr
}
type stubCloudTerminator struct {

21
cli/internal/cmd/create.go
View File

@ -12,11 +12,14 @@ import (
"io/fs"
"github.com/edgelesssys/constellation/v2/cli/internal/cloudcmd"
"github.com/edgelesssys/constellation/v2/cli/internal/clusterid"
"github.com/edgelesssys/constellation/v2/cli/internal/cmd/pathprefix"
"github.com/edgelesssys/constellation/v2/cli/internal/state"
"github.com/edgelesssys/constellation/v2/cli/internal/terraform"
"github.com/edgelesssys/constellation/v2/internal/api/attestationconfigapi"
"github.com/edgelesssys/constellation/v2/internal/api/versionsapi"
"github.com/edgelesssys/constellation/v2/internal/attestation/variant"
"github.com/edgelesssys/constellation/v2/internal/cloud/cloudprovider"
"github.com/edgelesssys/constellation/v2/internal/config"
"github.com/edgelesssys/constellation/v2/internal/constants"
"github.com/edgelesssys/constellation/v2/internal/file"
@ -162,21 +165,37 @@ func (c *createCmd) create(cmd *cobra.Command, creator cloudCreator, fileHandler
TFLogLevel: flags.tfLogLevel,
TFWorkspace: constants.TerraformWorkingDir,
}
idFile, err := creator.Create(cmd.Context(), opts)
infraState, err := creator.Create(cmd.Context(), opts)
spinner.Stop()
if err != nil {
return translateCreateErrors(cmd, c.pf, err)
}
c.log.Debugf("Successfully created the cloud resources for the cluster")
idFile := convertToIDFile(infraState, provider)
if err := fileHandler.WriteJSON(constants.ClusterIDsFilename, idFile, file.OptNone); err != nil {
return err
}
state := state.NewState(infraState)
if err := fileHandler.WriteYAML(constants.StateFilename, state, file.OptNone); err != nil {
return err
}
cmd.Println("Your Constellation cluster was created successfully.")
return nil
}
func convertToIDFile(infra state.Infrastructure, provider cloudprovider.Provider) clusterid.File {
var file clusterid.File
file.CloudProvider = provider
file.IP = infra.ClusterEndpoint
file.APIServerCertSANs = infra.APIServerCertSANs
file.InitSecret = []byte(infra.InitSecret) // Convert string to []byte
file.UID = infra.UID
return file
}
// parseCreateFlags parses the flags of the create command.
func (c *createCmd) parseCreateFlags(cmd *cobra.Command) (createFlags, error) {
yes, err := cmd.Flags().GetBool("yes")

19
cli/internal/cmd/create_test.go
View File

@ -12,6 +12,7 @@ import (
"testing"
"github.com/edgelesssys/constellation/v2/cli/internal/clusterid"
"github.com/edgelesssys/constellation/v2/cli/internal/state"
"github.com/edgelesssys/constellation/v2/internal/cloud/cloudprovider"
"github.com/edgelesssys/constellation/v2/internal/config"
"github.com/edgelesssys/constellation/v2/internal/constants"
@ -30,7 +31,7 @@ func TestCreate(t *testing.T) {
require.NoError(file.WriteYAML(constants.ConfigFilename, defaultConfigWithExpectedMeasurements(t, config.Default(), provider)))
return fs
}
idFile := clusterid.File{IP: "192.0.2.1"}
infraState := state.Infrastructure{ClusterEndpoint: "192.0.2.1"}
someErr := errors.New("failed")
testCases := map[string]struct {
@ -46,13 +47,13 @@ func TestCreate(t *testing.T) {
}{
"create": {
setupFs: fsWithDefaultConfig,
creator: &stubCloudCreator{id: idFile},
creator: &stubCloudCreator{state: infraState},
provider: cloudprovider.GCP,
yesFlag: true,
},
"interactive": {
setupFs: fsWithDefaultConfig,
creator: &stubCloudCreator{id: idFile},
creator: &stubCloudCreator{state: infraState},
provider: cloudprovider.Azure,
stdin: "yes\n",
},
@ -156,9 +157,19 @@ func TestCreate(t *testing.T) {
var gotIDFile clusterid.File
require.NoError(fileHandler.ReadJSON(constants.ClusterIDsFilename, &gotIDFile))
assert.Equal(gotIDFile, clusterid.File{
IP: idFile.IP,
IP: infraState.ClusterEndpoint,
CloudProvider: tc.provider,
})
var gotState state.State
expectedState := state.Infrastructure{
ClusterEndpoint: "192.0.2.1",
APIServerCertSANs: []string{},
}
require.NoError(fileHandler.ReadYAML(constants.StateFilename, &gotState))
assert.Equal("v1", gotState.Version)
assert.Equal(expectedState, gotState.Infrastructure)
}
}
})

19
cli/internal/cmd/init.go
View File

@ -39,6 +39,7 @@ import (
"github.com/edgelesssys/constellation/v2/cli/internal/cmd/pathprefix"
"github.com/edgelesssys/constellation/v2/cli/internal/helm"
"github.com/edgelesssys/constellation/v2/cli/internal/kubecmd"
"github.com/edgelesssys/constellation/v2/cli/internal/state"
"github.com/edgelesssys/constellation/v2/cli/internal/terraform"
"github.com/edgelesssys/constellation/v2/internal/cloud/cloudprovider"
"github.com/edgelesssys/constellation/v2/internal/config"
@ -75,12 +76,12 @@ type initCmd struct {
merger configMerger
spinner spinnerInterf
fileHandler file.Handler
clusterShower clusterShower
clusterShower infrastructureShower
pf pathprefix.PathPrefixer
}
func newInitCmd(
clusterShower clusterShower, fileHandler file.Handler,
clusterShower infrastructureShower, fileHandler file.Handler,
spinner spinnerInterf, merger configMerger, log debugLog,
) *initCmd {
return &initCmd{
@ -261,9 +262,9 @@ func (i *initCmd) initialize(
return fmt.Errorf("applying attestation config: %w", err)
}
output, err := i.clusterShower.ShowCluster(cmd.Context(), conf.GetProvider())
infraState, err := i.clusterShower.ShowInfrastructure(cmd.Context(), conf.GetProvider())
if err != nil {
return fmt.Errorf("getting Terraform output: %w", err)
return fmt.Errorf("getting infrastructure state: %w", err)
}
i.spinner.Start("Installing Kubernetes components ", false)
@ -277,7 +278,7 @@ func (i *initCmd) initialize(
if err != nil {
return fmt.Errorf("creating Helm client: %w", err)
}
executor, includesUpgrades, err := helmApplier.PrepareApply(conf, idFile, options, output,
executor, includesUpgrades, err := helmApplier.PrepareApply(conf, idFile, options, infraState,
serviceAccURI, masterSecret)
if err != nil {
return fmt.Errorf("getting Helm chart executor: %w", err)
@ -671,9 +672,11 @@ type attestationConfigApplier interface {
}
type helmApplier interface {
PrepareApply(conf *config.Config, idFile clusterid.File, flags helm.Options, tfOutput terraform.ApplyOutput, serviceAccURI string, masterSecret uri.MasterSecret) (helm.Applier, bool, error)
PrepareApply(conf *config.Config, idFile clusterid.File,
flags helm.Options, infra state.Infrastructure, serviceAccURI string, masterSecret uri.MasterSecret) (
helm.Applier, bool, error)
}
type clusterShower interface {
ShowCluster(ctx context.Context, provider cloudprovider.Provider) (terraform.ApplyOutput, error)
type infrastructureShower interface {
ShowInfrastructure(ctx context.Context, provider cloudprovider.Provider) (state.Infrastructure, error)
}

16
cli/internal/cmd/init_test.go
View File

@ -23,7 +23,7 @@ import (
"github.com/edgelesssys/constellation/v2/cli/internal/clusterid"
"github.com/edgelesssys/constellation/v2/cli/internal/cmd/pathprefix"
"github.com/edgelesssys/constellation/v2/cli/internal/helm"
"github.com/edgelesssys/constellation/v2/cli/internal/terraform"
"github.com/edgelesssys/constellation/v2/cli/internal/state"
"github.com/edgelesssys/constellation/v2/internal/atls"
"github.com/edgelesssys/constellation/v2/internal/attestation/measurements"
"github.com/edgelesssys/constellation/v2/internal/attestation/variant"
@ -228,7 +228,7 @@ func TestInitialize(t *testing.T) {
ctx, cancel := context.WithTimeout(ctx, 4*time.Second)
defer cancel()
cmd.SetContext(ctx)
i := newInitCmd(&stubShowCluster{}, fileHandler, &nopSpinner{}, nil, logger.NewTest(t))
i := newInitCmd(&stubShowInfrastructure{}, fileHandler, &nopSpinner{}, nil, logger.NewTest(t))
err := i.initialize(cmd, newDialer, &stubLicenseClient{}, stubAttestationFetcher{},
func(io.Writer, string, debugLog) (attestationConfigApplier, error) {
return &stubAttestationApplier{}, nil
@ -264,7 +264,7 @@ type stubApplier struct {
err error
}
func (s stubApplier) PrepareApply(_ *config.Config, _ clusterid.File, _ helm.Options, _ terraform.ApplyOutput, _ string, _ uri.MasterSecret) (helm.Applier, bool, error) {
func (s stubApplier) PrepareApply(_ *config.Config, _ clusterid.File, _ helm.Options, _ state.Infrastructure, _ string, _ uri.MasterSecret) (helm.Applier, bool, error) {
return stubRunner{}, false, s.err
}
@ -722,15 +722,15 @@ func (c stubInitClient) Recv() (*initproto.InitResponse, error) {
return res, err
}
type stubShowCluster struct{}
type stubShowInfrastructure struct{}
func (s *stubShowCluster) ShowCluster(_ context.Context, csp cloudprovider.Provider) (terraform.ApplyOutput, error) {
res := terraform.ApplyOutput{}
func (s *stubShowInfrastructure) ShowInfrastructure(_ context.Context, csp cloudprovider.Provider) (state.Infrastructure, error) {
res := state.Infrastructure{}
switch csp {
case cloudprovider.Azure:
res.Azure = &terraform.AzureApplyOutput{}
res.Azure = &state.Azure{}
case cloudprovider.GCP:
res.GCP = &terraform.GCPApplyOutput{}
res.GCP = &state.GCP{}
}
return res, nil
}

42
cli/internal/cmd/upgradeapply.go
View File

@ -20,6 +20,7 @@ import (
"github.com/edgelesssys/constellation/v2/cli/internal/cmd/pathprefix"
"github.com/edgelesssys/constellation/v2/cli/internal/helm"
"github.com/edgelesssys/constellation/v2/cli/internal/kubecmd"
"github.com/edgelesssys/constellation/v2/cli/internal/state"
"github.com/edgelesssys/constellation/v2/cli/internal/terraform"
"github.com/edgelesssys/constellation/v2/internal/api/attestationconfigapi"
"github.com/edgelesssys/constellation/v2/internal/attestation/variant"
@ -139,7 +140,7 @@ type upgradeApplyCmd struct {
kubeUpgrader kubernetesUpgrader
clusterUpgrader clusterUpgrader
configFetcher attestationconfigapi.Fetcher
clusterShower clusterShower
clusterShower infrastructureShower
fileHandler file.Handler
log debugLog
}
@ -189,14 +190,14 @@ func (u *upgradeApplyCmd) upgradeApply(cmd *cobra.Command, upgradeDir string, fl
return fmt.Errorf("upgrading measurements: %w", err)
}
var tfOutput terraform.ApplyOutput
var infraState state.Infrastructure
if flags.skipPhases.contains(skipInfrastructurePhase) {
tfOutput, err = u.clusterShower.ShowCluster(cmd.Context(), conf.GetProvider())
infraState, err = u.clusterShower.ShowInfrastructure(cmd.Context(), conf.GetProvider())
if err != nil {
return fmt.Errorf("getting Terraform output: %w", err)
return fmt.Errorf("getting infra state: %w", err)
}
} else {
tfOutput, err = u.migrateTerraform(cmd, conf, upgradeDir, flags)
infraState, err = u.migrateTerraform(cmd, conf, upgradeDir, flags)
if err != nil {
return fmt.Errorf("performing Terraform migrations: %w", err)
}
@ -206,7 +207,11 @@ func (u *upgradeApplyCmd) upgradeApply(cmd *cobra.Command, upgradeDir string, fl
if err := u.fileHandler.ReadJSON(constants.ClusterIDsFilename, &idFile); err != nil {
return fmt.Errorf("reading updated cluster ID file: %w", err)
}
state := state.NewState(infraState)
// TODO(elchead): AB#3424 move this to updateClusterIDFile and correctly handle existing state when writing state
if err := u.fileHandler.WriteYAML(constants.StateFilename, state, file.OptOverwrite); err != nil {
return fmt.Errorf("writing state file: %w", err)
}
// extend the clusterConfig cert SANs with any of the supported endpoints:
// - (legacy) public IP
// - fallback endpoint
@ -223,7 +228,7 @@ func (u *upgradeApplyCmd) upgradeApply(cmd *cobra.Command, upgradeDir string, fl
var upgradeErr *compatibility.InvalidUpgradeError
if !flags.skipPhases.contains(skipHelmPhase) {
err = u.handleServiceUpgrade(cmd, conf, idFile, tfOutput, upgradeDir, flags)
err = u.handleServiceUpgrade(cmd, conf, idFile, infraState, upgradeDir, flags)
switch {
case errors.As(err, &upgradeErr):
cmd.PrintErrln(err)
@ -233,7 +238,6 @@ func (u *upgradeApplyCmd) upgradeApply(cmd *cobra.Command, upgradeDir string, fl
return fmt.Errorf("upgrading services: %w", err)
}
}
skipImageUpgrade := flags.skipPhases.contains(skipImagePhase)
skipK8sUpgrade := flags.skipPhases.contains(skipK8sPhase)
if !(skipImageUpgrade && skipK8sUpgrade) {
@ -267,7 +271,7 @@ func diffAttestationCfg(currentAttestationCfg config.AttestationCfg, newAttestat
// migrateTerraform checks if the Constellation version the cluster is being upgraded to requires a migration
// of cloud resources with Terraform. If so, the migration is performed.
func (u *upgradeApplyCmd) migrateTerraform(cmd *cobra.Command, conf *config.Config, upgradeDir string, flags upgradeApplyFlags,
) (res terraform.ApplyOutput, err error) {
) (res state.Infrastructure, err error) {
u.log.Debugf("Planning Terraform migrations")
vars, err := cloudcmd.TerraformUpgradeVars(conf)
@ -315,13 +319,14 @@ func (u *upgradeApplyCmd) migrateTerraform(cmd *cobra.Command, conf *config.Conf
}
u.log.Debugf("Applying Terraform migrations")
tfOutput, err := u.clusterUpgrader.ApplyClusterUpgrade(cmd.Context(), conf.GetProvider())
res = terraform.ConvertToInfrastructure(tfOutput)
if err != nil {
return tfOutput, fmt.Errorf("applying terraform migrations: %w", err)
return res, fmt.Errorf("applying terraform migrations: %w", err)
}
// Apply possible updates to cluster ID file
if err := updateClusterIDFile(tfOutput, u.fileHandler); err != nil {
return tfOutput, fmt.Errorf("merging cluster ID files: %w", err)
return res, fmt.Errorf("merging cluster ID files: %w", err)
}
cmd.Printf("Terraform migrations applied successfully and output written to: %s\n"+
@ -333,11 +338,16 @@ func (u *upgradeApplyCmd) migrateTerraform(cmd *cobra.Command, conf *config.Conf
u.log.Debugf("No Terraform diff detected")
}
u.log.Debugf("No Terraform diff detected")
tfOutput, err := u.clusterShower.ShowCluster(cmd.Context(), conf.GetProvider())
infraState, err := u.clusterShower.ShowInfrastructure(cmd.Context(), conf.GetProvider())
if err != nil {
return tfOutput, fmt.Errorf("getting Terraform output: %w", err)
return infraState, fmt.Errorf("getting Terraform output: %w", err)
}
return tfOutput, nil
state := state.NewState(infraState)
// TODO(elchead): AB#3424 move this to updateClusterIDFile and correctly handle existing state when writing state
if err := u.fileHandler.WriteYAML(constants.StateFilename, state, file.OptOverwrite); err != nil {
return infraState, fmt.Errorf("writing state file: %w", err)
}
return infraState, nil
}
// validK8sVersion checks if the Kubernetes patch version is supported and asks for confirmation if not.
@ -404,7 +414,7 @@ func (u *upgradeApplyCmd) confirmAndUpgradeAttestationConfig(
}
func (u *upgradeApplyCmd) handleServiceUpgrade(
cmd *cobra.Command, conf *config.Config, idFile clusterid.File, tfOutput terraform.ApplyOutput,
cmd *cobra.Command, conf *config.Config, idFile clusterid.File, infra state.Infrastructure,
upgradeDir string, flags upgradeApplyFlags,
) error {
var secret uri.MasterSecret
@ -424,7 +434,7 @@ func (u *upgradeApplyCmd) handleServiceUpgrade(
prepareApply := func(allowDestructive bool) (helm.Applier, bool, error) {
options.AllowDestructive = allowDestructive
executor, includesUpgrades, err := u.helmApplier.PrepareApply(conf, idFile, options,
tfOutput, serviceAccURI, secret)
infra, serviceAccURI, secret)
var upgradeErr *compatibility.InvalidUpgradeError
switch {
case errors.As(err, &upgradeErr):

18
cli/internal/cmd/upgradeapply_test.go
View File

@ -15,6 +15,7 @@ import (
"github.com/edgelesssys/constellation/v2/cli/internal/clusterid"
"github.com/edgelesssys/constellation/v2/cli/internal/helm"
"github.com/edgelesssys/constellation/v2/cli/internal/kubecmd"
"github.com/edgelesssys/constellation/v2/cli/internal/state"
"github.com/edgelesssys/constellation/v2/cli/internal/terraform"
"github.com/edgelesssys/constellation/v2/internal/attestation/variant"
"github.com/edgelesssys/constellation/v2/internal/cloud/cloudprovider"
@ -186,7 +187,7 @@ func TestUpgradeApply(t *testing.T) {
clusterUpgrader: tc.terraformUpgrader,
log: logger.NewTest(t),
configFetcher: stubAttestationFetcher{},
clusterShower: &stubShowCluster{},
clusterShower: &stubShowInfrastructure{},
fileHandler: handler,
}
@ -198,6 +199,15 @@ func TestUpgradeApply(t *testing.T) {
assert.NoError(err)
assert.Equal(!tc.flags.skipPhases.contains(skipImagePhase), tc.kubeUpgrader.calledNodeUpgrade,
"incorrect node upgrade skipping behavior")
var gotState state.State
expectedState := state.Infrastructure{
APIServerCertSANs: []string{},
Azure: &state.Azure{},
}
require.NoError(handler.ReadYAML(constants.StateFilename, &gotState))
assert.Equal("v1", gotState.Version)
assert.Equal(expectedState, gotState.Infrastructure)
})
}
}
@ -298,7 +308,9 @@ type mockApplier struct {
mock.Mock
}
func (m *mockApplier) PrepareApply(cfg *config.Config, clusterID clusterid.File, helmOpts helm.Options, terraformOut terraform.ApplyOutput, str string, masterSecret uri.MasterSecret) (helm.Applier, bool, error) {
args := m.Called(cfg, clusterID, helmOpts, terraformOut, str, masterSecret)
func (m *mockApplier) PrepareApply(cfg *config.Config, clusterID clusterid.File,
helmOpts helm.Options, infraState state.Infrastructure, str string, masterSecret uri.MasterSecret,
) (helm.Applier, bool, error) {
args := m.Called(cfg, clusterID, helmOpts, infraState, str, masterSecret)
return args.Get(0).(helm.Applier), args.Bool(1), args.Error(2)
}

4
cli/internal/helm/BUILD.bazel
View File

@ -422,7 +422,7 @@ go_library(
deps = [
"//cli/internal/clusterid",
"//cli/internal/helm/imageversion",
"//cli/internal/terraform",
"//cli/internal/state",
"//internal/cloud/azureshared",
"//internal/cloud/cloudprovider",
"//internal/cloud/gcpshared",
@ -463,7 +463,7 @@ go_test(
embed = [":helm"],
deps = [
"//cli/internal/clusterid",
"//cli/internal/terraform",
"//cli/internal/state",
"//internal/attestation/measurements",
"//internal/cloud/azureshared",
"//internal/cloud/cloudprovider",

16
cli/internal/helm/helm.go
View File

@ -33,7 +33,7 @@ import (
"fmt"
"github.com/edgelesssys/constellation/v2/cli/internal/clusterid"
"github.com/edgelesssys/constellation/v2/cli/internal/terraform"
"github.com/edgelesssys/constellation/v2/cli/internal/state"
"github.com/edgelesssys/constellation/v2/internal/config"
"github.com/edgelesssys/constellation/v2/internal/constants"
"github.com/edgelesssys/constellation/v2/internal/file"
@ -86,10 +86,11 @@ type Options struct {
// PrepareApply loads the charts and returns the executor to apply them.
// TODO(elchead): remove validK8sVersion by putting ValidK8sVersion into config.Config, see AB#3374.
func (h Client) PrepareApply(conf *config.Config, idFile clusterid.File, flags Options, tfOutput terraform.ApplyOutput,
serviceAccURI string, masterSecret uri.MasterSecret,
func (h Client) PrepareApply(
conf *config.Config, idFile clusterid.File,
flags Options, infra state.Infrastructure, serviceAccURI string, masterSecret uri.MasterSecret,
) (Applier, bool, error) {
releases, err := h.loadReleases(conf, masterSecret, idFile, flags, tfOutput, serviceAccURI)
releases, err := h.loadReleases(conf, masterSecret, idFile, flags, infra, serviceAccURI)
if err != nil {
return nil, false, fmt.Errorf("loading Helm releases: %w", err)
}
@ -98,13 +99,14 @@ func (h Client) PrepareApply(conf *config.Config, idFile clusterid.File, flags O
return &ChartApplyExecutor{actions: actions, log: h.log}, includesUpgrades, err
}
func (h Client) loadReleases(conf *config.Config, secret uri.MasterSecret, idFile clusterid.File, flags Options,
tfOutput terraform.ApplyOutput, serviceAccURI string,
func (h Client) loadReleases(
conf *config.Config, secret uri.MasterSecret,
idFile clusterid.File, flags Options, infra state.Infrastructure, serviceAccURI string,
) ([]Release, error) {
helmLoader := newLoader(conf, idFile, h.cliVersion)
h.log.Debugf("Created new Helm loader")
return helmLoader.loadReleases(flags.Conformance, flags.HelmWaitMode, secret,
serviceAccURI, tfOutput)
serviceAccURI, infra)
}
// Applier runs the Helm actions.

10
cli/internal/helm/helm_test.go
View File

@ -11,7 +11,7 @@ import (
"testing"
"github.com/edgelesssys/constellation/v2/cli/internal/clusterid"
"github.com/edgelesssys/constellation/v2/cli/internal/terraform"
"github.com/edgelesssys/constellation/v2/cli/internal/state"
"github.com/edgelesssys/constellation/v2/internal/cloud/cloudprovider"
"github.com/edgelesssys/constellation/v2/internal/compatibility"
"github.com/edgelesssys/constellation/v2/internal/config"
@ -209,7 +209,7 @@ func TestHelmApply(t *testing.T) {
options.AllowDestructive = tc.allowDestructive
ex, includesUpgrade, err := sut.PrepareApply(cfg,
clusterid.File{UID: "testuid", MeasurementSalt: []byte("measurementSalt")}, options,
fakeTerraformOutput(csp), fakeServiceAccURI(csp),
fakeInfraOutput(csp), fakeServiceAccURI(csp),
uri.MasterSecret{Key: []byte("secret"), Salt: []byte("masterSalt")})
var upgradeErr *compatibility.InvalidUpgradeError
if tc.expectError {
@ -225,12 +225,12 @@ func TestHelmApply(t *testing.T) {
}
}
func fakeTerraformOutput(csp cloudprovider.Provider) terraform.ApplyOutput {
func fakeInfraOutput(csp cloudprovider.Provider) state.Infrastructure {
switch csp {
case cloudprovider.AWS:
return terraform.ApplyOutput{}
return state.Infrastructure{}
case cloudprovider.GCP:
return terraform.ApplyOutput{GCP: &terraform.GCPApplyOutput{}}
return state.Infrastructure{GCP: &state.GCP{}}
default:
panic("invalid csp")
}

8
cli/internal/helm/loader.go
View File

@ -21,7 +21,7 @@ import (
"github.com/edgelesssys/constellation/v2/cli/internal/clusterid"
"github.com/edgelesssys/constellation/v2/cli/internal/helm/imageversion"
"github.com/edgelesssys/constellation/v2/cli/internal/terraform"
"github.com/edgelesssys/constellation/v2/cli/internal/state"
"github.com/edgelesssys/constellation/v2/internal/cloud/cloudprovider"
"github.com/edgelesssys/constellation/v2/internal/config"
"github.com/edgelesssys/constellation/v2/internal/constants"
@ -120,13 +120,13 @@ type releaseApplyOrder []Release
// loadReleases loads the embedded helm charts and returns them as a HelmReleases object.
func (i *chartLoader) loadReleases(conformanceMode bool, helmWaitMode WaitMode, masterSecret uri.MasterSecret,
serviceAccURI string, output terraform.ApplyOutput,
serviceAccURI string, infra state.Infrastructure,
) (releaseApplyOrder, error) {
ciliumRelease, err := i.loadRelease(ciliumInfo, helmWaitMode)
if err != nil {
return nil, fmt.Errorf("loading cilium: %w", err)
}
ciliumVals := extraCiliumValues(i.config.GetProvider(), conformanceMode, output)
ciliumVals := extraCiliumValues(i.config.GetProvider(), conformanceMode, infra)
ciliumRelease.Values = mergeMaps(ciliumRelease.Values, ciliumVals)
certManagerRelease, err := i.loadRelease(certManagerInfo, helmWaitMode)
@ -145,7 +145,7 @@ func (i *chartLoader) loadReleases(conformanceMode bool, helmWaitMode WaitMode,
return nil, fmt.Errorf("loading constellation-services: %w", err)
}
svcVals, err := extraConstellationServicesValues(i.config, masterSecret, i.idFile.UID, serviceAccURI, output)
svcVals, err := extraConstellationServicesValues(i.config, masterSecret, i.idFile.UID, serviceAccURI, infra)
if err != nil {
return nil, fmt.Errorf("extending constellation-services values: %w", err)
}

10
cli/internal/helm/loader_test.go
View File

@ -23,7 +23,7 @@ import (
"helm.sh/helm/v3/pkg/engine"
"github.com/edgelesssys/constellation/v2/cli/internal/clusterid"
"github.com/edgelesssys/constellation/v2/cli/internal/terraform"
"github.com/edgelesssys/constellation/v2/cli/internal/state"
"github.com/edgelesssys/constellation/v2/internal/attestation/measurements"
"github.com/edgelesssys/constellation/v2/internal/cloud/azureshared"
"github.com/edgelesssys/constellation/v2/internal/cloud/cloudprovider"
@ -71,7 +71,7 @@ func TestLoadReleases(t *testing.T) {
helmReleases, err := chartLoader.loadReleases(
true, WaitModeAtomic,
uri.MasterSecret{Key: []byte("secret"), Salt: []byte("masterSalt")},
fakeServiceAccURI(cloudprovider.GCP), terraform.ApplyOutput{GCP: &terraform.GCPApplyOutput{}},
fakeServiceAccURI(cloudprovider.GCP), state.Infrastructure{GCP: &state.GCP{}},
)
require.NoError(err)
for _, release := range helmReleases {
@ -175,9 +175,9 @@ func TestConstellationServices(t *testing.T) {
Key: []byte("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"),
Salt: []byte("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"),
},
"uid", serviceAccURI, terraform.ApplyOutput{
Azure: &terraform.AzureApplyOutput{},
GCP: &terraform.GCPApplyOutput{},
"uid", serviceAccURI, state.Infrastructure{
Azure: &state.Azure{},
GCP: &state.GCP{},
})
require.NoError(err)
values = mergeMaps(values, extraVals)

24
cli/internal/helm/overrides.go
View File

@ -13,7 +13,7 @@ import (
"encoding/json"
"fmt"
"github.com/edgelesssys/constellation/v2/cli/internal/terraform"
"github.com/edgelesssys/constellation/v2/cli/internal/state"
"github.com/edgelesssys/constellation/v2/internal/cloud/azureshared"
"github.com/edgelesssys/constellation/v2/internal/cloud/cloudprovider"
"github.com/edgelesssys/constellation/v2/internal/cloud/gcpshared"
@ -31,7 +31,7 @@ import (
// reuse user input from the init step. However, we can't rely on reuse-values, because
// during upgrades we all values need to be set locally as they might have changed.
// Also, the charts are not rendered correctly without all of these values.
func extraCiliumValues(provider cloudprovider.Provider, conformanceMode bool, output terraform.ApplyOutput) map[string]any {
func extraCiliumValues(provider cloudprovider.Provider, conformanceMode bool, output state.Infrastructure) map[string]any {
extraVals := map[string]any{}
if conformanceMode {
extraVals["kubeProxyReplacementHealthzBindAddr"] = ""
@ -42,7 +42,7 @@ func extraCiliumValues(provider cloudprovider.Provider, conformanceMode bool, ou
}
}
extraVals["k8sServiceHost"] = output.IP
extraVals["k8sServiceHost"] = output.ClusterEndpoint
extraVals["k8sServicePort"] = constants.KubernetesPort
if provider == cloudprovider.GCP {
extraVals["ipv4NativeRoutingCIDR"] = output.GCP.IPCidrPod
@ -54,7 +54,7 @@ func extraCiliumValues(provider cloudprovider.Provider, conformanceMode bool, ou
// extraConstellationServicesValues extends the given values map by some values depending on user input.
// Values set inside this function are only applied during init, not during upgrade.
func extraConstellationServicesValues(
cfg *config.Config, masterSecret uri.MasterSecret, uid, serviceAccURI string, output terraform.ApplyOutput,
cfg *config.Config, masterSecret uri.MasterSecret, uid, serviceAccURI string, output state.Infrastructure,
) (map[string]any, error) {
extraVals := map[string]any{}
extraVals["join-service"] = map[string]any{
@ -62,10 +62,10 @@ func extraConstellationServicesValues(
}
extraVals["verification-service"] = map[string]any{
"attestationVariant": cfg.GetAttestationConfig().GetVariant().String(),
"loadBalancerIP": output.IP,
"loadBalancerIP": output.ClusterEndpoint,
}
extraVals["konnectivity"] = map[string]any{
"loadBalancerIP": output.IP,
"loadBalancerIP": output.ClusterEndpoint,
}
extraVals["key-service"] = map[string]any{
@ -147,7 +147,7 @@ type cloudConfig struct {
}
// getCCMConfig returns the configuration needed for the Kubernetes Cloud Controller Manager on Azure.
func getCCMConfig(tfOutput terraform.AzureApplyOutput, serviceAccURI string) ([]byte, error) {
func getCCMConfig(azureState state.Azure, serviceAccURI string) ([]byte, error) {
creds, err := azureshared.ApplicationCredentialsFromURI(serviceAccURI)
if err != nil {
return nil, fmt.Errorf("getting service account key: %w", err)
@ -156,16 +156,16 @@ func getCCMConfig(tfOutput terraform.AzureApplyOutput, serviceAccURI string) ([]
config := cloudConfig{
Cloud: "AzurePublicCloud",
TenantID: creds.TenantID,
SubscriptionID: tfOutput.SubscriptionID,
ResourceGroup: tfOutput.ResourceGroup,
SubscriptionID: azureState.SubscriptionID,
ResourceGroup: azureState.ResourceGroup,
LoadBalancerSku: "standard",
SecurityGroupName: tfOutput.NetworkSecurityGroupName,
LoadBalancerName: tfOutput.LoadBalancerName,
SecurityGroupName: azureState.NetworkSecurityGroupName,
LoadBalancerName: azureState.LoadBalancerName,
UseInstanceMetadata: true,
VMType: "vmss",
Location: creds.Location,
UseManagedIdentityExtension: useManagedIdentityExtension,
UserAssignedIdentityID: tfOutput.UserAssignedIdentity,
UserAssignedIdentityID: azureState.UserAssignedIdentity,
}
return json.Marshal(config)

8
cli/internal/state/BUILD.bazel Normal file
View File

@ -0,0 +1,8 @@
load("@io_bazel_rules_go//go:def.bzl", "go_library")
go_library(
name = "state",
srcs = ["state.go"],
importpath = "github.com/edgelesssys/constellation/v2/cli/internal/state",
visibility = ["//cli:__subpackages__"],
)

54
cli/internal/state/state.go Normal file
View File

@ -0,0 +1,54 @@
/*
Copyright (c) Edgeless Systems GmbH
SPDX-License-Identifier: AGPL-3.0-only
*/
// package state defines the structure of the Constellation state file.
package state
const (
// Version1 is the first version of the state file.
Version1 = "v1"
)
// State describe the entire state to describe a Constellation cluster.
type State struct {
Version string `yaml:"version"`
Infrastructure Infrastructure `yaml:"infrastructure"`
}
// NewState creates a new state with the given infrastructure.
func NewState(Infrastructure Infrastructure) State {
return State{
Version: Version1,
Infrastructure: Infrastructure,
}
}
// Infrastructure describe the state related to the cloud resources of the cluster.
type Infrastructure struct {
UID string `yaml:"uid"`
ClusterEndpoint string `yaml:"clusterEndpoint"`
InitSecret string `yaml:"initSecret"`
APIServerCertSANs []string `yaml:"apiServerCertSANs"`
Azure *Azure `yaml:"azure"`
GCP *GCP `yaml:"gcp"`
}
// GCP describes the infra state related to GCP.
type GCP struct {
ProjectID string `yaml:"projectID"`
IPCidrNode string `yaml:"ipCidrNode"`
IPCidrPod string `yaml:"ipCidrPod"`
}
// Azure describes the infra state related to Azure.
type Azure struct {
ResourceGroup string `yaml:"resourceGroup"`
SubscriptionID string `yaml:"subscriptionID"`
NetworkSecurityGroupName string `yaml:"networkSecurityGroupName"`
LoadBalancerName string `yaml:"loadBalancerName"`
UserAssignedIdentity string `yaml:"userAssignedIdentity"`
AttestationURL string `yaml:"attestationURL"`
}

1
cli/internal/terraform/BUILD.bazel
View File

@ -77,6 +77,7 @@ go_library(
importpath = "github.com/edgelesssys/constellation/v2/cli/internal/terraform",
visibility = ["//cli:__subpackages__"],
deps = [
"//cli/internal/state",
"//internal/cloud/cloudprovider",
"//internal/constants",
"//internal/file",

39
cli/internal/terraform/terraform.go
View File

@ -24,6 +24,7 @@ import (
"io"
"path/filepath"
"github.com/edgelesssys/constellation/v2/cli/internal/state"
"github.com/edgelesssys/constellation/v2/internal/cloud/cloudprovider"
"github.com/edgelesssys/constellation/v2/internal/constants"
"github.com/edgelesssys/constellation/v2/internal/file"
@ -170,6 +171,44 @@ func (c *Client) ShowIAM(ctx context.Context, provider cloudprovider.Provider) (
}
}
// ShowInfrastructure reads the state of Constellation cluster resources from Terraform.
func (c *Client) ShowInfrastructure(ctx context.Context, provider cloudprovider.Provider) (state.Infrastructure, error) {
tfOutput, err := c.ShowCluster(ctx, provider)
if err != nil {
return state.Infrastructure{}, err
}
return ConvertToInfrastructure(tfOutput), nil
}
// ConvertToInfrastructure converts the Terraform output of a cluster creation or apply operation to a state.Infrastructure.
func ConvertToInfrastructure(applyOutput ApplyOutput) state.Infrastructure {
var infra state.Infrastructure
infra.UID = applyOutput.UID
infra.ClusterEndpoint = applyOutput.IP
infra.InitSecret = applyOutput.Secret
infra.APIServerCertSANs = applyOutput.APIServerCertSANs
if applyOutput.Azure != nil {
infra.Azure = &state.Azure{
ResourceGroup: applyOutput.Azure.ResourceGroup,
SubscriptionID: applyOutput.Azure.SubscriptionID,
UserAssignedIdentity: applyOutput.Azure.UserAssignedIdentity,
NetworkSecurityGroupName: applyOutput.Azure.NetworkSecurityGroupName,
LoadBalancerName: applyOutput.Azure.LoadBalancerName,
AttestationURL: applyOutput.Azure.AttestationURL,
}
}
if applyOutput.GCP != nil {
infra.GCP = &state.GCP{
ProjectID: applyOutput.GCP.ProjectID,
IPCidrNode: applyOutput.GCP.IPCidrNode,
IPCidrPod: applyOutput.GCP.IPCidrPod,
}
}
return infra
}
// ShowCluster reads the state of Constellation cluster resources from Terraform.
func (c *Client) ShowCluster(ctx context.Context, provider cloudprovider.Provider) (ApplyOutput, error) {
tfState, err := c.tf.Show(ctx)

2
internal/constants/constants.go
View File

@ -76,6 +76,8 @@ const (
// ClusterIDsFilename filename that contains Constellation clusterID and IP.
ClusterIDsFilename = "constellation-id.json"
// StateFilename filename that contains the entire state of the Constellation cluster.
StateFilename = "constellation-state.yaml"
// ConfigFilename filename of Constellation config file.
ConfigFilename = "constellation-conf.yaml"
// LicenseFilename filename of Constellation license file.