constellation-lib: move kubecmd package usage (#2673)

* Reduce external dependencies of kubecmd package
* Add kubecmd wrapper to constellation-lib
* Update CLI code to use constellation-lib
* Move kubecmd package to subpackage of constellation-lib
* Initialise helm and kubecmd clients when kubeConfig is set

---------

Signed-off-by: Daniel Weiße <dw@edgeless.systems>
This commit is contained in:
Daniel Weiße 2023-12-05 16:23:31 +01:00 committed by GitHub
parent c07c333d3d
commit 3691defce7
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
22 changed files with 751 additions and 714 deletions

View file

@ -1,19 +1,14 @@
.golangci.yml @katexochen .golangci.yml @katexochen
/3rdparty/gcp-guest-agent @malt3 /3rdparty/gcp-guest-agent @malt3
/bazel @malt3
/bazel/ci @katexochen /bazel/ci @katexochen
/bazel/container @katexochen /bazel/container @katexochen
/bazel @malt3
/bazel/sh @katexochen /bazel/sh @katexochen
/bootstrapper @3u13r /bootstrapper @3u13r
/internal/cloudcmd @daniel-weisse /cli/internal/cloudcmd @daniel-weisse
/internal/cmd/upgrade* @derpsteb /cli/internal/cmd/upgrade* @derpsteb
/internal/featureset @malt3 /cli/internal/libvirt @daniel-weisse
/internal/helm @derpsteb /cli/internal/terraform @elchead
/internal/kubecmd @daniel-weisse
/internal/libvirt @daniel-weisse
/internal/terraform @elchead
/internal/upgrade @elchead
/internal/state @elchead
/csi @daniel-weisse /csi @daniel-weisse
/debugd @malt3 /debugd @malt3
/disk-mapper @daniel-weisse /disk-mapper @daniel-weisse
@ -21,8 +16,8 @@
/e2e @3u13r /e2e @3u13r
/hack/azure-snp-report-verify @derpsteb /hack/azure-snp-report-verify @derpsteb
/hack/bazel-deps-mirror @malt3 /hack/bazel-deps-mirror @malt3
/hack/cli-k8s-compatibility @derpsteb
/hack/clidocgen @thomasten /hack/clidocgen @thomasten
/hack/cli-k8s-compatibility @derpsteb
/hack/fetch-broken-e2e @katexochen /hack/fetch-broken-e2e @katexochen
/hack/gocoverage @katexochen /hack/gocoverage @katexochen
/hack/oci-pin @malt3 /hack/oci-pin @malt3
@ -38,11 +33,14 @@
/internal/cloud @3u13r /internal/cloud @3u13r
/internal/compatibility @derpsteb /internal/compatibility @derpsteb
/internal/config @derpsteb /internal/config @derpsteb
/internal/constellation/kubecmd @daniel-weisse
/internal/containerimage @malt3 /internal/containerimage @malt3
/internal/crypto @thomasten /internal/crypto @thomasten
/internal/cryptsetup @daniel-weisse /internal/cryptsetup @daniel-weisse
/internal/featureset @malt3
/internal/file @daniel-weisse /internal/file @daniel-weisse
/internal/grpc @thomasten /internal/grpc @thomasten
/internal/helm @derpsteb
/internal/imagefetcher @malt3 /internal/imagefetcher @malt3
/internal/installer @3u13r /internal/installer @3u13r
/internal/kms @daniel-weisse /internal/kms @daniel-weisse
@ -54,6 +52,7 @@
/internal/retry @katexochen /internal/retry @katexochen
/internal/semver @derpsteb /internal/semver @derpsteb
/internal/sigstore @elchead /internal/sigstore @elchead
/internal/state @elchead
/internal/staticupload @malt3 /internal/staticupload @malt3
/internal/versions @3u13r /internal/versions @3u13r
/joinservice @daniel-weisse /joinservice @daniel-weisse

View file

@ -73,14 +73,15 @@ go_library(
"//internal/config/migration", "//internal/config/migration",
"//internal/constants", "//internal/constants",
"//internal/constellation", "//internal/constellation",
"//internal/constellation/kubecmd",
"//internal/crypto", "//internal/crypto",
"//internal/featureset", "//internal/featureset",
"//internal/file", "//internal/file",
"//internal/grpc/dialer", "//internal/grpc/dialer",
"//internal/grpc/retry", "//internal/grpc/retry",
"//internal/helm", "//internal/helm",
"//internal/imagefetcher",
"//internal/kms/uri", "//internal/kms/uri",
"//internal/kubecmd",
# keep # keep
"//internal/license", "//internal/license",
"//internal/logger", "//internal/logger",
@ -164,6 +165,7 @@ go_test(
"//internal/config", "//internal/config",
"//internal/constants", "//internal/constants",
"//internal/constellation", "//internal/constellation",
"//internal/constellation/kubecmd",
"//internal/crypto", "//internal/crypto",
"//internal/crypto/testvector", "//internal/crypto/testvector",
"//internal/file", "//internal/file",
@ -172,7 +174,6 @@ go_test(
"//internal/grpc/testdialer", "//internal/grpc/testdialer",
"//internal/helm", "//internal/helm",
"//internal/kms/uri", "//internal/kms/uri",
"//internal/kubecmd",
"//internal/logger", "//internal/logger",
"//internal/semver", "//internal/semver",
"//internal/state", "//internal/state",

View file

@ -22,6 +22,7 @@ import (
"github.com/edgelesssys/constellation/v2/bootstrapper/initproto" "github.com/edgelesssys/constellation/v2/bootstrapper/initproto"
"github.com/edgelesssys/constellation/v2/cli/internal/cloudcmd" "github.com/edgelesssys/constellation/v2/cli/internal/cloudcmd"
"github.com/edgelesssys/constellation/v2/internal/api/attestationconfigapi" "github.com/edgelesssys/constellation/v2/internal/api/attestationconfigapi"
"github.com/edgelesssys/constellation/v2/internal/api/versionsapi"
"github.com/edgelesssys/constellation/v2/internal/atls" "github.com/edgelesssys/constellation/v2/internal/atls"
"github.com/edgelesssys/constellation/v2/internal/attestation/variant" "github.com/edgelesssys/constellation/v2/internal/attestation/variant"
"github.com/edgelesssys/constellation/v2/internal/cloud/cloudprovider" "github.com/edgelesssys/constellation/v2/internal/cloud/cloudprovider"
@ -29,16 +30,19 @@ import (
"github.com/edgelesssys/constellation/v2/internal/config" "github.com/edgelesssys/constellation/v2/internal/config"
"github.com/edgelesssys/constellation/v2/internal/constants" "github.com/edgelesssys/constellation/v2/internal/constants"
"github.com/edgelesssys/constellation/v2/internal/constellation" "github.com/edgelesssys/constellation/v2/internal/constellation"
"github.com/edgelesssys/constellation/v2/internal/constellation/kubecmd"
"github.com/edgelesssys/constellation/v2/internal/file" "github.com/edgelesssys/constellation/v2/internal/file"
"github.com/edgelesssys/constellation/v2/internal/grpc/dialer" "github.com/edgelesssys/constellation/v2/internal/grpc/dialer"
"github.com/edgelesssys/constellation/v2/internal/helm" "github.com/edgelesssys/constellation/v2/internal/helm"
"github.com/edgelesssys/constellation/v2/internal/imagefetcher"
"github.com/edgelesssys/constellation/v2/internal/kms/uri" "github.com/edgelesssys/constellation/v2/internal/kms/uri"
"github.com/edgelesssys/constellation/v2/internal/kubecmd" "github.com/edgelesssys/constellation/v2/internal/semver"
"github.com/edgelesssys/constellation/v2/internal/state" "github.com/edgelesssys/constellation/v2/internal/state"
"github.com/edgelesssys/constellation/v2/internal/versions" "github.com/edgelesssys/constellation/v2/internal/versions"
"github.com/spf13/afero" "github.com/spf13/afero"
"github.com/spf13/cobra" "github.com/spf13/cobra"
"github.com/spf13/pflag" "github.com/spf13/pflag"
apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
k8serrors "k8s.io/apimachinery/pkg/api/errors" k8serrors "k8s.io/apimachinery/pkg/api/errors"
) )
@ -225,13 +229,7 @@ func runApply(cmd *cobra.Command, _ []string) error {
newDialer := func(validator atls.Validator) *dialer.Dialer { newDialer := func(validator atls.Validator) *dialer.Dialer {
return dialer.New(nil, validator, &net.Dialer{}) return dialer.New(nil, validator, &net.Dialer{})
} }
newKubeUpgrader := func(w io.Writer, kubeConfigPath string, log debugLog) (kubernetesUpgrader, error) {
kubeConfig, err := fileHandler.Read(kubeConfigPath)
if err != nil {
return nil, fmt.Errorf("reading kubeconfig: %w", err)
}
return kubecmd.New(w, kubeConfig, fileHandler, log)
}
newHelmClient := func(kubeConfigPath string, log debugLog) (helmApplier, error) { newHelmClient := func(kubeConfigPath string, log debugLog) (helmApplier, error) {
kubeConfig, err := fileHandler.Read(kubeConfigPath) kubeConfig, err := fileHandler.Read(kubeConfigPath)
if err != nil { if err != nil {
@ -264,9 +262,8 @@ func runApply(cmd *cobra.Command, _ []string) error {
spinner: spinner, spinner: spinner,
merger: &kubeconfigMerger{log: log}, merger: &kubeconfigMerger{log: log},
newHelmClient: newHelmClient, newHelmClient: newHelmClient,
newDialer: newDialer,
newKubeUpgrader: newKubeUpgrader,
newInfraApplier: newInfraApplier, newInfraApplier: newInfraApplier,
imageFetcher: imagefetcher.New(),
applier: applier, applier: applier,
} }
@ -287,15 +284,15 @@ type applyCmd struct {
merger configMerger merger configMerger
applier applier imageFetcher imageFetcher
applier applier
newHelmClient func(kubeConfigPath string, log debugLog) (helmApplier, error) newHelmClient func(kubeConfigPath string, log debugLog) (helmApplier, error)
newDialer func(validator atls.Validator) *dialer.Dialer
newKubeUpgrader func(out io.Writer, kubeConfigPath string, log debugLog) (kubernetesUpgrader, error)
newInfraApplier func(context.Context) (cloudApplier, func(), error) newInfraApplier func(context.Context) (cloudApplier, func(), error)
} }
type applier interface { type applier interface {
SetKubeConfig(kubeConfig []byte) error
CheckLicense(ctx context.Context, csp cloudprovider.Provider, licenseID string) (int, error) CheckLicense(ctx context.Context, csp cloudprovider.Provider, licenseID string) (int, error)
GenerateMasterSecret() (uri.MasterSecret, error) GenerateMasterSecret() (uri.MasterSecret, error)
GenerateMeasurementSalt() ([]byte, error) GenerateMeasurementSalt() ([]byte, error)
@ -309,6 +306,13 @@ type applier interface {
*initproto.InitSuccessResponse, *initproto.InitSuccessResponse,
error, error,
) )
ExtendClusterConfigCertSANs(ctx context.Context, clusterEndpoint, customEndpoint string, additionalAPIServerCertSANs []string) error
GetClusterAttestationConfig(ctx context.Context, variant variant.Variant) (config.AttestationCfg, error)
ApplyJoinConfig(ctx context.Context, newAttestConfig config.AttestationCfg, measurementSalt []byte) error
UpgradeNodeImage(ctx context.Context, imageVersion semver.Semver, imageReference string, force bool) error
UpgradeKubernetesVersion(ctx context.Context, kubernetesVersion versions.ValidK8sVersion, force bool) error
BackupCRDs(ctx context.Context, fileHandler file.Handler, upgradeDir string) ([]apiextensionsv1.CustomResourceDefinition, error)
BackupCRs(ctx context.Context, fileHandler file.Handler, crds []apiextensionsv1.CustomResourceDefinition, upgradeDir string) error
} }
type warnLog interface { type warnLog interface {
@ -415,42 +419,57 @@ func (a *applyCmd) apply(
} }
} }
if a.flags.skipPhases.contains(skipAttestationConfigPhase, skipCertSANsPhase, skipHelmPhase, skipK8sPhase, skipImagePhase) {
cmd.Print(bufferedOutput.String())
return nil
}
// From now on we can assume a valid Kubernetes admin config file exists // From now on we can assume a valid Kubernetes admin config file exists
var kubeUpgrader kubernetesUpgrader kubeConfig, err := a.fileHandler.Read(constants.AdminConfFilename)
if !a.flags.skipPhases.contains(skipAttestationConfigPhase, skipCertSANsPhase, skipHelmPhase, skipK8sPhase, skipImagePhase) { if err != nil {
a.log.Debugf("Creating Kubernetes client using %s", a.flags.pathPrefixer.PrefixPrintablePath(constants.AdminConfFilename)) return fmt.Errorf("reading kubeconfig: %w", err)
kubeUpgrader, err = a.newKubeUpgrader(cmd.OutOrStdout(), constants.AdminConfFilename, a.log) }
if err != nil { if err := a.applier.SetKubeConfig(kubeConfig); err != nil {
return err return err
}
} }
// Apply Attestation Config // Apply Attestation Config
if !a.flags.skipPhases.contains(skipAttestationConfigPhase) { if !a.flags.skipPhases.contains(skipAttestationConfigPhase) {
a.log.Debugf("Applying new attestation config to cluster") a.log.Debugf("Applying new attestation config to cluster")
if err := a.applyJoinConfig(cmd, kubeUpgrader, conf.GetAttestationConfig(), stateFile.ClusterValues.MeasurementSalt); err != nil { if err := a.applyJoinConfig(cmd, conf.GetAttestationConfig(), stateFile.ClusterValues.MeasurementSalt); err != nil {
return fmt.Errorf("applying attestation config: %w", err) return fmt.Errorf("applying attestation config: %w", err)
} }
} }
// Extend API Server Cert SANs // Extend API Server Cert SANs
if !a.flags.skipPhases.contains(skipCertSANsPhase) { if !a.flags.skipPhases.contains(skipCertSANsPhase) {
sans := append([]string{stateFile.Infrastructure.ClusterEndpoint, conf.CustomEndpoint}, stateFile.Infrastructure.APIServerCertSANs...) if err := a.applier.ExtendClusterConfigCertSANs(
if err := kubeUpgrader.ExtendClusterConfigCertSANs(cmd.Context(), sans); err != nil { cmd.Context(),
stateFile.Infrastructure.ClusterEndpoint,
conf.CustomEndpoint,
stateFile.Infrastructure.APIServerCertSANs,
); err != nil {
return fmt.Errorf("extending cert SANs: %w", err) return fmt.Errorf("extending cert SANs: %w", err)
} }
} }
// Apply Helm Charts // Apply Helm Charts
if !a.flags.skipPhases.contains(skipHelmPhase) { if !a.flags.skipPhases.contains(skipHelmPhase) {
if err := a.runHelmApply(cmd, conf, stateFile, kubeUpgrader, upgradeDir); err != nil { if err := a.runHelmApply(cmd, conf, stateFile, upgradeDir); err != nil {
return err return err
} }
} }
// Upgrade NodeVersion object // Upgrade node image
if !(a.flags.skipPhases.contains(skipK8sPhase, skipImagePhase)) { if !a.flags.skipPhases.contains(skipImagePhase) {
if err := a.runK8sUpgrade(cmd, conf, kubeUpgrader); err != nil { if err := a.runNodeImageUpgrade(cmd, conf); err != nil {
return err
}
}
// Upgrade Kubernetes version
if !a.flags.skipPhases.contains(skipK8sPhase) {
if err := a.runK8sVersionUpgrade(cmd, conf); err != nil {
return err return err
} }
} }
@ -598,15 +617,14 @@ func (a *applyCmd) validateInputs(cmd *cobra.Command, configFetcher attestationc
// applyJoinConfig creates or updates the cluster's join config. // applyJoinConfig creates or updates the cluster's join config.
// If the config already exists, and is different from the new config, the user is asked to confirm the upgrade. // If the config already exists, and is different from the new config, the user is asked to confirm the upgrade.
func (a *applyCmd) applyJoinConfig( func (a *applyCmd) applyJoinConfig(cmd *cobra.Command, newConfig config.AttestationCfg, measurementSalt []byte,
cmd *cobra.Command, kubeUpgrader kubernetesUpgrader, newConfig config.AttestationCfg, measurementSalt []byte,
) error { ) error {
clusterAttestationConfig, err := kubeUpgrader.GetClusterAttestationConfig(cmd.Context(), newConfig.GetVariant()) clusterAttestationConfig, err := a.applier.GetClusterAttestationConfig(cmd.Context(), newConfig.GetVariant())
if err != nil { if err != nil {
a.log.Debugf("Getting cluster attestation config failed: %s", err) a.log.Debugf("Getting cluster attestation config failed: %s", err)
if k8serrors.IsNotFound(err) { if k8serrors.IsNotFound(err) {
a.log.Debugf("Creating new join config") a.log.Debugf("Creating new join config")
return kubeUpgrader.ApplyJoinConfig(cmd.Context(), newConfig, measurementSalt) return a.applier.ApplyJoinConfig(cmd.Context(), newConfig, measurementSalt)
} }
return fmt.Errorf("getting cluster attestation config: %w", err) return fmt.Errorf("getting cluster attestation config: %w", err)
} }
@ -638,7 +656,7 @@ func (a *applyCmd) applyJoinConfig(
} }
} }
if err := kubeUpgrader.ApplyJoinConfig(cmd.Context(), newConfig, measurementSalt); err != nil { if err := a.applier.ApplyJoinConfig(cmd.Context(), newConfig, measurementSalt); err != nil {
return fmt.Errorf("updating attestation config: %w", err) return fmt.Errorf("updating attestation config: %w", err)
} }
cmd.Println("Successfully updated the cluster's attestation config") cmd.Println("Successfully updated the cluster's attestation config")
@ -646,19 +664,29 @@ func (a *applyCmd) applyJoinConfig(
return nil return nil
} }
// runK8sUpgrade upgrades image and Kubernetes version of the Constellation cluster. func (a *applyCmd) runNodeImageUpgrade(cmd *cobra.Command, conf *config.Config) error {
func (a *applyCmd) runK8sUpgrade(cmd *cobra.Command, conf *config.Config, kubeUpgrader kubernetesUpgrader, provider := conf.GetProvider()
) error { attestationVariant := conf.GetAttestationConfig().GetVariant()
err := kubeUpgrader.UpgradeNodeVersion( region := conf.GetRegion()
cmd.Context(), conf, a.flags.force, imageReference, err := a.imageFetcher.FetchReference(cmd.Context(), provider, attestationVariant, conf.Image, region)
a.flags.skipPhases.contains(skipImagePhase), if err != nil {
a.flags.skipPhases.contains(skipK8sPhase), return fmt.Errorf("fetching image reference: %w", err)
) }
imageVersionInfo, err := versionsapi.NewVersionFromShortPath(conf.Image, versionsapi.VersionKindImage)
if err != nil {
return fmt.Errorf("parsing version from image short path: %w", err)
}
imageVersion, err := semver.New(imageVersionInfo.Version())
if err != nil {
return fmt.Errorf("parsing image version: %w", err)
}
err = a.applier.UpgradeNodeImage(cmd.Context(), imageVersion, imageReference, a.flags.force)
var upgradeErr *compatibility.InvalidUpgradeError var upgradeErr *compatibility.InvalidUpgradeError
switch { switch {
case errors.Is(err, kubecmd.ErrInProgress): case errors.Is(err, kubecmd.ErrInProgress):
cmd.PrintErrln("Skipping image and Kubernetes upgrades. Another upgrade is in progress.") cmd.PrintErrln("Skipping image upgrade: Another upgrade is already in progress.")
case errors.As(err, &upgradeErr): case errors.As(err, &upgradeErr):
cmd.PrintErrln(err) cmd.PrintErrln(err)
case err != nil: case err != nil:
@ -668,6 +696,19 @@ func (a *applyCmd) runK8sUpgrade(cmd *cobra.Command, conf *config.Config, kubeUp
return nil return nil
} }
func (a *applyCmd) runK8sVersionUpgrade(cmd *cobra.Command, conf *config.Config) error {
err := a.applier.UpgradeKubernetesVersion(cmd.Context(), conf.KubernetesVersion, a.flags.force)
var upgradeErr *compatibility.InvalidUpgradeError
switch {
case errors.As(err, &upgradeErr):
cmd.PrintErrln(err)
case err != nil:
return fmt.Errorf("upgrading Kubernetes version: %w", err)
}
return nil
}
// checkCreateFilesClean ensures that the workspace is clean before creating a new cluster. // checkCreateFilesClean ensures that the workspace is clean before creating a new cluster.
func (a *applyCmd) checkCreateFilesClean() error { func (a *applyCmd) checkCreateFilesClean() error {
if err := a.checkInitFilesClean(); err != nil { if err := a.checkInitFilesClean(); err != nil {
@ -803,3 +844,11 @@ func (wl warnLogger) Infof(fmtStr string, args ...any) {
func (wl warnLogger) Warnf(fmtStr string, args ...any) { func (wl warnLogger) Warnf(fmtStr string, args ...any) {
wl.cmd.PrintErrf("Warning: %s\n", fmt.Sprintf(fmtStr, args...)) wl.cmd.PrintErrf("Warning: %s\n", fmt.Sprintf(fmtStr, args...))
} }
// imageFetcher gets an image reference from the versionsapi.
type imageFetcher interface {
FetchReference(ctx context.Context,
provider cloudprovider.Provider, attestationVariant variant.Variant,
image, region string,
) (string, error)
}

View file

@ -193,10 +193,11 @@ func TestBackupHelmCharts(t *testing.T) {
a := applyCmd{ a := applyCmd{
fileHandler: file.NewHandler(afero.NewMemMapFs()), fileHandler: file.NewHandler(afero.NewMemMapFs()),
applier: &stubConstellApplier{stubKubernetesUpgrader: tc.backupClient},
log: logger.NewTest(t), log: logger.NewTest(t),
} }
err := a.backupHelmCharts(context.Background(), tc.backupClient, tc.helmApplier, tc.includesUpgrades, "") err := a.backupHelmCharts(context.Background(), tc.helmApplier, tc.includesUpgrades, "")
if tc.wantErr { if tc.wantErr {
assert.Error(err) assert.Error(err)
return return
@ -494,23 +495,29 @@ func newPhases(phases ...skipPhase) skipPhases {
type stubConstellApplier struct { type stubConstellApplier struct {
checkLicenseErr error checkLicenseErr error
masterSecret uri.MasterSecret
measurementSalt []byte
generateMasterSecretErr error generateMasterSecretErr error
generateMeasurementSaltErr error generateMeasurementSaltErr error
initErr error initErr error
initResponse *initproto.InitSuccessResponse
*stubKubernetesUpgrader
} }
func (s *stubConstellApplier) SetKubeConfig([]byte) error { return nil }
func (s *stubConstellApplier) CheckLicense(context.Context, cloudprovider.Provider, string) (int, error) { func (s *stubConstellApplier) CheckLicense(context.Context, cloudprovider.Provider, string) (int, error) {
return 0, s.checkLicenseErr return 0, s.checkLicenseErr
} }
func (s *stubConstellApplier) GenerateMasterSecret() (uri.MasterSecret, error) { func (s *stubConstellApplier) GenerateMasterSecret() (uri.MasterSecret, error) {
return uri.MasterSecret{}, s.generateMasterSecretErr return s.masterSecret, s.generateMasterSecretErr
} }
func (s *stubConstellApplier) GenerateMeasurementSalt() ([]byte, error) { func (s *stubConstellApplier) GenerateMeasurementSalt() ([]byte, error) {
return nil, s.generateMeasurementSaltErr return s.measurementSalt, s.generateMeasurementSaltErr
} }
func (s *stubConstellApplier) Init(context.Context, atls.Validator, *state.State, io.Writer, constellation.InitPayload) (*initproto.InitSuccessResponse, error) { func (s *stubConstellApplier) Init(context.Context, atls.Validator, *state.State, io.Writer, constellation.InitPayload) (*initproto.InitSuccessResponse, error) {
return nil, s.initErr return s.initResponse, s.initErr
} }

View file

@ -23,9 +23,7 @@ import (
) )
// runHelmApply handles installing or upgrading helm charts for the cluster. // runHelmApply handles installing or upgrading helm charts for the cluster.
func (a *applyCmd) runHelmApply( func (a *applyCmd) runHelmApply(cmd *cobra.Command, conf *config.Config, stateFile *state.State, upgradeDir string,
cmd *cobra.Command, conf *config.Config, stateFile *state.State,
kubeUpgrader kubernetesUpgrader, upgradeDir string,
) error { ) error {
a.log.Debugf("Installing or upgrading Helm charts") a.log.Debugf("Installing or upgrading Helm charts")
var masterSecret uri.MasterSecret var masterSecret uri.MasterSecret
@ -80,7 +78,7 @@ func (a *applyCmd) runHelmApply(
} }
a.log.Debugf("Backing up Helm charts") a.log.Debugf("Backing up Helm charts")
if err := a.backupHelmCharts(cmd.Context(), kubeUpgrader, executor, includesUpgrades, upgradeDir); err != nil { if err := a.backupHelmCharts(cmd.Context(), executor, includesUpgrades, upgradeDir); err != nil {
return err return err
} }
@ -105,7 +103,7 @@ func (a *applyCmd) runHelmApply(
// backupHelmCharts saves the Helm charts for the upgrade to disk and creates a backup of existing CRDs and CRs. // backupHelmCharts saves the Helm charts for the upgrade to disk and creates a backup of existing CRDs and CRs.
func (a *applyCmd) backupHelmCharts( func (a *applyCmd) backupHelmCharts(
ctx context.Context, kubeUpgrader kubernetesUpgrader, executor helm.Applier, includesUpgrades bool, upgradeDir string, ctx context.Context, executor helm.Applier, includesUpgrades bool, upgradeDir string,
) error { ) error {
// Save the Helm charts for the upgrade to disk // Save the Helm charts for the upgrade to disk
chartDir := filepath.Join(upgradeDir, "helm-charts") chartDir := filepath.Join(upgradeDir, "helm-charts")
@ -116,11 +114,11 @@ func (a *applyCmd) backupHelmCharts(
if includesUpgrades { if includesUpgrades {
a.log.Debugf("Creating backup of CRDs and CRs") a.log.Debugf("Creating backup of CRDs and CRs")
crds, err := kubeUpgrader.BackupCRDs(ctx, upgradeDir) crds, err := a.applier.BackupCRDs(ctx, a.fileHandler, upgradeDir)
if err != nil { if err != nil {
return fmt.Errorf("creating CRD backup: %w", err) return fmt.Errorf("creating CRD backup: %w", err)
} }
if err := kubeUpgrader.BackupCRs(ctx, crds, upgradeDir); err != nil { if err := a.applier.BackupCRs(ctx, a.fileHandler, crds, upgradeDir); err != nil {
return fmt.Errorf("creating CR backup: %w", err) return fmt.Errorf("creating CR backup: %w", err)
} }
} }

View file

@ -10,19 +10,13 @@ import (
"bytes" "bytes"
"context" "context"
"encoding/hex" "encoding/hex"
"encoding/json"
"errors"
"fmt" "fmt"
"io"
"net"
"strconv"
"strings" "strings"
"testing" "testing"
"time" "time"
"github.com/edgelesssys/constellation/v2/bootstrapper/initproto" "github.com/edgelesssys/constellation/v2/bootstrapper/initproto"
"github.com/edgelesssys/constellation/v2/cli/internal/cmd/pathprefix" "github.com/edgelesssys/constellation/v2/cli/internal/cmd/pathprefix"
"github.com/edgelesssys/constellation/v2/internal/atls"
"github.com/edgelesssys/constellation/v2/internal/attestation/measurements" "github.com/edgelesssys/constellation/v2/internal/attestation/measurements"
"github.com/edgelesssys/constellation/v2/internal/attestation/variant" "github.com/edgelesssys/constellation/v2/internal/attestation/variant"
"github.com/edgelesssys/constellation/v2/internal/cloud/cloudprovider" "github.com/edgelesssys/constellation/v2/internal/cloud/cloudprovider"
@ -31,9 +25,6 @@ import (
"github.com/edgelesssys/constellation/v2/internal/constants" "github.com/edgelesssys/constellation/v2/internal/constants"
"github.com/edgelesssys/constellation/v2/internal/constellation" "github.com/edgelesssys/constellation/v2/internal/constellation"
"github.com/edgelesssys/constellation/v2/internal/file" "github.com/edgelesssys/constellation/v2/internal/file"
"github.com/edgelesssys/constellation/v2/internal/grpc/atlscredentials"
"github.com/edgelesssys/constellation/v2/internal/grpc/dialer"
"github.com/edgelesssys/constellation/v2/internal/grpc/testdialer"
"github.com/edgelesssys/constellation/v2/internal/helm" "github.com/edgelesssys/constellation/v2/internal/helm"
"github.com/edgelesssys/constellation/v2/internal/kms/uri" "github.com/edgelesssys/constellation/v2/internal/kms/uri"
"github.com/edgelesssys/constellation/v2/internal/logger" "github.com/edgelesssys/constellation/v2/internal/logger"
@ -43,7 +34,6 @@ import (
"github.com/spf13/afero" "github.com/spf13/afero"
"github.com/stretchr/testify/assert" "github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require" "github.com/stretchr/testify/require"
"google.golang.org/grpc"
k8serrors "k8s.io/apimachinery/pkg/api/errors" k8serrors "k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/client-go/tools/clientcmd" "k8s.io/client-go/tools/clientcmd"
@ -102,7 +92,8 @@ func TestInitialize(t *testing.T) {
stateFile *state.State stateFile *state.State
configMutator func(*config.Config) configMutator func(*config.Config)
serviceAccKey *gcpshared.ServiceAccountKey serviceAccKey *gcpshared.ServiceAccountKey
initServerAPI *stubInitServer initResponse *initproto.InitSuccessResponse
initErr error
retriable bool retriable bool
masterSecretShouldExist bool masterSecretShouldExist bool
wantErr bool wantErr bool
@ -112,47 +103,30 @@ func TestInitialize(t *testing.T) {
stateFile: preInitStateFile(cloudprovider.GCP), stateFile: preInitStateFile(cloudprovider.GCP),
configMutator: func(c *config.Config) { c.Provider.GCP.ServiceAccountKeyPath = serviceAccPath }, configMutator: func(c *config.Config) { c.Provider.GCP.ServiceAccountKeyPath = serviceAccPath },
serviceAccKey: gcpServiceAccKey, serviceAccKey: gcpServiceAccKey,
initServerAPI: &stubInitServer{res: []*initproto.InitResponse{{Kind: &initproto.InitResponse_InitSuccess{InitSuccess: testInitResp}}}}, initResponse: testInitResp,
}, },
"initialize some azure instances": { "initialize some azure instances": {
provider: cloudprovider.Azure, provider: cloudprovider.Azure,
stateFile: preInitStateFile(cloudprovider.Azure), stateFile: preInitStateFile(cloudprovider.Azure),
initServerAPI: &stubInitServer{res: []*initproto.InitResponse{{Kind: &initproto.InitResponse_InitSuccess{InitSuccess: testInitResp}}}}, initResponse: testInitResp,
}, },
"initialize some qemu instances": { "initialize some qemu instances": {
provider: cloudprovider.QEMU, provider: cloudprovider.QEMU,
stateFile: preInitStateFile(cloudprovider.QEMU), stateFile: preInitStateFile(cloudprovider.QEMU),
initServerAPI: &stubInitServer{res: []*initproto.InitResponse{{Kind: &initproto.InitResponse_InitSuccess{InitSuccess: testInitResp}}}}, initResponse: testInitResp,
}, },
"non retriable error": { "non retriable error": {
provider: cloudprovider.QEMU, provider: cloudprovider.QEMU,
stateFile: preInitStateFile(cloudprovider.QEMU), stateFile: preInitStateFile(cloudprovider.QEMU),
initServerAPI: &stubInitServer{initErr: &constellation.NonRetriableInitError{Err: assert.AnError}}, initErr: &constellation.NonRetriableInitError{Err: assert.AnError},
retriable: false, retriable: false,
masterSecretShouldExist: true, masterSecretShouldExist: true,
wantErr: true, wantErr: true,
}, },
"non retriable error with failed log collection": { "non retriable error with failed log collection": {
provider: cloudprovider.QEMU, provider: cloudprovider.QEMU,
stateFile: preInitStateFile(cloudprovider.QEMU), stateFile: preInitStateFile(cloudprovider.QEMU),
initServerAPI: &stubInitServer{ initErr: &constellation.NonRetriableInitError{Err: assert.AnError, LogCollectionErr: assert.AnError},
res: []*initproto.InitResponse{
{
Kind: &initproto.InitResponse_InitFailure{
InitFailure: &initproto.InitFailureResponse{
Error: "error",
},
},
},
{
Kind: &initproto.InitResponse_InitFailure{
InitFailure: &initproto.InitFailureResponse{
Error: "error",
},
},
},
},
},
retriable: false, retriable: false,
masterSecretShouldExist: true, masterSecretShouldExist: true,
wantErr: true, wantErr: true,
@ -162,7 +136,7 @@ func TestInitialize(t *testing.T) {
stateFile: &state.State{Version: "invalid"}, stateFile: &state.State{Version: "invalid"},
configMutator: func(c *config.Config) { c.Provider.GCP.ServiceAccountKeyPath = serviceAccPath }, configMutator: func(c *config.Config) { c.Provider.GCP.ServiceAccountKeyPath = serviceAccPath },
serviceAccKey: gcpServiceAccKey, serviceAccKey: gcpServiceAccKey,
initServerAPI: &stubInitServer{}, initResponse: testInitResp,
retriable: true, retriable: true,
wantErr: true, wantErr: true,
}, },
@ -171,7 +145,7 @@ func TestInitialize(t *testing.T) {
stateFile: &state.State{}, stateFile: &state.State{},
configMutator: func(c *config.Config) { c.Provider.GCP.ServiceAccountKeyPath = serviceAccPath }, configMutator: func(c *config.Config) { c.Provider.GCP.ServiceAccountKeyPath = serviceAccPath },
serviceAccKey: gcpServiceAccKey, serviceAccKey: gcpServiceAccKey,
initServerAPI: &stubInitServer{}, initResponse: testInitResp,
retriable: true, retriable: true,
wantErr: true, wantErr: true,
}, },
@ -179,6 +153,7 @@ func TestInitialize(t *testing.T) {
provider: cloudprovider.GCP, provider: cloudprovider.GCP,
configMutator: func(c *config.Config) { c.Provider.GCP.ServiceAccountKeyPath = serviceAccPath }, configMutator: func(c *config.Config) { c.Provider.GCP.ServiceAccountKeyPath = serviceAccPath },
serviceAccKey: gcpServiceAccKey, serviceAccKey: gcpServiceAccKey,
initResponse: testInitResp,
retriable: true, retriable: true,
wantErr: true, wantErr: true,
}, },
@ -187,15 +162,15 @@ func TestInitialize(t *testing.T) {
configMutator: func(c *config.Config) { c.Provider.GCP.ServiceAccountKeyPath = serviceAccPath }, configMutator: func(c *config.Config) { c.Provider.GCP.ServiceAccountKeyPath = serviceAccPath },
stateFile: preInitStateFile(cloudprovider.GCP), stateFile: preInitStateFile(cloudprovider.GCP),
serviceAccKey: gcpServiceAccKey, serviceAccKey: gcpServiceAccKey,
initServerAPI: &stubInitServer{initErr: assert.AnError}, initErr: &constellation.NonRetriableInitError{Err: assert.AnError},
retriable: false, retriable: false,
masterSecretShouldExist: true, masterSecretShouldExist: true,
wantErr: true, wantErr: true,
}, },
"k8s version without v works": { "k8s version without v works": {
provider: cloudprovider.Azure, provider: cloudprovider.Azure,
stateFile: preInitStateFile(cloudprovider.Azure), stateFile: preInitStateFile(cloudprovider.Azure),
initServerAPI: &stubInitServer{res: []*initproto.InitResponse{{Kind: &initproto.InitResponse_InitSuccess{InitSuccess: testInitResp}}}}, initResponse: testInitResp,
configMutator: func(c *config.Config) { configMutator: func(c *config.Config) {
res, err := versions.NewValidK8sVersion(strings.TrimPrefix(string(versions.Default), "v"), true) res, err := versions.NewValidK8sVersion(strings.TrimPrefix(string(versions.Default), "v"), true)
require.NoError(t, err) require.NoError(t, err)
@ -203,9 +178,9 @@ func TestInitialize(t *testing.T) {
}, },
}, },
"outdated k8s patch version doesn't work": { "outdated k8s patch version doesn't work": {
provider: cloudprovider.Azure, provider: cloudprovider.Azure,
stateFile: preInitStateFile(cloudprovider.Azure), stateFile: preInitStateFile(cloudprovider.Azure),
initServerAPI: &stubInitServer{res: []*initproto.InitResponse{{Kind: &initproto.InitResponse_InitSuccess{InitSuccess: testInitResp}}}}, initResponse: testInitResp,
configMutator: func(c *config.Config) { configMutator: func(c *config.Config) {
v, err := semver.New(versions.SupportedK8sVersions()[0]) v, err := semver.New(versions.SupportedK8sVersions()[0])
require.NoError(t, err) require.NoError(t, err)
@ -221,18 +196,6 @@ func TestInitialize(t *testing.T) {
t.Run(name, func(t *testing.T) { t.Run(name, func(t *testing.T) {
assert := assert.New(t) assert := assert.New(t)
require := require.New(t) require := require.New(t)
// Networking
netDialer := testdialer.NewBufconnDialer()
newDialer := func(atls.Validator) *dialer.Dialer {
return dialer.New(nil, nil, netDialer)
}
serverCreds := atlscredentials.New(nil, nil)
initServer := grpc.NewServer(grpc.Creds(serverCreds))
initproto.RegisterAPIServer(initServer, tc.initServerAPI)
port := strconv.Itoa(constants.BootstrapperPort)
listener := netDialer.GetListener(net.JoinHostPort("192.0.2.1", port))
go initServer.Serve(listener)
defer initServer.GracefulStop()
// Command // Command
cmd := NewInitCmd() cmd := NewInitCmd()
@ -271,20 +234,21 @@ func TestInitialize(t *testing.T) {
spinner: &nopSpinner{}, spinner: &nopSpinner{},
merger: &stubMerger{}, merger: &stubMerger{},
newHelmClient: func(string, debugLog) (helmApplier, error) { newHelmClient: func(string, debugLog) (helmApplier, error) {
return &stubApplier{}, nil return &stubHelmApplier{}, nil
}, },
newDialer: newDialer, applier: &stubConstellApplier{
newKubeUpgrader: func(io.Writer, string, debugLog) (kubernetesUpgrader, error) { masterSecret: uri.MasterSecret{
return &stubKubernetesUpgrader{ Key: bytes.Repeat([]byte{0x01}, 32),
Salt: bytes.Repeat([]byte{0x02}, 32),
},
measurementSalt: bytes.Repeat([]byte{0x03}, 32),
initErr: tc.initErr,
initResponse: tc.initResponse,
stubKubernetesUpgrader: &stubKubernetesUpgrader{
// On init, no attestation config exists yet // On init, no attestation config exists yet
getClusterAttestationConfigErr: k8serrors.NewNotFound(schema.GroupResource{}, ""), getClusterAttestationConfigErr: k8serrors.NewNotFound(schema.GroupResource{}, ""),
}, nil },
}, },
applier: constellation.NewApplier(
logger.NewTest(t),
&nopSpinner{},
newDialer,
),
} }
err := i.apply(cmd, stubAttestationFetcher{}, "test") err := i.apply(cmd, stubAttestationFetcher{}, "test")
@ -314,11 +278,14 @@ func TestInitialize(t *testing.T) {
} }
} }
type stubApplier struct { type stubHelmApplier struct {
err error err error
} }
func (s stubApplier) PrepareApply(_ cloudprovider.Provider, _ variant.Variant, _ versions.ValidK8sVersion, _ semver.Semver, _ *state.State, _ helm.Options, _ string, _ uri.MasterSecret, _ *config.OpenStackConfig) (helm.Applier, bool, error) { func (s stubHelmApplier) PrepareApply(
_ cloudprovider.Provider, _ variant.Variant, _ versions.ValidK8sVersion, _ semver.Semver,
_ *state.State, _ helm.Options, _ string, _ uri.MasterSecret, _ *config.OpenStackConfig,
) (helm.Applier, bool, error) {
return stubRunner{}, false, s.err return stubRunner{}, false, s.err
} }
@ -513,153 +480,6 @@ func TestGenerateMasterSecret(t *testing.T) {
} }
} }
func TestAttestation(t *testing.T) {
assert := assert.New(t)
require := require.New(t)
initServerAPI := &stubInitServer{res: []*initproto.InitResponse{
{
Kind: &initproto.InitResponse_InitSuccess{
InitSuccess: &initproto.InitSuccessResponse{
Kubeconfig: []byte("kubeconfig"),
OwnerId: []byte("ownerID"),
ClusterId: []byte("clusterID"),
},
},
},
}}
existingStateFile := &state.State{Version: state.Version1, Infrastructure: state.Infrastructure{ClusterEndpoint: "192.0.2.4"}}
netDialer := testdialer.NewBufconnDialer()
issuer := &testIssuer{
Getter: variant.QEMUVTPM{},
pcrs: map[uint32][]byte{
0: bytes.Repeat([]byte{0xFF}, 32),
1: bytes.Repeat([]byte{0xFF}, 32),
2: bytes.Repeat([]byte{0xFF}, 32),
3: bytes.Repeat([]byte{0xFF}, 32),
},
}
serverCreds := atlscredentials.New(issuer, nil)
initServer := grpc.NewServer(grpc.Creds(serverCreds))
initproto.RegisterAPIServer(initServer, initServerAPI)
port := strconv.Itoa(constants.BootstrapperPort)
listener := netDialer.GetListener(net.JoinHostPort("192.0.2.4", port))
go initServer.Serve(listener)
defer initServer.GracefulStop()
cmd := NewInitCmd()
cmd.Flags().String("workspace", "", "") // register persistent flag manually
cmd.Flags().Bool("force", true, "") // register persistent flag manually
var out bytes.Buffer
cmd.SetOut(&out)
var errOut bytes.Buffer
cmd.SetErr(&errOut)
fs := afero.NewMemMapFs()
fileHandler := file.NewHandler(fs)
require.NoError(existingStateFile.WriteToFile(fileHandler, constants.StateFilename))
cfg := config.Default()
cfg.Image = constants.BinaryVersion().String()
cfg.RemoveProviderAndAttestationExcept(cloudprovider.QEMU)
cfg.Attestation.QEMUVTPM.Measurements[0] = measurements.WithAllBytes(0x00, measurements.Enforce, measurements.PCRMeasurementLength)
cfg.Attestation.QEMUVTPM.Measurements[1] = measurements.WithAllBytes(0x11, measurements.Enforce, measurements.PCRMeasurementLength)
cfg.Attestation.QEMUVTPM.Measurements[2] = measurements.WithAllBytes(0x22, measurements.Enforce, measurements.PCRMeasurementLength)
cfg.Attestation.QEMUVTPM.Measurements[3] = measurements.WithAllBytes(0x33, measurements.Enforce, measurements.PCRMeasurementLength)
cfg.Attestation.QEMUVTPM.Measurements[4] = measurements.WithAllBytes(0x44, measurements.Enforce, measurements.PCRMeasurementLength)
cfg.Attestation.QEMUVTPM.Measurements[9] = measurements.WithAllBytes(0x99, measurements.Enforce, measurements.PCRMeasurementLength)
cfg.Attestation.QEMUVTPM.Measurements[12] = measurements.WithAllBytes(0xcc, measurements.Enforce, measurements.PCRMeasurementLength)
require.NoError(fileHandler.WriteYAML(constants.ConfigFilename, cfg, file.OptNone))
newDialer := func(v atls.Validator) *dialer.Dialer {
validator := &testValidator{
Getter: variant.QEMUVTPM{},
pcrs: cfg.GetAttestationConfig().GetMeasurements(),
}
return dialer.New(nil, validator, netDialer)
}
ctx := context.Background()
ctx, cancel := context.WithTimeout(ctx, 4*time.Second)
defer cancel()
cmd.SetContext(ctx)
i := &applyCmd{
fileHandler: fileHandler,
spinner: &nopSpinner{},
merger: &stubMerger{},
log: logger.NewTest(t),
newKubeUpgrader: func(io.Writer, string, debugLog) (kubernetesUpgrader, error) {
return &stubKubernetesUpgrader{}, nil
},
newDialer: newDialer,
applier: constellation.NewApplier(logger.NewTest(t), &nopSpinner{}, newDialer),
}
_, err := i.runInit(cmd, cfg, existingStateFile)
assert.Error(err)
// make sure the error is actually a TLS handshake error
assert.Contains(err.Error(), "transport: authentication handshake failed")
if validationErr, ok := err.(*config.ValidationError); ok {
t.Log(validationErr.LongMessage())
}
}
type testValidator struct {
variant.Getter
pcrs measurements.M
}
func (v *testValidator) Validate(_ context.Context, attDoc []byte, _ []byte) ([]byte, error) {
var attestation struct {
UserData []byte
PCRs map[uint32][]byte
}
if err := json.Unmarshal(attDoc, &attestation); err != nil {
return nil, err
}
for k, pcr := range v.pcrs {
if !bytes.Equal(attestation.PCRs[k], pcr.Expected[:]) {
return nil, errors.New("invalid PCR value")
}
}
return attestation.UserData, nil
}
type testIssuer struct {
variant.Getter
pcrs map[uint32][]byte
}
func (i *testIssuer) Issue(_ context.Context, userData []byte, _ []byte) ([]byte, error) {
return json.Marshal(
struct {
UserData []byte
PCRs map[uint32][]byte
}{
UserData: userData,
PCRs: i.pcrs,
},
)
}
type stubInitServer struct {
res []*initproto.InitResponse
initErr error
initproto.UnimplementedAPIServer
}
func (s *stubInitServer) Init(_ *initproto.InitRequest, stream initproto.API_InitServer) error {
for _, r := range s.res {
_ = stream.Send(r)
}
return s.initErr
}
type stubMerger struct { type stubMerger struct {
envVar string envVar string
mergeErr error mergeErr error

View file

@ -16,9 +16,9 @@ import (
"github.com/edgelesssys/constellation/v2/internal/attestation/variant" "github.com/edgelesssys/constellation/v2/internal/attestation/variant"
"github.com/edgelesssys/constellation/v2/internal/config" "github.com/edgelesssys/constellation/v2/internal/config"
"github.com/edgelesssys/constellation/v2/internal/constants" "github.com/edgelesssys/constellation/v2/internal/constants"
"github.com/edgelesssys/constellation/v2/internal/constellation/kubecmd"
"github.com/edgelesssys/constellation/v2/internal/file" "github.com/edgelesssys/constellation/v2/internal/file"
"github.com/edgelesssys/constellation/v2/internal/helm" "github.com/edgelesssys/constellation/v2/internal/helm"
"github.com/edgelesssys/constellation/v2/internal/kubecmd"
"github.com/spf13/afero" "github.com/spf13/afero"
"github.com/spf13/cobra" "github.com/spf13/cobra"
"gopkg.in/yaml.v3" "gopkg.in/yaml.v3"
@ -61,7 +61,7 @@ func runStatus(cmd *cobra.Command, _ []string) error {
} }
fetcher := attestationconfigapi.NewFetcher() fetcher := attestationconfigapi.NewFetcher()
kubeClient, err := kubecmd.New(cmd.OutOrStdout(), kubeConfig, fileHandler, log) kubeClient, err := kubecmd.New(kubeConfig, log)
if err != nil { if err != nil {
return fmt.Errorf("setting up kubernetes client: %w", err) return fmt.Errorf("setting up kubernetes client: %w", err)
} }

View file

@ -17,8 +17,8 @@ import (
"github.com/edgelesssys/constellation/v2/internal/cloud/cloudprovider" "github.com/edgelesssys/constellation/v2/internal/cloud/cloudprovider"
"github.com/edgelesssys/constellation/v2/internal/config" "github.com/edgelesssys/constellation/v2/internal/config"
"github.com/edgelesssys/constellation/v2/internal/constants" "github.com/edgelesssys/constellation/v2/internal/constants"
"github.com/edgelesssys/constellation/v2/internal/constellation/kubecmd"
"github.com/edgelesssys/constellation/v2/internal/file" "github.com/edgelesssys/constellation/v2/internal/file"
"github.com/edgelesssys/constellation/v2/internal/kubecmd"
updatev1alpha1 "github.com/edgelesssys/constellation/v2/operators/constellation-node-operator/v2/api/v1alpha1" updatev1alpha1 "github.com/edgelesssys/constellation/v2/operators/constellation-node-operator/v2/api/v1alpha1"
"github.com/spf13/afero" "github.com/spf13/afero"
"github.com/stretchr/testify/assert" "github.com/stretchr/testify/assert"

View file

@ -7,16 +7,13 @@ SPDX-License-Identifier: AGPL-3.0-only
package cmd package cmd
import ( import (
"context"
"fmt" "fmt"
"time" "time"
"github.com/edgelesssys/constellation/v2/internal/attestation/variant"
"github.com/edgelesssys/constellation/v2/internal/config" "github.com/edgelesssys/constellation/v2/internal/config"
"github.com/rogpeppe/go-internal/diff" "github.com/rogpeppe/go-internal/diff"
"github.com/spf13/cobra" "github.com/spf13/cobra"
"gopkg.in/yaml.v3" "gopkg.in/yaml.v3"
apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
) )
func newUpgradeApplyCmd() *cobra.Command { func newUpgradeApplyCmd() *cobra.Command {
@ -60,12 +57,3 @@ func diffAttestationCfg(currentAttestationCfg config.AttestationCfg, newAttestat
diff := string(diff.Diff("current", currentYml, "new", newYml)) diff := string(diff.Diff("current", currentYml, "new", newYml))
return diff, nil return diff, nil
} }
type kubernetesUpgrader interface {
UpgradeNodeVersion(ctx context.Context, conf *config.Config, force, skipImage, skipK8s bool) error
ExtendClusterConfigCertSANs(ctx context.Context, alternativeNames []string) error
GetClusterAttestationConfig(ctx context.Context, variant variant.Variant) (config.AttestationCfg, error)
ApplyJoinConfig(ctx context.Context, newAttestConfig config.AttestationCfg, measurementSalt []byte) error
BackupCRs(ctx context.Context, crds []apiextensionsv1.CustomResourceDefinition, upgradeDir string) error
BackupCRDs(ctx context.Context, upgradeDir string) ([]apiextensionsv1.CustomResourceDefinition, error)
}

View file

@ -9,7 +9,6 @@ package cmd
import ( import (
"bytes" "bytes"
"context" "context"
"io"
"testing" "testing"
"github.com/edgelesssys/constellation/v2/cli/internal/cloudcmd" "github.com/edgelesssys/constellation/v2/cli/internal/cloudcmd"
@ -17,10 +16,10 @@ import (
"github.com/edgelesssys/constellation/v2/internal/cloud/cloudprovider" "github.com/edgelesssys/constellation/v2/internal/cloud/cloudprovider"
"github.com/edgelesssys/constellation/v2/internal/config" "github.com/edgelesssys/constellation/v2/internal/config"
"github.com/edgelesssys/constellation/v2/internal/constants" "github.com/edgelesssys/constellation/v2/internal/constants"
"github.com/edgelesssys/constellation/v2/internal/constellation/kubecmd"
"github.com/edgelesssys/constellation/v2/internal/file" "github.com/edgelesssys/constellation/v2/internal/file"
"github.com/edgelesssys/constellation/v2/internal/helm" "github.com/edgelesssys/constellation/v2/internal/helm"
"github.com/edgelesssys/constellation/v2/internal/kms/uri" "github.com/edgelesssys/constellation/v2/internal/kms/uri"
"github.com/edgelesssys/constellation/v2/internal/kubecmd"
"github.com/edgelesssys/constellation/v2/internal/logger" "github.com/edgelesssys/constellation/v2/internal/logger"
"github.com/edgelesssys/constellation/v2/internal/semver" "github.com/edgelesssys/constellation/v2/internal/semver"
"github.com/edgelesssys/constellation/v2/internal/state" "github.com/edgelesssys/constellation/v2/internal/state"
@ -46,6 +45,7 @@ func TestUpgradeApply(t *testing.T) {
fh func() file.Handler fh func() file.Handler
fhAssertions func(require *require.Assertions, assert *assert.Assertions, fh file.Handler) fhAssertions func(require *require.Assertions, assert *assert.Assertions, fh file.Handler)
terraformUpgrader cloudApplier terraformUpgrader cloudApplier
fetchImageErr error
wantErr bool wantErr bool
customK8sVersion string customK8sVersion string
flags applyFlags flags applyFlags
@ -53,7 +53,7 @@ func TestUpgradeApply(t *testing.T) {
}{ }{
"success": { "success": {
kubeUpgrader: &stubKubernetesUpgrader{currentConfig: config.DefaultForAzureSEVSNP()}, kubeUpgrader: &stubKubernetesUpgrader{currentConfig: config.DefaultForAzureSEVSNP()},
helmUpgrader: stubApplier{}, helmUpgrader: stubHelmApplier{},
terraformUpgrader: &stubTerraformUpgrader{}, terraformUpgrader: &stubTerraformUpgrader{},
flags: applyFlags{yes: true, skipPhases: skipPhases{skipInitPhase: struct{}{}}}, flags: applyFlags{yes: true, skipPhases: skipPhases{skipInitPhase: struct{}{}}},
fh: fsWithStateFileAndTfState, fh: fsWithStateFileAndTfState,
@ -66,7 +66,7 @@ func TestUpgradeApply(t *testing.T) {
}, },
"id file and state file do not exist": { "id file and state file do not exist": {
kubeUpgrader: &stubKubernetesUpgrader{currentConfig: config.DefaultForAzureSEVSNP()}, kubeUpgrader: &stubKubernetesUpgrader{currentConfig: config.DefaultForAzureSEVSNP()},
helmUpgrader: stubApplier{}, helmUpgrader: stubHelmApplier{},
terraformUpgrader: &stubTerraformUpgrader{}, terraformUpgrader: &stubTerraformUpgrader{},
flags: applyFlags{yes: true, skipPhases: skipPhases{skipInitPhase: struct{}{}}}, flags: applyFlags{yes: true, skipPhases: skipPhases{skipInitPhase: struct{}{}}},
fh: func() file.Handler { fh: func() file.Handler {
@ -79,7 +79,7 @@ func TestUpgradeApply(t *testing.T) {
currentConfig: config.DefaultForAzureSEVSNP(), currentConfig: config.DefaultForAzureSEVSNP(),
nodeVersionErr: assert.AnError, nodeVersionErr: assert.AnError,
}, },
helmUpgrader: stubApplier{}, helmUpgrader: stubHelmApplier{},
terraformUpgrader: &stubTerraformUpgrader{}, terraformUpgrader: &stubTerraformUpgrader{},
wantErr: true, wantErr: true,
flags: applyFlags{yes: true, skipPhases: skipPhases{skipInitPhase: struct{}{}}}, flags: applyFlags{yes: true, skipPhases: skipPhases{skipInitPhase: struct{}{}}},
@ -90,7 +90,7 @@ func TestUpgradeApply(t *testing.T) {
currentConfig: config.DefaultForAzureSEVSNP(), currentConfig: config.DefaultForAzureSEVSNP(),
nodeVersionErr: kubecmd.ErrInProgress, nodeVersionErr: kubecmd.ErrInProgress,
}, },
helmUpgrader: stubApplier{}, helmUpgrader: stubHelmApplier{},
terraformUpgrader: &stubTerraformUpgrader{}, terraformUpgrader: &stubTerraformUpgrader{},
flags: applyFlags{yes: true, skipPhases: skipPhases{skipInitPhase: struct{}{}}}, flags: applyFlags{yes: true, skipPhases: skipPhases{skipInitPhase: struct{}{}}},
fh: fsWithStateFileAndTfState, fh: fsWithStateFileAndTfState,
@ -99,7 +99,7 @@ func TestUpgradeApply(t *testing.T) {
kubeUpgrader: &stubKubernetesUpgrader{ kubeUpgrader: &stubKubernetesUpgrader{
currentConfig: config.DefaultForAzureSEVSNP(), currentConfig: config.DefaultForAzureSEVSNP(),
}, },
helmUpgrader: stubApplier{err: assert.AnError}, helmUpgrader: stubHelmApplier{err: assert.AnError},
terraformUpgrader: &stubTerraformUpgrader{}, terraformUpgrader: &stubTerraformUpgrader{},
wantErr: true, wantErr: true,
flags: applyFlags{yes: true, skipPhases: skipPhases{skipInitPhase: struct{}{}}}, flags: applyFlags{yes: true, skipPhases: skipPhases{skipInitPhase: struct{}{}}},
@ -109,7 +109,7 @@ func TestUpgradeApply(t *testing.T) {
kubeUpgrader: &stubKubernetesUpgrader{ kubeUpgrader: &stubKubernetesUpgrader{
currentConfig: config.DefaultForAzureSEVSNP(), currentConfig: config.DefaultForAzureSEVSNP(),
}, },
helmUpgrader: stubApplier{}, helmUpgrader: stubHelmApplier{},
terraformUpgrader: &stubTerraformUpgrader{terraformDiff: true}, terraformUpgrader: &stubTerraformUpgrader{terraformDiff: true},
wantErr: true, wantErr: true,
stdin: "no\n", stdin: "no\n",
@ -119,7 +119,7 @@ func TestUpgradeApply(t *testing.T) {
kubeUpgrader: &stubKubernetesUpgrader{ kubeUpgrader: &stubKubernetesUpgrader{
currentConfig: config.DefaultForAzureSEVSNP(), currentConfig: config.DefaultForAzureSEVSNP(),
}, },
helmUpgrader: stubApplier{}, helmUpgrader: stubHelmApplier{},
terraformUpgrader: &stubTerraformUpgrader{terraformDiff: true, rollbackWorkspaceErr: assert.AnError}, terraformUpgrader: &stubTerraformUpgrader{terraformDiff: true, rollbackWorkspaceErr: assert.AnError},
wantErr: true, wantErr: true,
stdin: "no\n", stdin: "no\n",
@ -129,7 +129,7 @@ func TestUpgradeApply(t *testing.T) {
kubeUpgrader: &stubKubernetesUpgrader{ kubeUpgrader: &stubKubernetesUpgrader{
currentConfig: config.DefaultForAzureSEVSNP(), currentConfig: config.DefaultForAzureSEVSNP(),
}, },
helmUpgrader: stubApplier{}, helmUpgrader: stubHelmApplier{},
terraformUpgrader: &stubTerraformUpgrader{planTerraformErr: assert.AnError}, terraformUpgrader: &stubTerraformUpgrader{planTerraformErr: assert.AnError},
wantErr: true, wantErr: true,
flags: applyFlags{yes: true, skipPhases: skipPhases{skipInitPhase: struct{}{}}}, flags: applyFlags{yes: true, skipPhases: skipPhases{skipInitPhase: struct{}{}}},
@ -139,7 +139,7 @@ func TestUpgradeApply(t *testing.T) {
kubeUpgrader: &stubKubernetesUpgrader{ kubeUpgrader: &stubKubernetesUpgrader{
currentConfig: config.DefaultForAzureSEVSNP(), currentConfig: config.DefaultForAzureSEVSNP(),
}, },
helmUpgrader: stubApplier{}, helmUpgrader: stubHelmApplier{},
terraformUpgrader: &stubTerraformUpgrader{ terraformUpgrader: &stubTerraformUpgrader{
applyTerraformErr: assert.AnError, applyTerraformErr: assert.AnError,
terraformDiff: true, terraformDiff: true,
@ -152,7 +152,7 @@ func TestUpgradeApply(t *testing.T) {
kubeUpgrader: &stubKubernetesUpgrader{ kubeUpgrader: &stubKubernetesUpgrader{
currentConfig: config.DefaultForAzureSEVSNP(), currentConfig: config.DefaultForAzureSEVSNP(),
}, },
helmUpgrader: stubApplier{}, helmUpgrader: stubHelmApplier{},
terraformUpgrader: &stubTerraformUpgrader{}, terraformUpgrader: &stubTerraformUpgrader{},
customK8sVersion: func() string { customK8sVersion: func() string {
v, err := semver.New(versions.SupportedK8sVersions()[0]) v, err := semver.New(versions.SupportedK8sVersions()[0])
@ -166,7 +166,7 @@ func TestUpgradeApply(t *testing.T) {
kubeUpgrader: &stubKubernetesUpgrader{ kubeUpgrader: &stubKubernetesUpgrader{
currentConfig: config.DefaultForAzureSEVSNP(), currentConfig: config.DefaultForAzureSEVSNP(),
}, },
helmUpgrader: stubApplier{}, helmUpgrader: stubHelmApplier{},
terraformUpgrader: &stubTerraformUpgrader{}, terraformUpgrader: &stubTerraformUpgrader{},
customK8sVersion: "v1.20.0", customK8sVersion: "v1.20.0",
flags: applyFlags{yes: true, skipPhases: skipPhases{skipInitPhase: struct{}{}}}, flags: applyFlags{yes: true, skipPhases: skipPhases{skipInitPhase: struct{}{}}},
@ -201,7 +201,7 @@ func TestUpgradeApply(t *testing.T) {
kubeUpgrader: &stubKubernetesUpgrader{ kubeUpgrader: &stubKubernetesUpgrader{
currentConfig: config.DefaultForAzureSEVSNP(), currentConfig: config.DefaultForAzureSEVSNP(),
}, },
helmUpgrader: &stubApplier{}, helmUpgrader: &stubHelmApplier{},
terraformUpgrader: &mockTerraformUpgrader{}, terraformUpgrader: &mockTerraformUpgrader{},
flags: applyFlags{ flags: applyFlags{
yes: true, yes: true,
@ -215,12 +215,21 @@ func TestUpgradeApply(t *testing.T) {
}, },
"attempt to change attestation variant": { "attempt to change attestation variant": {
kubeUpgrader: &stubKubernetesUpgrader{currentConfig: &config.AzureTrustedLaunch{}}, kubeUpgrader: &stubKubernetesUpgrader{currentConfig: &config.AzureTrustedLaunch{}},
helmUpgrader: stubApplier{}, helmUpgrader: stubHelmApplier{},
terraformUpgrader: &stubTerraformUpgrader{}, terraformUpgrader: &stubTerraformUpgrader{},
flags: applyFlags{yes: true, skipPhases: skipPhases{skipInitPhase: struct{}{}}}, flags: applyFlags{yes: true, skipPhases: skipPhases{skipInitPhase: struct{}{}}},
fh: fsWithStateFileAndTfState, fh: fsWithStateFileAndTfState,
wantErr: true, wantErr: true,
}, },
"image fetching fails": {
kubeUpgrader: &stubKubernetesUpgrader{currentConfig: config.DefaultForAzureSEVSNP()},
helmUpgrader: stubHelmApplier{},
terraformUpgrader: &stubTerraformUpgrader{},
fetchImageErr: assert.AnError,
flags: applyFlags{yes: true, skipPhases: skipPhases{skipInitPhase: struct{}{}}},
fh: fsWithStateFileAndTfState,
wantErr: true,
},
} }
for name, tc := range testCases { for name, tc := range testCases {
@ -248,13 +257,11 @@ func TestUpgradeApply(t *testing.T) {
newHelmClient: func(string, debugLog) (helmApplier, error) { newHelmClient: func(string, debugLog) (helmApplier, error) {
return tc.helmUpgrader, nil return tc.helmUpgrader, nil
}, },
newKubeUpgrader: func(_ io.Writer, _ string, _ debugLog) (kubernetesUpgrader, error) {
return tc.kubeUpgrader, nil
},
newInfraApplier: func(ctx context.Context) (cloudApplier, func(), error) { newInfraApplier: func(ctx context.Context) (cloudApplier, func(), error) {
return tc.terraformUpgrader, func() {}, nil return tc.terraformUpgrader, func() {}, nil
}, },
applier: &stubConstellApplier{}, applier: &stubConstellApplier{stubKubernetesUpgrader: tc.kubeUpgrader},
imageFetcher: &stubImageFetcher{fetchReferenceErr: tc.fetchImageErr},
} }
err := upgrader.apply(cmd, stubAttestationFetcher{}, "test") err := upgrader.apply(cmd, stubAttestationFetcher{}, "test")
if tc.wantErr { if tc.wantErr {
@ -274,30 +281,37 @@ func TestUpgradeApply(t *testing.T) {
type stubKubernetesUpgrader struct { type stubKubernetesUpgrader struct {
nodeVersionErr error nodeVersionErr error
kubernetesVersionErr error
currentConfig config.AttestationCfg currentConfig config.AttestationCfg
getClusterAttestationConfigErr error getClusterAttestationConfigErr error
calledNodeUpgrade bool calledNodeUpgrade bool
calledKubernetesUpgrade bool
backupCRDsErr error backupCRDsErr error
backupCRDsCalled bool backupCRDsCalled bool
backupCRsErr error backupCRsErr error
backupCRsCalled bool backupCRsCalled bool
} }
func (u *stubKubernetesUpgrader) BackupCRDs(_ context.Context, _ string) ([]apiextensionsv1.CustomResourceDefinition, error) { func (u *stubKubernetesUpgrader) BackupCRDs(_ context.Context, _ file.Handler, _ string) ([]apiextensionsv1.CustomResourceDefinition, error) {
u.backupCRDsCalled = true u.backupCRDsCalled = true
return []apiextensionsv1.CustomResourceDefinition{}, u.backupCRDsErr return []apiextensionsv1.CustomResourceDefinition{}, u.backupCRDsErr
} }
func (u *stubKubernetesUpgrader) BackupCRs(_ context.Context, _ []apiextensionsv1.CustomResourceDefinition, _ string) error { func (u *stubKubernetesUpgrader) BackupCRs(_ context.Context, _ file.Handler, _ []apiextensionsv1.CustomResourceDefinition, _ string) error {
u.backupCRsCalled = true u.backupCRsCalled = true
return u.backupCRsErr return u.backupCRsErr
} }
func (u *stubKubernetesUpgrader) UpgradeNodeVersion(_ context.Context, _ *config.Config, _, _, _ bool) error { func (u *stubKubernetesUpgrader) UpgradeNodeImage(_ context.Context, _ semver.Semver, _ string, _ bool) error {
u.calledNodeUpgrade = true u.calledNodeUpgrade = true
return u.nodeVersionErr return u.nodeVersionErr
} }
func (u *stubKubernetesUpgrader) UpgradeKubernetesVersion(_ context.Context, _ versions.ValidK8sVersion, _ bool) error {
u.calledKubernetesUpgrade = true
return u.kubernetesVersionErr
}
func (u *stubKubernetesUpgrader) ApplyJoinConfig(_ context.Context, _ config.AttestationCfg, _ []byte) error { func (u *stubKubernetesUpgrader) ApplyJoinConfig(_ context.Context, _ config.AttestationCfg, _ []byte) error {
return nil return nil
} }
@ -306,7 +320,7 @@ func (u *stubKubernetesUpgrader) GetClusterAttestationConfig(_ context.Context,
return u.currentConfig, u.getClusterAttestationConfigErr return u.currentConfig, u.getClusterAttestationConfigErr
} }
func (u *stubKubernetesUpgrader) ExtendClusterConfigCertSANs(_ context.Context, _ []string) error { func (u *stubKubernetesUpgrader) ExtendClusterConfigCertSANs(_ context.Context, _, _ string, _ []string) error {
return nil return nil
} }
@ -367,3 +381,15 @@ func (m *mockApplier) PrepareApply(csp cloudprovider.Provider, variant variant.V
args := m.Called(csp, variant, k8sVersion, microserviceVersion, stateFile, helmOpts, str, masterSecret, openStackCfg) args := m.Called(csp, variant, k8sVersion, microserviceVersion, stateFile, helmOpts, str, masterSecret, openStackCfg)
return args.Get(0).(helm.Applier), args.Bool(1), args.Error(2) return args.Get(0).(helm.Applier), args.Bool(1), args.Error(2)
} }
type stubImageFetcher struct {
reference string
fetchReferenceErr error
}
func (f *stubImageFetcher) FetchReference(_ context.Context,
_ cloudprovider.Provider, _ variant.Variant,
_, _ string,
) (string, error) {
return f.reference, f.fetchReferenceErr
}

View file

@ -26,10 +26,10 @@ import (
"github.com/edgelesssys/constellation/v2/internal/compatibility" "github.com/edgelesssys/constellation/v2/internal/compatibility"
"github.com/edgelesssys/constellation/v2/internal/config" "github.com/edgelesssys/constellation/v2/internal/config"
"github.com/edgelesssys/constellation/v2/internal/constants" "github.com/edgelesssys/constellation/v2/internal/constants"
"github.com/edgelesssys/constellation/v2/internal/constellation/kubecmd"
"github.com/edgelesssys/constellation/v2/internal/featureset" "github.com/edgelesssys/constellation/v2/internal/featureset"
"github.com/edgelesssys/constellation/v2/internal/file" "github.com/edgelesssys/constellation/v2/internal/file"
"github.com/edgelesssys/constellation/v2/internal/helm" "github.com/edgelesssys/constellation/v2/internal/helm"
"github.com/edgelesssys/constellation/v2/internal/kubecmd"
consemver "github.com/edgelesssys/constellation/v2/internal/semver" consemver "github.com/edgelesssys/constellation/v2/internal/semver"
"github.com/edgelesssys/constellation/v2/internal/sigstore" "github.com/edgelesssys/constellation/v2/internal/sigstore"
"github.com/edgelesssys/constellation/v2/internal/sigstore/keyselect" "github.com/edgelesssys/constellation/v2/internal/sigstore/keyselect"
@ -120,7 +120,7 @@ func runUpgradeCheck(cmd *cobra.Command, _ []string) error {
if err != nil { if err != nil {
return fmt.Errorf("reading kubeconfig: %w", err) return fmt.Errorf("reading kubeconfig: %w", err)
} }
kubeChecker, err := kubecmd.New(cmd.OutOrStdout(), kubeConfig, fileHandler, log) kubeChecker, err := kubecmd.New(kubeConfig, log)
if err != nil { if err != nil {
return fmt.Errorf("setting up Kubernetes upgrader: %w", err) return fmt.Errorf("setting up Kubernetes upgrader: %w", err)
} }

View file

@ -226,7 +226,7 @@ ok github.com/edgelesssys/constellation/v2/operators/constellation-node-operat
` `
const ( const (
exampleReportCLI = `{"Metadate":{"Created":"2023-08-24T16:09:02Z"},"Coverage":{"github.com/edgelesssys/constellation/v2/cli":{"Coverage":0,"Notest":true,"Nostmt":false},"github.com/edgelesssys/constellation/v2/cli/cmd":{"Coverage":0,"Notest":true,"Nostmt":false},"github.com/edgelesssys/constellation/v2/cli/internal/cloudcmd":{"Coverage":65.5,"Notest":false,"Nostmt":false},"github.com/edgelesssys/constellation/v2/cli/internal/clusterid":{"Coverage":56.2,"Notest":false,"Nostmt":false},"github.com/edgelesssys/constellation/v2/cli/internal/cmd":{"Coverage":53.5,"Notest":false,"Nostmt":false},"github.com/edgelesssys/constellation/v2/cli/internal/cmd/pathprefix":{"Coverage":0,"Notest":true,"Nostmt":false},"github.com/edgelesssys/constellation/v2/internal/featureset":{"Coverage":0,"Notest":true,"Nostmt":false},"github.com/edgelesssys/constellation/v2/internal/helm":{"Coverage":47.7,"Notest":false,"Nostmt":false},"github.com/edgelesssys/constellation/v2/internal/helm/imageversion":{"Coverage":0,"Notest":true,"Nostmt":false},"github.com/edgelesssys/constellation/v2/internal/kubecmd":{"Coverage":54.1,"Notest":false,"Nostmt":false},"github.com/edgelesssys/constellation/v2/cli/internal/libvirt":{"Coverage":0,"Notest":true,"Nostmt":false},"github.com/edgelesssys/constellation/v2/cli/internal/terraform":{"Coverage":71.3,"Notest":false,"Nostmt":false}}}` exampleReportCLI = `{"Metadate":{"Created":"2023-08-24T16:09:02Z"},"Coverage":{"github.com/edgelesssys/constellation/v2/cli":{"Coverage":0,"Notest":true,"Nostmt":false},"github.com/edgelesssys/constellation/v2/cli/cmd":{"Coverage":0,"Notest":true,"Nostmt":false},"github.com/edgelesssys/constellation/v2/cli/internal/cloudcmd":{"Coverage":65.5,"Notest":false,"Nostmt":false},"github.com/edgelesssys/constellation/v2/cli/internal/clusterid":{"Coverage":56.2,"Notest":false,"Nostmt":false},"github.com/edgelesssys/constellation/v2/cli/internal/cmd":{"Coverage":53.5,"Notest":false,"Nostmt":false},"github.com/edgelesssys/constellation/v2/cli/internal/cmd/pathprefix":{"Coverage":0,"Notest":true,"Nostmt":false},"github.com/edgelesssys/constellation/v2/internal/featureset":{"Coverage":0,"Notest":true,"Nostmt":false},"github.com/edgelesssys/constellation/v2/internal/helm":{"Coverage":47.7,"Notest":false,"Nostmt":false},"github.com/edgelesssys/constellation/v2/internal/helm/imageversion":{"Coverage":0,"Notest":true,"Nostmt":false},"github.com/edgelesssys/constellation/v2/internal/constellation/kubecmd":{"Coverage":54.1,"Notest":false,"Nostmt":false},"github.com/edgelesssys/constellation/v2/cli/internal/libvirt":{"Coverage":0,"Notest":true,"Nostmt":false},"github.com/edgelesssys/constellation/v2/cli/internal/terraform":{"Coverage":71.3,"Notest":false,"Nostmt":false}}}`
exampleReportCLIOld = `{"Metadate":{"Created":"2023-08-24T16:48:39Z"},"Coverage":{"github.com/edgelesssys/constellation/v2/cli":{"Coverage":0,"Notest":true,"Nostmt":false},"github.com/edgelesssys/constellation/v2/cli/cmd":{"Coverage":0,"Notest":true,"Nostmt":false},"github.com/edgelesssys/constellation/v2/cli/internal/cloudcmd":{"Coverage":73.1,"Notest":false,"Nostmt":false},"github.com/edgelesssys/constellation/v2/cli/internal/clusterid":{"Coverage":0,"Notest":true,"Nostmt":false},"github.com/edgelesssys/constellation/v2/cli/internal/cmd":{"Coverage":61.6,"Notest":false,"Nostmt":false},"github.com/edgelesssys/constellation/v2/internal/featureset":{"Coverage":0,"Notest":true,"Nostmt":false},"github.com/edgelesssys/constellation/v2/internal/helm":{"Coverage":51.7,"Notest":false,"Nostmt":false},"github.com/edgelesssys/constellation/v2/internal/helm/imageversion":{"Coverage":0,"Notest":true,"Nostmt":false},"github.com/edgelesssys/constellation/v2/cli/internal/iamid":{"Coverage":0,"Notest":true,"Nostmt":false},"github.com/edgelesssys/constellation/v2/cli/internal/kubernetes":{"Coverage":49.8,"Notest":false,"Nostmt":false},"github.com/edgelesssys/constellation/v2/cli/internal/libvirt":{"Coverage":0,"Notest":true,"Nostmt":false},"github.com/edgelesssys/constellation/v2/cli/internal/terraform":{"Coverage":66.7,"Notest":false,"Nostmt":false},"github.com/edgelesssys/constellation/v2/cli/internal/upgrade":{"Coverage":83,"Notest":false,"Nostmt":false}}}` exampleReportCLIOld = `{"Metadate":{"Created":"2023-08-24T16:48:39Z"},"Coverage":{"github.com/edgelesssys/constellation/v2/cli":{"Coverage":0,"Notest":true,"Nostmt":false},"github.com/edgelesssys/constellation/v2/cli/cmd":{"Coverage":0,"Notest":true,"Nostmt":false},"github.com/edgelesssys/constellation/v2/cli/internal/cloudcmd":{"Coverage":73.1,"Notest":false,"Nostmt":false},"github.com/edgelesssys/constellation/v2/cli/internal/clusterid":{"Coverage":0,"Notest":true,"Nostmt":false},"github.com/edgelesssys/constellation/v2/cli/internal/cmd":{"Coverage":61.6,"Notest":false,"Nostmt":false},"github.com/edgelesssys/constellation/v2/internal/featureset":{"Coverage":0,"Notest":true,"Nostmt":false},"github.com/edgelesssys/constellation/v2/internal/helm":{"Coverage":51.7,"Notest":false,"Nostmt":false},"github.com/edgelesssys/constellation/v2/internal/helm/imageversion":{"Coverage":0,"Notest":true,"Nostmt":false},"github.com/edgelesssys/constellation/v2/cli/internal/iamid":{"Coverage":0,"Notest":true,"Nostmt":false},"github.com/edgelesssys/constellation/v2/cli/internal/kubernetes":{"Coverage":49.8,"Notest":false,"Nostmt":false},"github.com/edgelesssys/constellation/v2/cli/internal/libvirt":{"Coverage":0,"Notest":true,"Nostmt":false},"github.com/edgelesssys/constellation/v2/cli/internal/terraform":{"Coverage":66.7,"Notest":false,"Nostmt":false},"github.com/edgelesssys/constellation/v2/cli/internal/upgrade":{"Coverage":83,"Notest":false,"Nostmt":false}}}`
exampleReportDisk = `{"Metadate":{"Created":"2023-08-24T16:40:25Z"},"Coverage":{"github.com/edgelesssys/constellation/v2/disk-mapper/cmd":{"Coverage":0,"Notest":true,"Nostmt":false},"github.com/edgelesssys/constellation/v2/disk-mapper/internal/diskencryption":{"Coverage":0,"Notest":true,"Nostmt":false},"github.com/edgelesssys/constellation/v2/disk-mapper/internal/recoveryserver":{"Coverage":89.1,"Notest":false,"Nostmt":false},"github.com/edgelesssys/constellation/v2/disk-mapper/internal/rejoinclient":{"Coverage":91.8,"Notest":false,"Nostmt":false},"github.com/edgelesssys/constellation/v2/disk-mapper/internal/setup":{"Coverage":68.9,"Notest":false,"Nostmt":false},"github.com/edgelesssys/constellation/v2/disk-mapper/internal/systemd":{"Coverage":25.8,"Notest":false,"Nostmt":false},"github.com/edgelesssys/constellation/v2/disk-mapper/recoverproto":{"Coverage":0,"Notest":true,"Nostmt":false}}}` exampleReportDisk = `{"Metadate":{"Created":"2023-08-24T16:40:25Z"},"Coverage":{"github.com/edgelesssys/constellation/v2/disk-mapper/cmd":{"Coverage":0,"Notest":true,"Nostmt":false},"github.com/edgelesssys/constellation/v2/disk-mapper/internal/diskencryption":{"Coverage":0,"Notest":true,"Nostmt":false},"github.com/edgelesssys/constellation/v2/disk-mapper/internal/recoveryserver":{"Coverage":89.1,"Notest":false,"Nostmt":false},"github.com/edgelesssys/constellation/v2/disk-mapper/internal/rejoinclient":{"Coverage":91.8,"Notest":false,"Nostmt":false},"github.com/edgelesssys/constellation/v2/disk-mapper/internal/setup":{"Coverage":68.9,"Notest":false,"Nostmt":false},"github.com/edgelesssys/constellation/v2/disk-mapper/internal/systemd":{"Coverage":25.8,"Notest":false,"Nostmt":false},"github.com/edgelesssys/constellation/v2/disk-mapper/recoverproto":{"Coverage":0,"Notest":true,"Nostmt":false}}}`
) )

View file

@ -7,23 +7,31 @@ go_library(
"apply.go", "apply.go",
"applyinit.go", "applyinit.go",
"constellation.go", "constellation.go",
"kubernetes.go",
], ],
importpath = "github.com/edgelesssys/constellation/v2/internal/constellation", importpath = "github.com/edgelesssys/constellation/v2/internal/constellation",
visibility = ["//:__subpackages__"], visibility = ["//:__subpackages__"],
deps = [ deps = [
"//bootstrapper/initproto", "//bootstrapper/initproto",
"//internal/atls", "//internal/atls",
"//internal/attestation/variant",
"//internal/cloud/cloudprovider", "//internal/cloud/cloudprovider",
"//internal/config",
"//internal/constants", "//internal/constants",
"//internal/constellation/kubecmd",
"//internal/crypto", "//internal/crypto",
"//internal/file",
"//internal/grpc/dialer", "//internal/grpc/dialer",
"//internal/grpc/grpclog", "//internal/grpc/grpclog",
"//internal/grpc/retry", "//internal/grpc/retry",
"//internal/helm",
"//internal/kms/uri", "//internal/kms/uri",
"//internal/license", "//internal/license",
"//internal/retry", "//internal/retry",
"//internal/semver",
"//internal/state", "//internal/state",
"//internal/versions", "//internal/versions",
"@io_k8s_apiextensions_apiserver//pkg/apis/apiextensions/v1:apiextensions",
"@org_golang_google_grpc//:go_default_library", "@org_golang_google_grpc//:go_default_library",
], ],
) )
@ -38,7 +46,10 @@ go_test(
deps = [ deps = [
"//bootstrapper/initproto", "//bootstrapper/initproto",
"//internal/atls", "//internal/atls",
"//internal/attestation/measurements",
"//internal/attestation/variant",
"//internal/cloud/cloudprovider", "//internal/cloud/cloudprovider",
"//internal/config",
"//internal/constants", "//internal/constants",
"//internal/crypto", "//internal/crypto",
"//internal/grpc/atlscredentials", "//internal/grpc/atlscredentials",

View file

@ -11,11 +11,20 @@ import (
"fmt" "fmt"
"github.com/edgelesssys/constellation/v2/internal/atls" "github.com/edgelesssys/constellation/v2/internal/atls"
"github.com/edgelesssys/constellation/v2/internal/attestation/variant"
"github.com/edgelesssys/constellation/v2/internal/cloud/cloudprovider" "github.com/edgelesssys/constellation/v2/internal/cloud/cloudprovider"
"github.com/edgelesssys/constellation/v2/internal/config"
"github.com/edgelesssys/constellation/v2/internal/constellation/kubecmd"
"github.com/edgelesssys/constellation/v2/internal/crypto" "github.com/edgelesssys/constellation/v2/internal/crypto"
"github.com/edgelesssys/constellation/v2/internal/file"
"github.com/edgelesssys/constellation/v2/internal/grpc/dialer" "github.com/edgelesssys/constellation/v2/internal/grpc/dialer"
"github.com/edgelesssys/constellation/v2/internal/helm"
"github.com/edgelesssys/constellation/v2/internal/kms/uri" "github.com/edgelesssys/constellation/v2/internal/kms/uri"
"github.com/edgelesssys/constellation/v2/internal/license" "github.com/edgelesssys/constellation/v2/internal/license"
"github.com/edgelesssys/constellation/v2/internal/semver"
"github.com/edgelesssys/constellation/v2/internal/state"
"github.com/edgelesssys/constellation/v2/internal/versions"
apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
) )
// An Applier handles applying a specific configuration to a Constellation cluster // An Applier handles applying a specific configuration to a Constellation cluster
@ -27,7 +36,9 @@ type Applier struct {
spinner spinnerInterf spinner spinnerInterf
// newDialer creates a new aTLS gRPC dialer. // newDialer creates a new aTLS gRPC dialer.
newDialer func(validator atls.Validator) *dialer.Dialer newDialer func(validator atls.Validator) *dialer.Dialer
kubecmdClient kubecmdClient
helmClient helmApplier
} }
type licenseChecker interface { type licenseChecker interface {
@ -52,6 +63,21 @@ func NewApplier(
} }
} }
// SetKubeConfig sets the config file to use for creating Kubernetes clients.
func (a *Applier) SetKubeConfig(kubeConfig []byte) error {
kubecmdClient, err := kubecmd.New(kubeConfig, a.log)
if err != nil {
return err
}
helmClient, err := helm.NewClient(kubeConfig, a.log)
if err != nil {
return err
}
a.kubecmdClient = kubecmdClient
a.helmClient = helmClient
return nil
}
// CheckLicense checks the given Constellation license with the license server // CheckLicense checks the given Constellation license with the license server
// and returns the allowed quota for the license. // and returns the allowed quota for the license.
func (a *Applier) CheckLicense(ctx context.Context, csp cloudprovider.Provider, licenseID string) (int, error) { func (a *Applier) CheckLicense(ctx context.Context, csp cloudprovider.Provider, licenseID string) (int, error) {
@ -94,3 +120,21 @@ func (a *Applier) GenerateMeasurementSalt() ([]byte, error) {
a.log.Debugf("Generated measurement salt") a.log.Debugf("Generated measurement salt")
return measurementSalt, nil return measurementSalt, nil
} }
type helmApplier interface {
PrepareApply(
csp cloudprovider.Provider, attestationVariant variant.Variant, k8sVersion versions.ValidK8sVersion, microserviceVersion semver.Semver, stateFile *state.State,
flags helm.Options, serviceAccURI string, masterSecret uri.MasterSecret, openStackCfg *config.OpenStackConfig,
) (
helm.Applier, bool, error)
}
type kubecmdClient interface {
UpgradeNodeImage(ctx context.Context, imageVersion semver.Semver, imageReference string, force bool) error
UpgradeKubernetesVersion(ctx context.Context, kubernetesVersion versions.ValidK8sVersion, force bool) error
ExtendClusterConfigCertSANs(ctx context.Context, alternativeNames []string) error
GetClusterAttestationConfig(ctx context.Context, variant variant.Variant) (config.AttestationCfg, error)
ApplyJoinConfig(ctx context.Context, newAttestConfig config.AttestationCfg, measurementSalt []byte) error
BackupCRs(ctx context.Context, fileHandler file.Handler, crds []apiextensionsv1.CustomResourceDefinition, upgradeDir string) error
BackupCRDs(ctx context.Context, fileHandler file.Handler, upgradeDir string) ([]apiextensionsv1.CustomResourceDefinition, error)
}

View file

@ -9,6 +9,8 @@ package constellation
import ( import (
"bytes" "bytes"
"context" "context"
"encoding/json"
"errors"
"io" "io"
"net" "net"
"strconv" "strconv"
@ -17,6 +19,9 @@ import (
"github.com/edgelesssys/constellation/v2/bootstrapper/initproto" "github.com/edgelesssys/constellation/v2/bootstrapper/initproto"
"github.com/edgelesssys/constellation/v2/internal/atls" "github.com/edgelesssys/constellation/v2/internal/atls"
"github.com/edgelesssys/constellation/v2/internal/attestation/measurements"
"github.com/edgelesssys/constellation/v2/internal/attestation/variant"
"github.com/edgelesssys/constellation/v2/internal/config"
"github.com/edgelesssys/constellation/v2/internal/constants" "github.com/edgelesssys/constellation/v2/internal/constants"
"github.com/edgelesssys/constellation/v2/internal/grpc/atlscredentials" "github.com/edgelesssys/constellation/v2/internal/grpc/atlscredentials"
"github.com/edgelesssys/constellation/v2/internal/grpc/dialer" "github.com/edgelesssys/constellation/v2/internal/grpc/dialer"
@ -200,6 +205,119 @@ func TestInit(t *testing.T) {
} }
} }
func TestAttestation(t *testing.T) {
assert := assert.New(t)
initServerAPI := &stubInitServer{res: []*initproto.InitResponse{
{
Kind: &initproto.InitResponse_InitSuccess{
InitSuccess: &initproto.InitSuccessResponse{
Kubeconfig: []byte("kubeconfig"),
OwnerId: []byte("ownerID"),
ClusterId: []byte("clusterID"),
},
},
},
}}
netDialer := testdialer.NewBufconnDialer()
issuer := &testIssuer{
Getter: variant.QEMUVTPM{},
pcrs: map[uint32][]byte{
0: bytes.Repeat([]byte{0xFF}, 32),
1: bytes.Repeat([]byte{0xFF}, 32),
2: bytes.Repeat([]byte{0xFF}, 32),
3: bytes.Repeat([]byte{0xFF}, 32),
},
}
serverCreds := atlscredentials.New(issuer, nil)
initServer := grpc.NewServer(grpc.Creds(serverCreds))
initproto.RegisterAPIServer(initServer, initServerAPI)
port := strconv.Itoa(constants.BootstrapperPort)
listener := netDialer.GetListener(net.JoinHostPort("192.0.2.4", port))
go initServer.Serve(listener)
defer initServer.GracefulStop()
validator := &testValidator{
Getter: variant.QEMUVTPM{},
pcrs: measurements.M{
0: measurements.WithAllBytes(0x00, measurements.Enforce, measurements.PCRMeasurementLength),
1: measurements.WithAllBytes(0x11, measurements.Enforce, measurements.PCRMeasurementLength),
2: measurements.WithAllBytes(0x22, measurements.Enforce, measurements.PCRMeasurementLength),
3: measurements.WithAllBytes(0x33, measurements.Enforce, measurements.PCRMeasurementLength),
4: measurements.WithAllBytes(0x44, measurements.Enforce, measurements.PCRMeasurementLength),
9: measurements.WithAllBytes(0x99, measurements.Enforce, measurements.PCRMeasurementLength),
12: measurements.WithAllBytes(0xcc, measurements.Enforce, measurements.PCRMeasurementLength),
},
}
state := &state.State{Version: state.Version1, Infrastructure: state.Infrastructure{ClusterEndpoint: "192.0.2.4"}}
ctx := context.Background()
ctx, cancel := context.WithTimeout(ctx, 4*time.Second)
defer cancel()
initer := &Applier{
log: logger.NewTest(t),
newDialer: func(v atls.Validator) *dialer.Dialer {
return dialer.New(nil, v, netDialer)
},
spinner: &nopSpinner{},
}
_, err := initer.Init(ctx, validator, state, io.Discard, InitPayload{
MasterSecret: uri.MasterSecret{},
MeasurementSalt: []byte{},
K8sVersion: "v1.26.5",
ConformanceMode: false,
})
assert.Error(err)
// make sure the error is actually a TLS handshake error
assert.Contains(err.Error(), "transport: authentication handshake failed")
if validationErr, ok := err.(*config.ValidationError); ok {
t.Log(validationErr.LongMessage())
}
}
type testValidator struct {
variant.Getter
pcrs measurements.M
}
func (v *testValidator) Validate(_ context.Context, attDoc []byte, _ []byte) ([]byte, error) {
var attestation struct {
UserData []byte
PCRs map[uint32][]byte
}
if err := json.Unmarshal(attDoc, &attestation); err != nil {
return nil, err
}
for k, pcr := range v.pcrs {
if !bytes.Equal(attestation.PCRs[k], pcr.Expected[:]) {
return nil, errors.New("invalid PCR value")
}
}
return attestation.UserData, nil
}
type testIssuer struct {
variant.Getter
pcrs map[uint32][]byte
}
func (i *testIssuer) Issue(_ context.Context, userData []byte, _ []byte) ([]byte, error) {
return json.Marshal(
struct {
UserData []byte
PCRs map[uint32][]byte
}{
UserData: userData,
PCRs: i.pcrs,
},
)
}
type nopSpinner struct { type nopSpinner struct {
io.Writer io.Writer
} }

View file

@ -8,20 +8,18 @@ go_library(
"kubecmd.go", "kubecmd.go",
"status.go", "status.go",
], ],
importpath = "github.com/edgelesssys/constellation/v2/internal/kubecmd", importpath = "github.com/edgelesssys/constellation/v2/internal/constellation/kubecmd",
visibility = ["//cli:__subpackages__"], visibility = ["//:__subpackages__"],
deps = [ deps = [
"//internal/api/versionsapi",
"//internal/attestation/variant", "//internal/attestation/variant",
"//internal/cloud/cloudprovider",
"//internal/compatibility", "//internal/compatibility",
"//internal/config", "//internal/config",
"//internal/constants", "//internal/constants",
"//internal/file", "//internal/file",
"//internal/imagefetcher",
"//internal/kubernetes", "//internal/kubernetes",
"//internal/kubernetes/kubectl", "//internal/kubernetes/kubectl",
"//internal/retry", "//internal/retry",
"//internal/semver",
"//internal/versions", "//internal/versions",
"//internal/versions/components", "//internal/versions/components",
"//operators/constellation-node-operator/api/v1alpha1", "//operators/constellation-node-operator/api/v1alpha1",
@ -47,13 +45,12 @@ go_test(
embed = [":kubecmd"], embed = [":kubecmd"],
deps = [ deps = [
"//internal/attestation/measurements", "//internal/attestation/measurements",
"//internal/attestation/variant",
"//internal/cloud/cloudprovider",
"//internal/compatibility", "//internal/compatibility",
"//internal/config", "//internal/config",
"//internal/constants", "//internal/constants",
"//internal/file", "//internal/file",
"//internal/logger", "//internal/logger",
"//internal/semver",
"//internal/versions", "//internal/versions",
"//internal/versions/components", "//internal/versions/components",
"//operators/constellation-node-operator/api/v1alpha1", "//operators/constellation-node-operator/api/v1alpha1",

View file

@ -11,6 +11,7 @@ import (
"fmt" "fmt"
"path/filepath" "path/filepath"
"github.com/edgelesssys/constellation/v2/internal/file"
apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
k8serrors "k8s.io/apimachinery/pkg/api/errors" k8serrors "k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
@ -24,7 +25,7 @@ type crdLister interface {
} }
// BackupCRDs backs up all CRDs to the upgrade workspace. // BackupCRDs backs up all CRDs to the upgrade workspace.
func (k *KubeCmd) BackupCRDs(ctx context.Context, upgradeDir string) ([]apiextensionsv1.CustomResourceDefinition, error) { func (k *KubeCmd) BackupCRDs(ctx context.Context, fileHandler file.Handler, upgradeDir string) ([]apiextensionsv1.CustomResourceDefinition, error) {
k.log.Debugf("Starting CRD backup") k.log.Debugf("Starting CRD backup")
crds, err := k.kubectl.ListCRDs(ctx) crds, err := k.kubectl.ListCRDs(ctx)
if err != nil { if err != nil {
@ -32,7 +33,7 @@ func (k *KubeCmd) BackupCRDs(ctx context.Context, upgradeDir string) ([]apiexten
} }
crdBackupFolder := k.crdBackupFolder(upgradeDir) crdBackupFolder := k.crdBackupFolder(upgradeDir)
if err := k.fileHandler.MkdirAll(crdBackupFolder); err != nil { if err := fileHandler.MkdirAll(crdBackupFolder); err != nil {
return nil, fmt.Errorf("creating backup dir: %w", err) return nil, fmt.Errorf("creating backup dir: %w", err)
} }
for i := range crds { for i := range crds {
@ -51,7 +52,7 @@ func (k *KubeCmd) BackupCRDs(ctx context.Context, upgradeDir string) ([]apiexten
if err != nil { if err != nil {
return nil, err return nil, err
} }
if err := k.fileHandler.Write(path, yamlBytes); err != nil { if err := fileHandler.Write(path, yamlBytes); err != nil {
return nil, err return nil, err
} }
} }
@ -60,7 +61,7 @@ func (k *KubeCmd) BackupCRDs(ctx context.Context, upgradeDir string) ([]apiexten
} }
// BackupCRs backs up all CRs to the upgrade workspace. // BackupCRs backs up all CRs to the upgrade workspace.
func (k *KubeCmd) BackupCRs(ctx context.Context, crds []apiextensionsv1.CustomResourceDefinition, upgradeDir string) error { func (k *KubeCmd) BackupCRs(ctx context.Context, fileHandler file.Handler, crds []apiextensionsv1.CustomResourceDefinition, upgradeDir string) error {
k.log.Debugf("Starting CR backup") k.log.Debugf("Starting CR backup")
for _, crd := range crds { for _, crd := range crds {
k.log.Debugf("Creating backup for resource type: %s", crd.Name) k.log.Debugf("Creating backup for resource type: %s", crd.Name)
@ -86,7 +87,7 @@ func (k *KubeCmd) BackupCRs(ctx context.Context, crds []apiextensionsv1.CustomRe
backupFolder := k.backupFolder(upgradeDir) backupFolder := k.backupFolder(upgradeDir)
for _, cr := range crs { for _, cr := range crs {
targetFolder := filepath.Join(backupFolder, gvr.Group, gvr.Version, cr.GetNamespace(), cr.GetKind()) targetFolder := filepath.Join(backupFolder, gvr.Group, gvr.Version, cr.GetNamespace(), cr.GetKind())
if err := k.fileHandler.MkdirAll(targetFolder); err != nil { if err := fileHandler.MkdirAll(targetFolder); err != nil {
return fmt.Errorf("creating resource dir: %w", err) return fmt.Errorf("creating resource dir: %w", err)
} }
path := filepath.Join(targetFolder, cr.GetName()+".yaml") path := filepath.Join(targetFolder, cr.GetName()+".yaml")
@ -94,7 +95,7 @@ func (k *KubeCmd) BackupCRs(ctx context.Context, crds []apiextensionsv1.CustomRe
if err != nil { if err != nil {
return err return err
} }
if err := k.fileHandler.Write(path, yamlBytes); err != nil { if err := fileHandler.Write(path, yamlBytes); err != nil {
return err return err
} }
} }

View file

@ -53,12 +53,11 @@ func TestBackupCRDs(t *testing.T) {
err := yaml.Unmarshal([]byte(tc.crd), &crd) err := yaml.Unmarshal([]byte(tc.crd), &crd)
require.NoError(err) require.NoError(err)
client := KubeCmd{ client := KubeCmd{
kubectl: &stubKubectl{crds: []apiextensionsv1.CustomResourceDefinition{crd}, getCRDsError: tc.getCRDsError}, kubectl: &stubKubectl{crds: []apiextensionsv1.CustomResourceDefinition{crd}, getCRDsError: tc.getCRDsError},
fileHandler: file.NewHandler(memFs), log: stubLog{},
log: stubLog{},
} }
_, err = client.BackupCRDs(context.Background(), tc.upgradeID) _, err = client.BackupCRDs(context.Background(), file.NewHandler(memFs), tc.upgradeID)
if tc.wantError { if tc.wantError {
assert.Error(err) assert.Error(err)
return return
@ -143,12 +142,11 @@ func TestBackupCRs(t *testing.T) {
memFs := afero.NewMemMapFs() memFs := afero.NewMemMapFs()
client := KubeCmd{ client := KubeCmd{
kubectl: &stubKubectl{crs: []unstructured.Unstructured{tc.resource}, getCRsError: tc.getCRsError}, kubectl: &stubKubectl{crs: []unstructured.Unstructured{tc.resource}, getCRsError: tc.getCRsError},
fileHandler: file.NewHandler(memFs), log: stubLog{},
log: stubLog{},
} }
err := client.BackupCRs(context.Background(), []apiextensionsv1.CustomResourceDefinition{tc.crd}, tc.upgradeID) err := client.BackupCRs(context.Background(), file.NewHandler(memFs), []apiextensionsv1.CustomResourceDefinition{tc.crd}, tc.upgradeID)
if tc.wantError { if tc.wantError {
assert.Error(err) assert.Error(err)
return return

View file

@ -21,23 +21,19 @@ import (
"encoding/json" "encoding/json"
"errors" "errors"
"fmt" "fmt"
"io"
"slices" "slices"
"sort" "sort"
"strings" "strings"
"time" "time"
"github.com/edgelesssys/constellation/v2/internal/api/versionsapi"
"github.com/edgelesssys/constellation/v2/internal/attestation/variant" "github.com/edgelesssys/constellation/v2/internal/attestation/variant"
"github.com/edgelesssys/constellation/v2/internal/cloud/cloudprovider"
"github.com/edgelesssys/constellation/v2/internal/compatibility" "github.com/edgelesssys/constellation/v2/internal/compatibility"
"github.com/edgelesssys/constellation/v2/internal/config" "github.com/edgelesssys/constellation/v2/internal/config"
"github.com/edgelesssys/constellation/v2/internal/constants" "github.com/edgelesssys/constellation/v2/internal/constants"
"github.com/edgelesssys/constellation/v2/internal/file"
"github.com/edgelesssys/constellation/v2/internal/imagefetcher"
internalk8s "github.com/edgelesssys/constellation/v2/internal/kubernetes" internalk8s "github.com/edgelesssys/constellation/v2/internal/kubernetes"
"github.com/edgelesssys/constellation/v2/internal/kubernetes/kubectl" "github.com/edgelesssys/constellation/v2/internal/kubernetes/kubectl"
conretry "github.com/edgelesssys/constellation/v2/internal/retry" conretry "github.com/edgelesssys/constellation/v2/internal/retry"
"github.com/edgelesssys/constellation/v2/internal/semver"
"github.com/edgelesssys/constellation/v2/internal/versions" "github.com/edgelesssys/constellation/v2/internal/versions"
"github.com/edgelesssys/constellation/v2/internal/versions/components" "github.com/edgelesssys/constellation/v2/internal/versions/components"
updatev1alpha1 "github.com/edgelesssys/constellation/v2/operators/constellation-node-operator/v2/api/v1alpha1" updatev1alpha1 "github.com/edgelesssys/constellation/v2/operators/constellation-node-operator/v2/api/v1alpha1"
@ -73,15 +69,12 @@ func (e *applyError) Error() string {
// KubeCmd handles interaction with the cluster's components using the CLI. // KubeCmd handles interaction with the cluster's components using the CLI.
type KubeCmd struct { type KubeCmd struct {
kubectl kubectlInterface kubectl kubectlInterface
imageFetcher imageFetcher
outWriter io.Writer
fileHandler file.Handler
retryInterval time.Duration retryInterval time.Duration
log debugLog log debugLog
} }
// New returns a new KubeCmd. // New returns a new KubeCmd.
func New(outWriter io.Writer, kubeConfig []byte, fileHandler file.Handler, log debugLog) (*KubeCmd, error) { func New(kubeConfig []byte, log debugLog) (*KubeCmd, error) {
client, err := kubectl.NewFromConfig(kubeConfig) client, err := kubectl.NewFromConfig(kubeConfig)
if err != nil { if err != nil {
return nil, fmt.Errorf("creating kubectl client: %w", err) return nil, fmt.Errorf("creating kubectl client: %w", err)
@ -89,103 +82,93 @@ func New(outWriter io.Writer, kubeConfig []byte, fileHandler file.Handler, log d
return &KubeCmd{ return &KubeCmd{
kubectl: client, kubectl: client,
fileHandler: fileHandler,
imageFetcher: imagefetcher.New(),
outWriter: outWriter,
retryInterval: time.Second * 5, retryInterval: time.Second * 5,
log: log, log: log,
}, nil }, nil
} }
// UpgradeNodeVersion upgrades the cluster's NodeVersion object and in turn triggers image & k8s version upgrades. // UpgradeNodeImage upgrades the image version of a Constellation cluster.
// The versions set in the config are validated against the versions running in the cluster. func (k *KubeCmd) UpgradeNodeImage(ctx context.Context, imageVersion semver.Semver, imageReference string, force bool) error {
// TODO(elchead): AB#3434 Split K8s and image upgrade of UpgradeNodeVersion.
func (k *KubeCmd) UpgradeNodeVersion(ctx context.Context, conf *config.Config, force, skipImage, skipK8s bool) error {
provider := conf.GetProvider()
attestationVariant := conf.GetAttestationConfig().GetVariant()
region := conf.GetRegion()
imageReference, err := k.imageFetcher.FetchReference(ctx, provider, attestationVariant, conf.Image, region)
if err != nil {
return fmt.Errorf("fetching image reference: %w", err)
}
imageVersion, err := versionsapi.NewVersionFromShortPath(conf.Image, versionsapi.VersionKindImage)
if err != nil {
return fmt.Errorf("parsing version from image short path: %w", err)
}
nodeVersion, err := k.getConstellationVersion(ctx) nodeVersion, err := k.getConstellationVersion(ctx)
if err != nil { if err != nil {
return err return err
} }
upgradeErrs := []error{} k.log.Debugf("Checking if image upgrade is valid")
var upgradeErr *compatibility.InvalidUpgradeError var upgradeErr *compatibility.InvalidUpgradeError
if !skipImage { err = k.isValidImageUpgrade(nodeVersion, imageVersion.String(), force)
err = k.isValidImageUpgrade(nodeVersion, imageVersion.Version(), force) switch {
switch { case errors.As(err, &upgradeErr):
case errors.As(err, &upgradeErr): return fmt.Errorf("skipping image upgrade: %w", err)
upgradeErrs = append(upgradeErrs, fmt.Errorf("skipping image upgrades: %w", err)) case err != nil:
case err != nil: return fmt.Errorf("updating image version: %w", err)
return fmt.Errorf("updating image version: %w", err)
}
// TODO(3u13r): remove `reconcileKubeadmConfigMap` after v2.14.0 has been released.
if err := k.reconcileKubeadmConfigMap(ctx); err != nil {
return fmt.Errorf("reconciling kubeadm config: %w", err)
}
k.log.Debugf("Updating local copy of nodeVersion image version from %s to %s", nodeVersion.Spec.ImageVersion, imageVersion.Version())
nodeVersion.Spec.ImageReference = imageReference
nodeVersion.Spec.ImageVersion = imageVersion.Version()
} }
if !skipK8s { // TODO(3u13r): remove `reconcileKubeadmConfigMap` after v2.14.0 has been released.
// We have to allow users to specify outdated k8s patch versions. if err := k.reconcileKubeadmConfigMap(ctx); err != nil {
// Therefore, this code has to skip k8s updates if a user configures an outdated (i.e. invalid) k8s version. return fmt.Errorf("reconciling kubeadm config: %w", err)
var components *corev1.ConfigMap }
_, err = versions.NewValidK8sVersion(string(conf.KubernetesVersion), true)
if err != nil { k.log.Debugf("Updating local copy of nodeVersion image version from %s to %s", nodeVersion.Spec.ImageVersion, imageVersion.String())
innerErr := fmt.Errorf("unsupported Kubernetes version, supported versions are %s", nodeVersion.Spec.ImageReference = imageReference
strings.Join(versions.SupportedK8sVersions(), ", ")) nodeVersion.Spec.ImageVersion = imageVersion.String()
err = compatibility.NewInvalidUpgradeError(nodeVersion.Spec.KubernetesClusterVersion,
string(conf.KubernetesVersion), innerErr) updatedNodeVersion, err := k.applyNodeVersion(ctx, nodeVersion)
if err != nil {
return fmt.Errorf("applying upgrade: %w", err)
}
return checkForApplyError(nodeVersion, updatedNodeVersion)
}
// UpgradeKubernetesVersion upgrades the Kubernetes version of a Constellation cluster.
func (k *KubeCmd) UpgradeKubernetesVersion(ctx context.Context, kubernetesVersion versions.ValidK8sVersion, force bool) error {
nodeVersion, err := k.getConstellationVersion(ctx)
if err != nil {
return err
}
var upgradeErr *compatibility.InvalidUpgradeError
// We have to allow users to specify outdated k8s patch versions.
// Therefore, this code has to skip k8s updates if a user configures an outdated (i.e. invalid) k8s version.
var components *corev1.ConfigMap
_, err = versions.NewValidK8sVersion(string(kubernetesVersion), true)
if err != nil {
err = compatibility.NewInvalidUpgradeError(
nodeVersion.Spec.KubernetesClusterVersion,
string(kubernetesVersion),
fmt.Errorf("unsupported Kubernetes version, supported versions are %s", strings.Join(versions.SupportedK8sVersions(), ", ")),
)
} else {
versionConfig, ok := versions.VersionConfigs[kubernetesVersion]
if !ok {
err = compatibility.NewInvalidUpgradeError(
nodeVersion.Spec.KubernetesClusterVersion,
string(kubernetesVersion),
fmt.Errorf("no version config matching K8s %s", kubernetesVersion),
)
} else { } else {
versionConfig, ok := versions.VersionConfigs[conf.KubernetesVersion] components, err = k.prepareUpdateK8s(&nodeVersion, versionConfig.ClusterVersion,
if !ok { versionConfig.KubernetesComponents, force)
err = compatibility.NewInvalidUpgradeError(nodeVersion.Spec.KubernetesClusterVersion,
string(conf.KubernetesVersion), fmt.Errorf("no version config matching K8s %s", conf.KubernetesVersion))
} else {
components, err = k.prepareUpdateK8s(&nodeVersion, versionConfig.ClusterVersion,
versionConfig.KubernetesComponents, force)
}
}
switch {
case err == nil:
err := k.applyComponentsCM(ctx, components)
if err != nil {
return fmt.Errorf("applying k8s components ConfigMap: %w", err)
}
case errors.As(err, &upgradeErr):
upgradeErrs = append(upgradeErrs, fmt.Errorf("skipping Kubernetes upgrades: %w", err))
default:
return fmt.Errorf("updating Kubernetes version: %w", err)
} }
} }
if len(upgradeErrs) == 2 {
return errors.Join(upgradeErrs...) switch {
case err == nil:
err := k.applyComponentsCM(ctx, components)
if err != nil {
return fmt.Errorf("applying k8s components ConfigMap: %w", err)
}
case errors.As(err, &upgradeErr):
return fmt.Errorf("skipping Kubernetes upgrade: %w", err)
default:
return fmt.Errorf("updating Kubernetes version: %w", err)
} }
updatedNodeVersion, err := k.applyNodeVersion(ctx, nodeVersion) updatedNodeVersion, err := k.applyNodeVersion(ctx, nodeVersion)
if err != nil { if err != nil {
return fmt.Errorf("applying upgrade: %w", err) return fmt.Errorf("applying upgrade: %w", err)
} }
return checkForApplyError(nodeVersion, updatedNodeVersion)
if err := checkForApplyError(nodeVersion, updatedNodeVersion); err != nil {
return err
}
return errors.Join(upgradeErrs...)
} }
// ClusterStatus returns a map from node name to NodeStatus. // ClusterStatus returns a map from node name to NodeStatus.
@ -203,8 +186,8 @@ func (k *KubeCmd) ClusterStatus(ctx context.Context) (map[string]NodeStatus, err
return clusterStatus, nil return clusterStatus, nil
} }
// GetClusterAttestationConfig fetches the join-config configmap from the cluster, extracts the config // GetClusterAttestationConfig fetches the join-config configmap from the cluster,
// and returns both the full configmap and the attestation config. // and returns the attestation config.
func (k *KubeCmd) GetClusterAttestationConfig(ctx context.Context, variant variant.Variant) (config.AttestationCfg, error) { func (k *KubeCmd) GetClusterAttestationConfig(ctx context.Context, variant variant.Variant) (config.AttestationCfg, error) {
existingConf, err := retryGetJoinConfig(ctx, k.kubectl, k.retryInterval, k.log) existingConf, err := retryGetJoinConfig(ctx, k.kubectl, k.retryInterval, k.log)
if err != nil { if err != nil {
@ -262,7 +245,7 @@ func (k *KubeCmd) ApplyJoinConfig(ctx context.Context, newAttestConfig config.At
} }
// ExtendClusterConfigCertSANs extends the ClusterConfig stored under "kube-system/kubeadm-config" with the given SANs. // ExtendClusterConfigCertSANs extends the ClusterConfig stored under "kube-system/kubeadm-config" with the given SANs.
// Existing SANs are preserved. // Empty strings are ignored, existing SANs are preserved.
func (k *KubeCmd) ExtendClusterConfigCertSANs(ctx context.Context, alternativeNames []string) error { func (k *KubeCmd) ExtendClusterConfigCertSANs(ctx context.Context, alternativeNames []string) error {
clusterConfiguration, kubeadmConfig, err := k.getClusterConfiguration(ctx) clusterConfiguration, kubeadmConfig, err := k.getClusterConfiguration(ctx)
if err != nil { if err != nil {
@ -305,7 +288,7 @@ func (k *KubeCmd) ExtendClusterConfigCertSANs(ctx context.Context, alternativeNa
return fmt.Errorf("setting new kubeadm config: %w", err) return fmt.Errorf("setting new kubeadm config: %w", err)
} }
fmt.Fprintln(k.outWriter, "Successfully extended the cluster's apiserver SAN field") k.log.Debugf("Successfully extended the cluster's apiserver SAN field")
return nil return nil
} }
@ -434,7 +417,7 @@ func (k *KubeCmd) reconcileKubeadmConfigMap(ctx context.Context) error {
return fmt.Errorf("setting new kubeadm config: %w", err) return fmt.Errorf("setting new kubeadm config: %w", err)
} }
fmt.Fprintln(k.outWriter, "Successfully reconciled the cluster's kubeadm config") k.log.Debugf("Successfully reconciled the cluster's kubeadm config")
return nil return nil
} }
@ -565,11 +548,3 @@ type kubectlInterface interface {
type debugLog interface { type debugLog interface {
Debugf(format string, args ...any) Debugf(format string, args ...any)
} }
// imageFetcher gets an image reference from the versionsapi.
type imageFetcher interface {
FetchReference(ctx context.Context,
provider cloudprovider.Provider, attestationVariant variant.Variant,
image, region string,
) (string, error)
}

View file

@ -10,18 +10,17 @@ import (
"context" "context"
"encoding/json" "encoding/json"
"errors" "errors"
"io" "fmt"
"strings" "strings"
"testing" "testing"
"time" "time"
"github.com/edgelesssys/constellation/v2/internal/attestation/measurements" "github.com/edgelesssys/constellation/v2/internal/attestation/measurements"
"github.com/edgelesssys/constellation/v2/internal/attestation/variant"
"github.com/edgelesssys/constellation/v2/internal/cloud/cloudprovider"
"github.com/edgelesssys/constellation/v2/internal/compatibility" "github.com/edgelesssys/constellation/v2/internal/compatibility"
"github.com/edgelesssys/constellation/v2/internal/config" "github.com/edgelesssys/constellation/v2/internal/config"
"github.com/edgelesssys/constellation/v2/internal/constants" "github.com/edgelesssys/constellation/v2/internal/constants"
"github.com/edgelesssys/constellation/v2/internal/logger" "github.com/edgelesssys/constellation/v2/internal/logger"
"github.com/edgelesssys/constellation/v2/internal/semver"
"github.com/edgelesssys/constellation/v2/internal/versions" "github.com/edgelesssys/constellation/v2/internal/versions"
"github.com/edgelesssys/constellation/v2/internal/versions/components" "github.com/edgelesssys/constellation/v2/internal/versions/components"
updatev1alpha1 "github.com/edgelesssys/constellation/v2/operators/constellation-node-operator/v2/api/v1alpha1" updatev1alpha1 "github.com/edgelesssys/constellation/v2/operators/constellation-node-operator/v2/api/v1alpha1"
@ -38,7 +37,7 @@ import (
kubeadmv1beta3 "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta3" kubeadmv1beta3 "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta3"
) )
func TestUpgradeNodeVersion(t *testing.T) { func TestUpgradeNodeImage(t *testing.T) {
clusterConf := kubeadmv1beta3.ClusterConfiguration{ clusterConf := kubeadmv1beta3.ClusterConfiguration{
APIServer: kubeadmv1beta3.APIServer{ APIServer: kubeadmv1beta3.APIServer{
ControlPlaneComponent: kubeadmv1beta3.ControlPlaneComponent{ ControlPlaneComponent: kubeadmv1beta3.ControlPlaneComponent{
@ -91,286 +90,99 @@ func TestUpgradeNodeVersion(t *testing.T) {
} }
testCases := map[string]struct { testCases := map[string]struct {
kubectl *stubKubectl conditions []metav1.Condition
conditions []metav1.Condition currentImageVersion semver.Semver
currentImageVersion string newImageVersion semver.Semver
newImageReference string badImageVersion string
badImageVersion string force bool
currentClusterVersion versions.ValidK8sVersion customKubeadmConfig *corev1.ConfigMap
conf *config.Config getCRErr error
force bool wantErr bool
getCRErr error wantUpdate bool
wantErr bool assertCorrectError func(t *testing.T, err error) bool
wantUpdate bool customClientFn func(nodeVersion updatev1alpha1.NodeVersion) unstructuredInterface
assertCorrectError func(t *testing.T, err error) bool
customClientFn func(nodeVersion updatev1alpha1.NodeVersion) unstructuredInterface
}{ }{
"success": {
conf: func() *config.Config {
conf := config.Default()
conf.Image = "v1.2.3"
conf.KubernetesVersion = supportedValidK8sVersions()[1]
return conf
}(),
currentImageVersion: "v1.2.2",
currentClusterVersion: supportedValidK8sVersions()[0],
kubectl: &stubKubectl{
configMaps: map[string]*corev1.ConfigMap{
constants.JoinConfigMap: newJoinConfigMap(`{"0":{"expected":"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA","warnOnly":false}}`),
constants.KubeadmConfigMap: validKubeadmConfig,
},
},
wantUpdate: true,
},
"success with konnectivity migration": { "success with konnectivity migration": {
conf: func() *config.Config { currentImageVersion: semver.NewFromInt(1, 2, 2, ""),
conf := config.Default() newImageVersion: semver.NewFromInt(1, 2, 3, ""),
conf.Image = "v1.2.3" customKubeadmConfig: validKubeadmConfigWithKonnectivity,
conf.KubernetesVersion = supportedValidK8sVersions()[1] wantUpdate: true,
return conf
}(),
currentImageVersion: "v1.2.2",
currentClusterVersion: supportedValidK8sVersions()[0],
kubectl: &stubKubectl{
configMaps: map[string]*corev1.ConfigMap{
constants.JoinConfigMap: newJoinConfigMap(`{"0":{"expected":"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA","warnOnly":false}}`),
constants.KubeadmConfigMap: validKubeadmConfigWithKonnectivity,
},
},
wantUpdate: true,
}, },
"only k8s upgrade": { "success": {
conf: func() *config.Config { currentImageVersion: semver.NewFromInt(1, 2, 2, ""),
conf := config.Default() newImageVersion: semver.NewFromInt(1, 2, 3, ""),
conf.Image = "v1.2.2" wantUpdate: true,
conf.KubernetesVersion = supportedValidK8sVersions()[1]
return conf
}(),
currentImageVersion: "v1.2.2",
currentClusterVersion: supportedValidK8sVersions()[0],
kubectl: &stubKubectl{
configMaps: map[string]*corev1.ConfigMap{
constants.JoinConfigMap: newJoinConfigMap(`{"0":{"expected":"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA","warnOnly":false}}`),
constants.KubeadmConfigMap: validKubeadmConfig,
},
},
wantUpdate: true,
wantErr: true,
assertCorrectError: func(t *testing.T, err error) bool {
var upgradeErr *compatibility.InvalidUpgradeError
return assert.ErrorAs(t, err, &upgradeErr)
},
},
"only image upgrade": {
conf: func() *config.Config {
conf := config.Default()
conf.Image = "v1.2.3"
conf.KubernetesVersion = supportedValidK8sVersions()[0]
return conf
}(),
currentImageVersion: "v1.2.2",
currentClusterVersion: supportedValidK8sVersions()[0],
kubectl: &stubKubectl{
configMaps: map[string]*corev1.ConfigMap{
constants.JoinConfigMap: newJoinConfigMap(`{"0":{"expected":"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA","warnOnly":false}}`),
constants.KubeadmConfigMap: validKubeadmConfig,
},
},
wantUpdate: true,
wantErr: true,
assertCorrectError: func(t *testing.T, err error) bool {
var upgradeErr *compatibility.InvalidUpgradeError
return assert.ErrorAs(t, err, &upgradeErr)
},
}, },
"not an upgrade": { "not an upgrade": {
conf: func() *config.Config { currentImageVersion: semver.NewFromInt(1, 2, 2, ""),
conf := config.Default() newImageVersion: semver.NewFromInt(1, 2, 2, ""),
conf.Image = "v1.2.2" wantErr: true,
conf.KubernetesVersion = supportedValidK8sVersions()[0]
return conf
}(),
currentImageVersion: "v1.2.2",
currentClusterVersion: supportedValidK8sVersions()[0],
kubectl: &stubKubectl{
configMaps: map[string]*corev1.ConfigMap{
constants.KubeadmConfigMap: validKubeadmConfig,
},
},
wantErr: true,
assertCorrectError: func(t *testing.T, err error) bool { assertCorrectError: func(t *testing.T, err error) bool {
var upgradeErr *compatibility.InvalidUpgradeError var upgradeErr *compatibility.InvalidUpgradeError
return assert.ErrorAs(t, err, &upgradeErr) return assert.ErrorAs(t, err, &upgradeErr)
}, },
}, },
"upgrade in progress": { "upgrade in progress": {
conf: func() *config.Config {
conf := config.Default()
conf.Image = "v1.2.3"
conf.KubernetesVersion = supportedValidK8sVersions()[1]
return conf
}(),
conditions: []metav1.Condition{{ conditions: []metav1.Condition{{
Type: updatev1alpha1.ConditionOutdated, Type: updatev1alpha1.ConditionOutdated,
Status: metav1.ConditionTrue, Status: metav1.ConditionTrue,
}}, }},
currentImageVersion: "v1.2.2", currentImageVersion: semver.NewFromInt(1, 2, 2, ""),
currentClusterVersion: supportedValidK8sVersions()[0], newImageVersion: semver.NewFromInt(1, 2, 3, ""),
kubectl: &stubKubectl{ wantErr: true,
configMaps: map[string]*corev1.ConfigMap{
constants.KubeadmConfigMap: validKubeadmConfig,
},
},
wantErr: true,
assertCorrectError: func(t *testing.T, err error) bool { assertCorrectError: func(t *testing.T, err error) bool {
return assert.ErrorIs(t, err, ErrInProgress) return assert.ErrorIs(t, err, ErrInProgress)
}, },
}, },
"success with force and upgrade in progress": { "success with force and upgrade in progress": {
conf: func() *config.Config {
conf := config.Default()
conf.Image = "v1.2.3"
conf.KubernetesVersion = supportedValidK8sVersions()[1]
return conf
}(),
conditions: []metav1.Condition{{ conditions: []metav1.Condition{{
Type: updatev1alpha1.ConditionOutdated, Type: updatev1alpha1.ConditionOutdated,
Status: metav1.ConditionTrue, Status: metav1.ConditionTrue,
}}, }},
currentImageVersion: "v1.2.2", currentImageVersion: semver.NewFromInt(1, 2, 2, ""),
currentClusterVersion: supportedValidK8sVersions()[0], newImageVersion: semver.NewFromInt(1, 2, 3, ""),
kubectl: &stubKubectl{ force: true,
configMaps: map[string]*corev1.ConfigMap{ wantUpdate: true,
constants.KubeadmConfigMap: validKubeadmConfig,
},
},
force: true,
wantUpdate: true,
}, },
"get error": { "get error": {
conf: func() *config.Config { currentImageVersion: semver.NewFromInt(1, 2, 2, ""),
conf := config.Default() newImageVersion: semver.NewFromInt(1, 2, 3, ""),
conf.Image = "v1.2.3" getCRErr: assert.AnError,
conf.KubernetesVersion = supportedValidK8sVersions()[1] wantErr: true,
return conf
}(),
currentImageVersion: "v1.2.2",
currentClusterVersion: supportedValidK8sVersions()[0],
kubectl: &stubKubectl{
configMaps: map[string]*corev1.ConfigMap{
constants.JoinConfigMap: newJoinConfigMap(`{"0":{"expected":"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA","warnOnly":false}}`),
constants.KubeadmConfigMap: validKubeadmConfig,
},
},
getCRErr: assert.AnError,
wantErr: true,
assertCorrectError: func(t *testing.T, err error) bool { assertCorrectError: func(t *testing.T, err error) bool {
return assert.ErrorIs(t, err, assert.AnError) return assert.ErrorIs(t, err, assert.AnError)
}, },
}, },
"image too new valid k8s": { "image too new": {
conf: func() *config.Config { currentImageVersion: semver.NewFromInt(1, 2, 2, ""),
conf := config.Default() newImageVersion: semver.NewFromInt(1, 4, 3, ""),
conf.Image = "v1.4.2" wantErr: true,
conf.KubernetesVersion = supportedValidK8sVersions()[1]
return conf
}(),
newImageReference: "path/to/image:v1.4.2",
currentImageVersion: "v1.2.2",
currentClusterVersion: supportedValidK8sVersions()[0],
kubectl: &stubKubectl{
configMaps: map[string]*corev1.ConfigMap{
constants.JoinConfigMap: newJoinConfigMap(`{"0":{"expected":"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA","warnOnly":true}}`),
constants.KubeadmConfigMap: validKubeadmConfig,
},
},
wantUpdate: true,
wantErr: true,
assertCorrectError: func(t *testing.T, err error) bool { assertCorrectError: func(t *testing.T, err error) bool {
var upgradeErr *compatibility.InvalidUpgradeError var upgradeErr *compatibility.InvalidUpgradeError
return assert.ErrorAs(t, err, &upgradeErr) return assert.ErrorAs(t, err, &upgradeErr)
}, },
}, },
"success with force and image too new": { "success with force and image too new": {
conf: func() *config.Config { currentImageVersion: semver.NewFromInt(1, 2, 2, ""),
conf := config.Default() newImageVersion: semver.NewFromInt(1, 2, 4, ""),
conf.Image = "v1.4.2" wantUpdate: true,
conf.KubernetesVersion = supportedValidK8sVersions()[1] force: true,
return conf
}(),
newImageReference: "path/to/image:v1.4.2",
currentImageVersion: "v1.2.2",
currentClusterVersion: supportedValidK8sVersions()[0],
kubectl: &stubKubectl{
configMaps: map[string]*corev1.ConfigMap{
constants.JoinConfigMap: newJoinConfigMap(`{"0":{"expected":"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA","warnOnly":false}}`),
constants.KubeadmConfigMap: validKubeadmConfig,
},
},
wantUpdate: true,
force: true,
}, },
"apply returns bad object": { "apply returns bad object": {
conf: func() *config.Config { currentImageVersion: semver.NewFromInt(1, 2, 2, ""),
conf := config.Default() newImageVersion: semver.NewFromInt(1, 2, 3, ""),
conf.Image = "v1.2.3" badImageVersion: "v3.2.1",
conf.KubernetesVersion = supportedValidK8sVersions()[1] wantUpdate: true,
return conf wantErr: true,
}(),
currentImageVersion: "v1.2.2",
currentClusterVersion: supportedValidK8sVersions()[0],
badImageVersion: "v3.2.1",
kubectl: &stubKubectl{
configMaps: map[string]*corev1.ConfigMap{
constants.JoinConfigMap: newJoinConfigMap(`{"0":{"expected":"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA","warnOnly":false}}`),
constants.KubeadmConfigMap: validKubeadmConfig,
},
},
wantUpdate: true,
wantErr: true,
assertCorrectError: func(t *testing.T, err error) bool { assertCorrectError: func(t *testing.T, err error) bool {
var target *applyError var target *applyError
return assert.ErrorAs(t, err, &target) return assert.ErrorAs(t, err, &target)
}, },
}, },
"outdated k8s version skips k8s upgrade": {
conf: func() *config.Config {
conf := config.Default()
conf.Image = "v1.2.2"
conf.KubernetesVersion = "v1.25.8"
return conf
}(),
currentImageVersion: "v1.2.2",
currentClusterVersion: supportedValidK8sVersions()[0],
kubectl: &stubKubectl{
configMaps: map[string]*corev1.ConfigMap{
constants.JoinConfigMap: newJoinConfigMap(`{"0":{"expected":"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA","warnOnly":false}}`),
constants.KubeadmConfigMap: validKubeadmConfig,
},
},
wantUpdate: false,
wantErr: true,
assertCorrectError: func(t *testing.T, err error) bool {
var upgradeErr *compatibility.InvalidUpgradeError
return assert.ErrorAs(t, err, &upgradeErr)
},
},
"succeed after update retry when the updated node object is outdated": { "succeed after update retry when the updated node object is outdated": {
conf: func() *config.Config { currentImageVersion: semver.NewFromInt(1, 2, 2, ""),
conf := config.Default() newImageVersion: semver.NewFromInt(1, 2, 3, ""),
conf.Image = "v1.2.3" wantUpdate: false, // because customClient is used
conf.KubernetesVersion = supportedValidK8sVersions()[1]
return conf
}(),
currentImageVersion: "v1.2.2",
currentClusterVersion: supportedValidK8sVersions()[0],
kubectl: &stubKubectl{
configMaps: map[string]*corev1.ConfigMap{
constants.JoinConfigMap: newJoinConfigMap(`{"0":{"expected":"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA","warnOnly":false}}`),
constants.KubeadmConfigMap: validKubeadmConfig,
},
},
wantUpdate: false, // because customClient is used
customClientFn: func(nodeVersion updatev1alpha1.NodeVersion) unstructuredInterface { customClientFn: func(nodeVersion updatev1alpha1.NodeVersion) unstructuredInterface {
fakeClient := &fakeUnstructuredClient{} fakeClient := &fakeUnstructuredClient{}
fakeClient.On("GetCR", mock.Anything, mock.Anything).Return(unstructedObjectWithGeneration(nodeVersion, 1), nil) fakeClient.On("GetCR", mock.Anything, mock.Anything).Return(unstructedObjectWithGeneration(nodeVersion, 1), nil)
@ -388,13 +200,15 @@ func TestUpgradeNodeVersion(t *testing.T) {
nodeVersion := updatev1alpha1.NodeVersion{ nodeVersion := updatev1alpha1.NodeVersion{
Spec: updatev1alpha1.NodeVersionSpec{ Spec: updatev1alpha1.NodeVersionSpec{
ImageVersion: tc.currentImageVersion, ImageVersion: tc.currentImageVersion.String(),
KubernetesClusterVersion: string(tc.currentClusterVersion), KubernetesClusterVersion: "v1.2.3",
}, },
Status: updatev1alpha1.NodeVersionStatus{ Status: updatev1alpha1.NodeVersionStatus{
Conditions: tc.conditions, Conditions: tc.conditions,
}, },
} }
unstrNodeVersion, err := runtime.DefaultUnstructuredConverter.ToUnstructured(&nodeVersion)
require.NoError(err)
var badUpdatedObject *unstructured.Unstructured var badUpdatedObject *unstructured.Unstructured
if tc.badImageVersion != "" { if tc.badImageVersion != "" {
@ -404,26 +218,30 @@ func TestUpgradeNodeVersion(t *testing.T) {
badUpdatedObject = &unstructured.Unstructured{Object: unstrBadNodeVersion} badUpdatedObject = &unstructured.Unstructured{Object: unstrBadNodeVersion}
} }
unstrNodeVersion, err := runtime.DefaultUnstructuredConverter.ToUnstructured(&nodeVersion)
require.NoError(err)
unstructuredClient := &stubUnstructuredClient{ unstructuredClient := &stubUnstructuredClient{
object: &unstructured.Unstructured{Object: unstrNodeVersion}, object: &unstructured.Unstructured{Object: unstrNodeVersion},
badUpdatedObject: badUpdatedObject, badUpdatedObject: badUpdatedObject,
getCRErr: tc.getCRErr, getCRErr: tc.getCRErr,
} }
tc.kubectl.unstructuredInterface = unstructuredClient kubectl := &stubKubectl{
unstructuredInterface: unstructuredClient,
configMaps: map[string]*corev1.ConfigMap{
constants.KubeadmConfigMap: validKubeadmConfig,
},
}
if tc.customClientFn != nil { if tc.customClientFn != nil {
tc.kubectl.unstructuredInterface = tc.customClientFn(nodeVersion) kubectl.unstructuredInterface = tc.customClientFn(nodeVersion)
}
if tc.customKubeadmConfig != nil {
kubectl.configMaps[constants.KubeadmConfigMap] = tc.customKubeadmConfig
} }
upgrader := KubeCmd{ upgrader := KubeCmd{
kubectl: tc.kubectl, kubectl: kubectl,
imageFetcher: &stubImageFetcher{reference: tc.newImageReference}, log: logger.NewTest(t),
log: logger.NewTest(t),
outWriter: io.Discard,
} }
err = upgrader.UpgradeNodeVersion(context.Background(), tc.conf, tc.force, false, false) err = upgrader.UpgradeNodeImage(context.Background(), tc.newImageVersion, fmt.Sprintf("/path/to/image:%s", tc.newImageVersion.String()), tc.force)
// Check upgrades first because if we checked err first, UpgradeImage may error due to other reasons and still trigger an upgrade. // Check upgrades first because if we checked err first, UpgradeImage may error due to other reasons and still trigger an upgrade.
if tc.wantUpdate { if tc.wantUpdate {
assert.NotNil(unstructuredClient.updatedObject) assert.NotNil(unstructuredClient.updatedObject)
@ -437,17 +255,128 @@ func TestUpgradeNodeVersion(t *testing.T) {
return return
} }
assert.NoError(err) assert.NoError(err)
// The ConfigMap only exists in the updatedConfigMaps map it needed to remove the Konnectivity values // If the ConfigMap only exists in the updatedConfigMaps map, the Konnectivity values should have been removed
if strings.Contains(tc.kubectl.configMaps[constants.KubeadmConfigMap].Data[constants.ClusterConfigurationKey], "konnectivity-uds") { if strings.Contains(kubectl.configMaps[constants.KubeadmConfigMap].Data[constants.ClusterConfigurationKey], "konnectivity-uds") {
assert.NotContains(tc.kubectl.updatedConfigMaps[constants.KubeadmConfigMap].Data[constants.ClusterConfigurationKey], "konnectivity-uds") assert.NotContains(kubectl.updatedConfigMaps[constants.KubeadmConfigMap].Data[constants.ClusterConfigurationKey], "konnectivity-uds")
assert.NotContains(tc.kubectl.updatedConfigMaps[constants.KubeadmConfigMap].Data[constants.ClusterConfigurationKey], "egress-config") assert.NotContains(kubectl.updatedConfigMaps[constants.KubeadmConfigMap].Data[constants.ClusterConfigurationKey], "egress-config")
assert.NotContains(tc.kubectl.updatedConfigMaps[constants.KubeadmConfigMap].Data[constants.ClusterConfigurationKey], "egress-selector-config-file") assert.NotContains(kubectl.updatedConfigMaps[constants.KubeadmConfigMap].Data[constants.ClusterConfigurationKey], "egress-selector-config-file")
} }
}) })
} }
} }
func TestUpdateImage(t *testing.T) { func TestUpgradeKubernetesVersion(t *testing.T) {
testCases := map[string]struct {
conditions []metav1.Condition
newKubernetesVersion versions.ValidK8sVersion
currentKubernetesVersion versions.ValidK8sVersion
force bool
getCRErr error
wantErr bool
wantUpdate bool
assertCorrectError func(t *testing.T, err error) bool
customClientFn func(nodeVersion updatev1alpha1.NodeVersion) unstructuredInterface
}{
"success": {
currentKubernetesVersion: supportedValidK8sVersions()[0],
newKubernetesVersion: supportedValidK8sVersions()[1],
wantUpdate: true,
wantErr: false,
},
"not an upgrade": {
currentKubernetesVersion: supportedValidK8sVersions()[0],
newKubernetesVersion: supportedValidK8sVersions()[0],
wantErr: true,
assertCorrectError: func(t *testing.T, err error) bool {
var upgradeErr *compatibility.InvalidUpgradeError
return assert.ErrorAs(t, err, &upgradeErr)
},
},
"get error": {
currentKubernetesVersion: supportedValidK8sVersions()[0],
newKubernetesVersion: supportedValidK8sVersions()[1],
getCRErr: assert.AnError,
wantErr: true,
assertCorrectError: func(t *testing.T, err error) bool {
return assert.ErrorIs(t, err, assert.AnError)
},
},
"outdated k8s version skips k8s upgrade": {
currentKubernetesVersion: supportedValidK8sVersions()[0],
newKubernetesVersion: versions.ValidK8sVersion("v1.1.0"),
wantUpdate: false,
wantErr: true,
assertCorrectError: func(t *testing.T, err error) bool {
var upgradeErr *compatibility.InvalidUpgradeError
return assert.ErrorAs(t, err, &upgradeErr)
},
},
"succeed after update retry when the updated node object is outdated": {
currentKubernetesVersion: supportedValidK8sVersions()[0],
newKubernetesVersion: supportedValidK8sVersions()[1],
wantUpdate: false, // because customClient is used
customClientFn: func(nodeVersion updatev1alpha1.NodeVersion) unstructuredInterface {
fakeClient := &fakeUnstructuredClient{}
fakeClient.On("GetCR", mock.Anything, mock.Anything).Return(unstructedObjectWithGeneration(nodeVersion, 1), nil)
fakeClient.On("UpdateCR", mock.Anything, mock.Anything).Return(nil, k8serrors.NewConflict(schema.GroupResource{Resource: nodeVersion.Name}, nodeVersion.Name, nil)).Once()
fakeClient.On("UpdateCR", mock.Anything, mock.Anything).Return(unstructedObjectWithGeneration(nodeVersion, 2), nil).Once()
return fakeClient
},
},
}
for name, tc := range testCases {
t.Run(name, func(t *testing.T) {
assert := assert.New(t)
require := require.New(t)
nodeVersion := updatev1alpha1.NodeVersion{
Spec: updatev1alpha1.NodeVersionSpec{
ImageVersion: "v1.2.3",
KubernetesClusterVersion: string(tc.currentKubernetesVersion),
},
Status: updatev1alpha1.NodeVersionStatus{
Conditions: tc.conditions,
},
}
unstrNodeVersion, err := runtime.DefaultUnstructuredConverter.ToUnstructured(&nodeVersion)
require.NoError(err)
unstructuredClient := &stubUnstructuredClient{
object: &unstructured.Unstructured{Object: unstrNodeVersion},
getCRErr: tc.getCRErr,
}
kubectl := &stubKubectl{
unstructuredInterface: unstructuredClient,
}
if tc.customClientFn != nil {
kubectl.unstructuredInterface = tc.customClientFn(nodeVersion)
}
upgrader := KubeCmd{
kubectl: kubectl,
log: logger.NewTest(t),
}
err = upgrader.UpgradeKubernetesVersion(context.Background(), tc.newKubernetesVersion, tc.force)
// Check upgrades first because if we checked err first, UpgradeImage may error due to other reasons and still trigger an upgrade.
if tc.wantUpdate {
assert.NotNil(unstructuredClient.updatedObject)
} else {
assert.Nil(unstructuredClient.updatedObject)
}
if tc.wantErr {
assert.Error(err)
tc.assertCorrectError(t, err)
return
}
assert.NoError(err)
})
}
}
func TestIsValidImageUpgrade(t *testing.T) {
someErr := errors.New("error") someErr := errors.New("error")
testCases := map[string]struct { testCases := map[string]struct {
newImageReference string newImageReference string
@ -729,7 +658,6 @@ func TestApplyJoinConfig(t *testing.T) {
kubectl: tc.kubectl, kubectl: tc.kubectl,
log: logger.NewTest(t), log: logger.NewTest(t),
retryInterval: time.Millisecond, retryInterval: time.Millisecond,
outWriter: io.Discard,
} }
err := cmd.ApplyJoinConfig(context.Background(), tc.newAttestationCfg, []byte{0x11}) err := cmd.ApplyJoinConfig(context.Background(), tc.newAttestationCfg, []byte{0x11})
@ -839,18 +767,6 @@ func (s *stubKubectl) GetNodes(_ context.Context) ([]corev1.Node, error) {
return s.nodes, s.nodesErr return s.nodes, s.nodesErr
} }
type stubImageFetcher struct {
reference string
fetchReferenceErr error
}
func (f *stubImageFetcher) FetchReference(_ context.Context,
_ cloudprovider.Provider, _ variant.Variant,
_, _ string,
) (string, error) {
return f.reference, f.fetchReferenceErr
}
func unstructedObjectWithGeneration(nodeVersion updatev1alpha1.NodeVersion, generation int64) *unstructured.Unstructured { func unstructedObjectWithGeneration(nodeVersion updatev1alpha1.NodeVersion, generation int64) *unstructured.Unstructured {
unstrNodeVersion, _ := runtime.DefaultUnstructuredConverter.ToUnstructured(&nodeVersion) unstrNodeVersion, _ := runtime.DefaultUnstructuredConverter.ToUnstructured(&nodeVersion)
object := &unstructured.Unstructured{Object: unstrNodeVersion} object := &unstructured.Unstructured{Object: unstrNodeVersion}

View file

@ -0,0 +1,89 @@
/*
Copyright (c) Edgeless Systems GmbH
SPDX-License-Identifier: AGPL-3.0-only
*/
package constellation
import (
"context"
"errors"
"fmt"
"github.com/edgelesssys/constellation/v2/internal/attestation/variant"
"github.com/edgelesssys/constellation/v2/internal/config"
"github.com/edgelesssys/constellation/v2/internal/file"
"github.com/edgelesssys/constellation/v2/internal/semver"
"github.com/edgelesssys/constellation/v2/internal/versions"
apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
)
var errKubecmdNotInitialised = errors.New("kubernetes client not initialized")
// ExtendClusterConfigCertSANs extends the ClusterConfig stored under "kube-system/kubeadm-config" with the given SANs.
func (a *Applier) ExtendClusterConfigCertSANs(ctx context.Context, clusterEndpoint, customEndpoint string, additionalAPIServerCertSANs []string) error {
if a.kubecmdClient == nil {
return errKubecmdNotInitialised
}
sans := append([]string{clusterEndpoint, customEndpoint}, additionalAPIServerCertSANs...)
if err := a.kubecmdClient.ExtendClusterConfigCertSANs(ctx, sans); err != nil {
return fmt.Errorf("extending cert SANs: %w", err)
}
return nil
}
// GetClusterAttestationConfig returns the attestation config currently set for the cluster.
func (a *Applier) GetClusterAttestationConfig(ctx context.Context, variant variant.Variant) (config.AttestationCfg, error) {
if a.kubecmdClient == nil {
return nil, errKubecmdNotInitialised
}
return a.kubecmdClient.GetClusterAttestationConfig(ctx, variant)
}
// ApplyJoinConfig creates or updates the Constellation cluster's join-config ConfigMap.
func (a *Applier) ApplyJoinConfig(ctx context.Context, newAttestConfig config.AttestationCfg, measurementSalt []byte) error {
if a.kubecmdClient == nil {
return errKubecmdNotInitialised
}
return a.kubecmdClient.ApplyJoinConfig(ctx, newAttestConfig, measurementSalt)
}
// UpgradeNodeImage upgrades the node image of the cluster to the given version.
func (a *Applier) UpgradeNodeImage(ctx context.Context, imageVersion semver.Semver, imageReference string, force bool) error {
if a.kubecmdClient == nil {
return errKubecmdNotInitialised
}
return a.kubecmdClient.UpgradeNodeImage(ctx, imageVersion, imageReference, force)
}
// UpgradeKubernetesVersion upgrades the Kubernetes version of the cluster to the given version.
func (a *Applier) UpgradeKubernetesVersion(ctx context.Context, kubernetesVersion versions.ValidK8sVersion, force bool) error {
if a.kubecmdClient == nil {
return errKubecmdNotInitialised
}
return a.kubecmdClient.UpgradeKubernetesVersion(ctx, kubernetesVersion, force)
}
// BackupCRDs backs up all CRDs to the upgrade workspace.
func (a *Applier) BackupCRDs(ctx context.Context, fileHandler file.Handler, upgradeDir string) ([]apiextensionsv1.CustomResourceDefinition, error) {
if a.kubecmdClient == nil {
return nil, errKubecmdNotInitialised
}
return a.kubecmdClient.BackupCRDs(ctx, fileHandler, upgradeDir)
}
// BackupCRs backs up all CRs to the upgrade workspace.
func (a *Applier) BackupCRs(ctx context.Context, fileHandler file.Handler, crds []apiextensionsv1.CustomResourceDefinition, upgradeDir string) error {
if a.kubecmdClient == nil {
return errKubecmdNotInitialised
}
return a.kubecmdClient.BackupCRs(ctx, fileHandler, crds, upgradeDir)
}