cli: remove helm management from join-config (#2251)

* Replace UpdateAttestationConfig with ApplyJoinConfig

* Dont set up join-config over Helm, it is now only managed by our CLI directly during init and upgrade

* Remove measurementSalt and attestationConfig parsing from helm, they were only needed for the JoinConfig

* Add migration step to remove join-config from Helm management

* Update attestation config trouble shooting tip

---------

Signed-off-by: Daniel Weiße <dw@edgeless.systems>

cli/internal/cmd/init.go

@@ -39,6 +39,7 @@ import (
 	"github.com/edgelesssys/constellation/v2/cli/internal/clusterid"
 	"github.com/edgelesssys/constellation/v2/cli/internal/cmd/pathprefix"
 	"github.com/edgelesssys/constellation/v2/cli/internal/helm"
+	"github.com/edgelesssys/constellation/v2/cli/internal/kubecmd"
 	"github.com/edgelesssys/constellation/v2/cli/internal/terraform"
 	"github.com/edgelesssys/constellation/v2/internal/cloud/cloudprovider"
 	"github.com/edgelesssys/constellation/v2/internal/config"
@@ -129,13 +130,20 @@ func runInitialize(cmd *cobra.Command, _ []string) error {
 		return fmt.Errorf("creating Helm installer: %w", err)
 	}
 	i := newInitCmd(tfClient, helmInstaller, fileHandler, spinner, &kubeconfigMerger{log: log}, log)
+
 	fetcher := attestationconfigapi.NewFetcher()
-	return i.initialize(cmd, newDialer, license.NewClient(), fetcher)
+	newAttestationApplier := func(w io.Writer, kubeConfig string, log debugLog) (attestationConfigApplier, error) {
+		return kubecmd.New(w, kubeConfig, log)
+	}
+
+	return i.initialize(cmd, newDialer, license.NewClient(), fetcher, newAttestationApplier)
 }
 
 // initialize initializes a Constellation.
-func (i *initCmd) initialize(cmd *cobra.Command, newDialer func(validator atls.Validator) *dialer.Dialer,
+func (i *initCmd) initialize(
+	cmd *cobra.Command, newDialer func(validator atls.Validator) *dialer.Dialer,
 	quotaChecker license.QuotaChecker, configFetcher attestationconfigapi.Fetcher,
+	newAttestationApplier func(io.Writer, string, debugLog) (attestationConfigApplier, error),
 ) error {
 	flags, err := i.evalFlagArgs(cmd)
 	if err != nil {
@@ -249,6 +257,14 @@ func (i *initCmd) initialize(cmd *cobra.Command, newDialer func(validator atls.V
 		return err
 	}
 
+	attestationApplier, err := newAttestationApplier(cmd.OutOrStdout(), constants.AdminConfFilename, i.log)
+	if err != nil {
+		return err
+	}
+	if err := attestationApplier.ApplyJoinConfig(cmd.Context(), conf.GetAttestationConfig(), measurementSalt); err != nil {
+		return fmt.Errorf("applying attestation config: %w", err)
+	}
+
 	helmLoader := helm.NewLoader(provider, k8sVersion, clusterName)
 	i.log.Debugf("Created new Helm loader")
 	output, err := i.clusterShower.ShowCluster(cmd.Context(), conf.GetProvider())
@@ -609,3 +625,7 @@ func (e *nonRetriableError) Unwrap() error {
 type initializer interface {
 	Install(ctx context.Context, releases *helm.Releases) error
 }
+
+type attestationConfigApplier interface {
+	ApplyJoinConfig(ctx context.Context, newAttestConfig config.AttestationCfg, measurementSalt []byte) error
+}

cli/internal/cmd/init_test.go

@@ -187,7 +187,15 @@ func TestInitialize(t *testing.T) {
 			defer cancel()
 			cmd.SetContext(ctx)
 			i := newInitCmd(&stubShowCluster{}, &stubHelmInstaller{}, fileHandler, &nopSpinner{}, nil, logger.NewTest(t))
-			err := i.initialize(cmd, newDialer, &stubLicenseClient{}, stubAttestationFetcher{})
+			err := i.initialize(
+				cmd,
+				newDialer,
+				&stubLicenseClient{},
+				stubAttestationFetcher{},
+				func(io.Writer, string, debugLog) (attestationConfigApplier, error) {
+					return &stubAttestationApplier{}, nil
+				},
+			)
 
 			if tc.wantErr {
 				assert.Error(err)
@@ -486,7 +494,15 @@ func TestAttestation(t *testing.T) {
 	cmd.SetContext(ctx)
 
 	i := newInitCmd(nil, nil, fileHandler, &nopSpinner{}, nil, logger.NewTest(t))
-	err := i.initialize(cmd, newDialer, &stubLicenseClient{}, stubAttestationFetcher{})
+	err := i.initialize(
+		cmd,
+		newDialer,
+		&stubLicenseClient{},
+		stubAttestationFetcher{},
+		func(io.Writer, string, debugLog) (attestationConfigApplier, error) {
+			return &stubAttestationApplier{}, nil
+		},
+	)
 	assert.Error(err)
 	// make sure the error is actually a TLS handshake error
 	assert.Contains(err.Error(), "transport: authentication handshake failed")
@@ -666,3 +682,11 @@ func (s *stubShowCluster) ShowCluster(_ context.Context, csp cloudprovider.Provi
 	}
 	return res, nil
 }
+
+type stubAttestationApplier struct {
+	applyErr error
+}
+
+func (a *stubAttestationApplier) ApplyJoinConfig(_ context.Context, _ config.AttestationCfg, _ []byte) error {
+	return a.applyErr
+}

cli/internal/cmd/miniup.go

@@ -10,12 +10,14 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"io"
 	"net"
 
 	"github.com/edgelesssys/constellation/v2/cli/internal/cloudcmd"
 	"github.com/edgelesssys/constellation/v2/cli/internal/cmd/pathprefix"
 	"github.com/edgelesssys/constellation/v2/cli/internal/featureset"
 	"github.com/edgelesssys/constellation/v2/cli/internal/helm"
+	"github.com/edgelesssys/constellation/v2/cli/internal/kubecmd"
 	"github.com/edgelesssys/constellation/v2/cli/internal/libvirt"
 	"github.com/edgelesssys/constellation/v2/cli/internal/terraform"
 	"github.com/edgelesssys/constellation/v2/internal/api/attestationconfigapi"
@@ -215,8 +217,12 @@ func (m *miniUpCmd) initializeMiniCluster(cmd *cobra.Command, fileHandler file.H
 		return fmt.Errorf("creating Terraform client: %w", err)
 	}
 
+	newAttestationApplier := func(w io.Writer, kubeConfig string, log debugLog) (attestationConfigApplier, error) {
+		return kubecmd.New(w, kubeConfig, log)
+	}
+
 	i := newInitCmd(tfClient, helmInstaller, fileHandler, spinner, &kubeconfigMerger{log: log}, log)
-	if err := i.initialize(cmd, newDialer, license.NewClient(), m.configFetcher); err != nil {
+	if err := i.initialize(cmd, newDialer, license.NewClient(), m.configFetcher, newAttestationApplier); err != nil {
 		return err
 	}
 	m.log.Debugf("Initialized mini cluster")

cli/internal/cmd/upgradeapply.go

@@ -29,6 +29,7 @@ import (
 	"github.com/edgelesssys/constellation/v2/internal/file"
 	"github.com/edgelesssys/constellation/v2/internal/kms/uri"
 	"github.com/edgelesssys/constellation/v2/internal/kubernetes/kubectl"
+	"github.com/edgelesssys/constellation/v2/internal/semver"
 	"github.com/edgelesssys/constellation/v2/internal/versions"
 	"github.com/rogpeppe/go-internal/diff"
 	"github.com/spf13/afero"
@@ -153,7 +154,15 @@ func (u *upgradeApplyCmd) upgradeApply(cmd *cobra.Command) error {
 	}
 	conf.UpdateMAAURL(idFile.AttestationURL)
 
-	if err := u.confirmAttestationConfigUpgrade(cmd, conf.GetAttestationConfig(), flags); err != nil {
+	// Apply migrations necessary for the upgrade
+	if err := migrateFrom2_10(cmd.Context(), u.kubeUpgrader); err != nil {
+		return fmt.Errorf("applying migration for upgrading from v2.10: %w", err)
+	}
+	if err := migrateFrom2_11(cmd.Context(), u.kubeUpgrader); err != nil {
+		return fmt.Errorf("applying migration for upgrading from v2.11: %w", err)
+	}
+
+	if err := u.confirmAndUpgradeAttestationConfig(cmd, conf.GetAttestationConfig(), idFile.MeasurementSalt, flags); err != nil {
 		return fmt.Errorf("upgrading measurements: %w", err)
 	}
 
@@ -177,7 +186,11 @@ func (u *upgradeApplyCmd) upgradeApply(cmd *cobra.Command) error {
 		return fmt.Errorf("extending cert SANs: %w", err)
 	}
 
-	if conf.GetProvider() == cloudprovider.Azure || conf.GetProvider() == cloudprovider.GCP || conf.GetProvider() == cloudprovider.AWS {
+	if conf.GetProvider() != cloudprovider.Azure && conf.GetProvider() != cloudprovider.GCP && conf.GetProvider() != cloudprovider.AWS {
+		cmd.PrintErrln("WARNING: Skipping service and image upgrades, which are currently only supported for AWS, Azure, and GCP.")
+		return nil
+	}
+
 	var upgradeErr *compatibility.InvalidUpgradeError
 	err = u.handleServiceUpgrade(cmd, conf, idFile, tfOutput, validK8sVersion, flags)
 	switch {
@@ -198,9 +211,6 @@ func (u *upgradeApplyCmd) upgradeApply(cmd *cobra.Command) error {
 	case err != nil:
 		return fmt.Errorf("upgrading NodeVersion: %w", err)
 	}
-	} else {
-		cmd.PrintErrln("WARNING: Skipping service and image upgrades, which are currently only supported for AWS, Azure, and GCP.")
-	}
 
 	return nil
 }
@@ -338,9 +348,11 @@ func validK8sVersion(cmd *cobra.Command, version string, yes bool) (validVersion
 	return validVersion, nil
 }
 
-// confirmAttestationConfigUpgrade checks if the locally configured measurements are different from the cluster's measurements.
+// confirmAndUpgradeAttestationConfig checks if the locally configured measurements are different from the cluster's measurements.
 // If so the function will ask the user to confirm (if --yes is not set) and upgrade the cluster's config.
-func (u *upgradeApplyCmd) confirmAttestationConfigUpgrade(cmd *cobra.Command, newConfig config.AttestationCfg, flags upgradeApplyFlags) error {
+func (u *upgradeApplyCmd) confirmAndUpgradeAttestationConfig(
+	cmd *cobra.Command, newConfig config.AttestationCfg, measurementSalt []byte, flags upgradeApplyFlags,
+) error {
 	clusterAttestationConfig, err := u.kubeUpgrader.GetClusterAttestationConfig(cmd.Context(), newConfig.GetVariant())
 	if err != nil {
 		return fmt.Errorf("getting cluster attestation config: %w", err)
@@ -371,9 +383,10 @@ func (u *upgradeApplyCmd) confirmAttestationConfigUpgrade(cmd *cobra.Command, ne
 		}
 	}
 
-	if err := u.kubeUpgrader.UpdateAttestationConfig(cmd.Context(), newConfig); err != nil {
+	if err := u.kubeUpgrader.ApplyJoinConfig(cmd.Context(), newConfig, measurementSalt); err != nil {
 		return fmt.Errorf("updating attestation config: %w", err)
 	}
+	cmd.Println("Successfully update the cluster's attestation config")
 	return nil
 }
 
@@ -413,6 +426,34 @@ func (u *upgradeApplyCmd) handleServiceUpgrade(cmd *cobra.Command, conf *config.
 	return err
 }
 
+// migrateFrom2_10 applies migrations necessary for upgrading from v2.10 to v2.11
+// TODO(v2.11): Remove this function after v2.11 is released.
+func migrateFrom2_10(ctx context.Context, kubeUpgrader kubernetesUpgrader) error {
+	// Sanity check to make sure we only run migrations on upgrades with CLI version 2.10 < v < 2.12
+	if !constants.BinaryVersion().MajorMinorEqual(semver.NewFromInt(2, 11, 0, "")) {
+		return nil
+	}
+
+	if err := kubeUpgrader.RemoveAttestationConfigHelmManagement(ctx); err != nil {
+		return fmt.Errorf("removing helm management from attestation config: %w", err)
+	}
+	return nil
+}
+
+// migrateFrom2_11 applies migrations necessary for upgrading from v2.11 to v2.12
+// TODO(v2.12): Remove this function after v2.12 is released.
+func migrateFrom2_11(ctx context.Context, kubeUpgrader kubernetesUpgrader) error {
+	// Sanity check to make sure we only run migrations on upgrades with CLI version 2.11 < v < 2.13
+	if !constants.BinaryVersion().MajorMinorEqual(semver.NewFromInt(2, 12, 0, "")) {
+		return nil
+	}
+
+	if err := kubeUpgrader.RemoveHelmKeepAnnotation(ctx); err != nil {
+		return fmt.Errorf("removing helm keep annotation: %w", err)
+	}
+	return nil
+}
+
 func parseUpgradeApplyFlags(cmd *cobra.Command) (upgradeApplyFlags, error) {
 	workDir, err := cmd.Flags().GetString("workspace")
 	if err != nil {
@@ -493,7 +534,11 @@ type kubernetesUpgrader interface {
 	UpgradeNodeVersion(ctx context.Context, conf *config.Config, force bool) error
 	ExtendClusterConfigCertSANs(ctx context.Context, alternativeNames []string) error
 	GetClusterAttestationConfig(ctx context.Context, variant variant.Variant) (config.AttestationCfg, error)
-	UpdateAttestationConfig(ctx context.Context, newAttestConfig config.AttestationCfg) error
+	ApplyJoinConfig(ctx context.Context, newAttestConfig config.AttestationCfg, measurementSalt []byte) error
+	// TODO(v2.11): Remove this function after v2.11 is released.
+	RemoveAttestationConfigHelmManagement(ctx context.Context) error
+	// TODO(v2.12): Remove this function after v2.12 is released.
+	RemoveHelmKeepAnnotation(ctx context.Context) error
 }
 
 type helmUpgrader interface {

cli/internal/cmd/upgradeapply_test.go

@@ -197,7 +197,7 @@ func (u stubKubernetesUpgrader) UpgradeNodeVersion(_ context.Context, _ *config.
 	return u.nodeVersionErr
 }
 
-func (u stubKubernetesUpgrader) UpdateAttestationConfig(_ context.Context, _ config.AttestationCfg) error {
+func (u stubKubernetesUpgrader) ApplyJoinConfig(_ context.Context, _ config.AttestationCfg, _ []byte) error {
 	return nil
 }
 
@@ -209,6 +209,16 @@ func (u stubKubernetesUpgrader) ExtendClusterConfigCertSANs(_ context.Context, _
 	return nil
 }
 
+// TODO(v2.11): Remove this function.
+func (u stubKubernetesUpgrader) RemoveAttestationConfigHelmManagement(_ context.Context) error {
+	return nil
+}
+
+// TODO(v2.12): Remove this function.
+func (u stubKubernetesUpgrader) RemoveHelmKeepAnnotation(_ context.Context) error {
+	return nil
+}
+
 type stubTerraformUpgrader struct {
 	terraformDiff     bool
 	planTerraformErr  error

cli/internal/helm/BUILD.bazel

@@ -226,7 +226,6 @@ go_library(
         "charts/edgeless/constellation-services/charts/join-service/Chart.yaml",
         "charts/edgeless/constellation-services/charts/join-service/templates/clusterrole.yaml",
         "charts/edgeless/constellation-services/charts/join-service/templates/clusterrolebinding.yaml",
-        "charts/edgeless/constellation-services/charts/join-service/templates/configmap.yaml",
         "charts/edgeless/constellation-services/charts/join-service/templates/daemonset.yaml",
         "charts/edgeless/constellation-services/charts/join-service/templates/service.yaml",
         "charts/edgeless/constellation-services/charts/join-service/templates/serviceaccount.yaml",
@@ -468,7 +467,6 @@ go_test(
     deps = [
         "//cli/internal/clusterid",
         "//cli/internal/terraform",
-        "//internal/attestation/idkeydigest",
         "//internal/attestation/measurements",
         "//internal/cloud/azureshared",
         "//internal/cloud/cloudprovider",

cli/internal/helm/charts/edgeless/constellation-services/charts/join-service/templates/configmap.yaml

@@ -1,10 +0,0 @@
-apiVersion: v1
-kind: ConfigMap
-metadata:
-  name: join-config
-  namespace: {{ .Release.Namespace }}
-data:
-  {{/* mustToJson is required so the json-strings passed from go are of type string in the rendered yaml. */}}
-  attestationConfig: {{ .Values.attestationConfig | mustToJson }}
-binaryData:
-  measurementSalt: {{ .Values.measurementSalt }}

cli/internal/helm/charts/edgeless/constellation-services/charts/join-service/values.schema.json

@@ -3,33 +3,33 @@
     "properties": {
         "csp": {
             "description": "CSP to which the chart is deployed.",
-            "enum": ["AWS", "Azure", "GCP", "OpenStack", "QEMU"]
+            "enum": [
-        },
+                "AWS",
-        "attestationConfig": {
+                "Azure",
-            "description": "JSON-string to describe the config to use for attestation validation.",
+                "GCP",
-            "type": "string",
+                "OpenStack",
-            "examples": ["{'measurements':{'1':{'expected':'AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA','warnOnly':true},'15':{'expected':'AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=','warnOnly':true}}}"]
+                "QEMU"
+            ]
         },
         "image": {
             "description": "Container image to use for the spawned pods.",
             "type": "string",
-            "examples": ["ghcr.io/edgelesssys/constellation/join-service:latest"]
+            "examples": [
-        },
+                "ghcr.io/edgelesssys/constellation/join-service:latest"
-        "measurementSalt": {
+            ]
-            "description": "Salt used to generate node measurements",
-            "type": "string",
-            "examples": ["AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"]
         },
         "attestationVariant": {
             "description": "Attestation variant to use for aTLS connections.",
             "type": "string",
-            "examples": ["azure-sev-snp", "azure-trusted-launch", "gcp-sev-es"]
+            "examples": [
+                "azure-sev-snp",
+                "azure-trusted-launch",
+                "gcp-sev-es"
+            ]
         }
     },
     "required": [
         "csp",
-        "attestationConfig",
-        "measurementSalt",
         "image",
         "attestationVariant"
     ],

cli/internal/helm/charts/edgeless/constellation-services/charts/join-service/values.yaml

@@ -1,5 +1,4 @@
 csp: "gcp"
 attestationVariant: ""
-measurementSalt: ""
 joinServicePort: 9090
 joinServiceNodePort: 30090

cli/internal/helm/loader.go

@@ -134,10 +134,7 @@ func (i *ChartLoader) LoadReleases(
 		return nil, fmt.Errorf("loading constellation-services: %w", err)
 	}
 
-	if idFile.MeasurementSalt == nil {
+	svcVals, err := extraConstellationServicesValues(config, masterSecret, idFile.UID, serviceAccURI, output)
-		return nil, errors.New("missing measurement salt in idFile")
-	}
-	svcVals, err := extraConstellationServicesValues(config, masterSecret, idFile.MeasurementSalt, idFile.UID, serviceAccURI, output)
 	if err != nil {
 		return nil, fmt.Errorf("extending constellation-services values: %w", err)
 	}

cli/internal/helm/loader_test.go

@@ -7,7 +7,6 @@ SPDX-License-Identifier: AGPL-3.0-only
 package helm
 
 import (
-	"bytes"
 	"fmt"
 	"io/fs"
 	"os"
@@ -25,7 +24,6 @@ import (
 
 	"github.com/edgelesssys/constellation/v2/cli/internal/clusterid"
 	"github.com/edgelesssys/constellation/v2/cli/internal/terraform"
-	"github.com/edgelesssys/constellation/v2/internal/attestation/idkeydigest"
 	"github.com/edgelesssys/constellation/v2/internal/attestation/measurements"
 	"github.com/edgelesssys/constellation/v2/internal/cloud/azureshared"
 	"github.com/edgelesssys/constellation/v2/internal/cloud/cloudprovider"
@@ -71,7 +69,7 @@ func TestLoadReleases(t *testing.T) {
 	helmReleases, err := chartLoader.LoadReleases(
 		config, true, WaitModeAtomic,
 		uri.MasterSecret{Key: []byte("secret"), Salt: []byte("masterSalt")},
-		fakeServiceAccURI(cloudprovider.GCP), clusterid.File{UID: "testuid", MeasurementSalt: []byte("measurementSalt")}, terraform.ApplyOutput{GCP: &terraform.GCPApplyOutput{}},
+		fakeServiceAccURI(cloudprovider.GCP), clusterid.File{UID: "testuid"}, terraform.ApplyOutput{GCP: &terraform.GCPApplyOutput{}},
 	)
 	require.NoError(err)
 	chart := helmReleases.ConstellationServices.Chart
@@ -113,18 +111,7 @@ func TestConstellationServices(t *testing.T) {
 				Provider: config.ProviderConfig{Azure: &config.AzureConfig{
 					DeployCSIDriver: toPtr(true),
 				}},
-				Attestation: config.AttestationConfig{AzureSEVSNP: &config.AzureSEVSNP{
+				Attestation: config.AttestationConfig{AzureSEVSNP: &config.AzureSEVSNP{}},
-					Measurements: measurements.M{1: measurements.WithAllBytes(0xAA, measurements.Enforce, measurements.PCRMeasurementLength)},
-					FirmwareSignerConfig: config.SNPFirmwareSignerConfig{
-						AcceptedKeyDigests: idkeydigest.List{bytes.Repeat([]byte{0xAA}, 32)},
-						EnforcementPolicy:  idkeydigest.MAAFallback,
-						MAAURL:             "https://192.0.2.1:8080/maa",
-					},
-					BootloaderVersion: config.AttestationVersion{Value: 1, WantLatest: true},
-					TEEVersion:        config.AttestationVersion{Value: 2, WantLatest: true},
-					SNPVersion:        config.AttestationVersion{Value: 3, WantLatest: true},
-					MicrocodeVersion:  config.AttestationVersion{Value: 4, WantLatest: true},
-				}},
 			},
 			enforceIDKeyDigest: true,
 			ccmImage:           "ccmImageForAzure",
@@ -135,27 +122,21 @@ func TestConstellationServices(t *testing.T) {
 				Provider: config.ProviderConfig{GCP: &config.GCPConfig{
 					DeployCSIDriver: toPtr(true),
 				}},
-				Attestation: config.AttestationConfig{GCPSEVES: &config.GCPSEVES{
+				Attestation: config.AttestationConfig{GCPSEVES: &config.GCPSEVES{}},
-					Measurements: measurements.M{1: measurements.WithAllBytes(0xAA, measurements.Enforce, measurements.PCRMeasurementLength)},
-				}},
 			},
 			ccmImage: "ccmImageForGCP",
 		},
 		"OpenStack": {
 			config: &config.Config{
 				Provider:    config.ProviderConfig{OpenStack: &config.OpenStackConfig{}},
-				Attestation: config.AttestationConfig{QEMUVTPM: &config.QEMUVTPM{
+				Attestation: config.AttestationConfig{QEMUVTPM: &config.QEMUVTPM{}},
-					Measurements: measurements.M{1: measurements.WithAllBytes(0xAA, measurements.Enforce, measurements.PCRMeasurementLength)},
-				}},
 			},
 			ccmImage: "ccmImageForOpenStack",
 		},
 		"QEMU": {
 			config: &config.Config{
 				Provider:    config.ProviderConfig{QEMU: &config.QEMUConfig{}},
-				Attestation: config.AttestationConfig{QEMUVTPM: &config.QEMUVTPM{
+				Attestation: config.AttestationConfig{QEMUVTPM: &config.QEMUVTPM{}},
-					Measurements: measurements.M{1: measurements.WithAllBytes(0xAA, measurements.Enforce, measurements.PCRMeasurementLength)},
-				}},
 			},
 		},
 	}
@@ -186,7 +167,6 @@ func TestConstellationServices(t *testing.T) {
 					Key:  []byte("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"),
 					Salt: []byte("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"),
 				},
-				[]byte("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"),
 				"uid", serviceAccURI, terraform.ApplyOutput{
 					Azure: &terraform.AzureApplyOutput{},
 					GCP:   &terraform.GCPApplyOutput{},
@@ -381,13 +361,6 @@ func buildTestdataMap(csp string, expectedData map[string]string, require *requi
 
 // addInClusterValues adds values that are only known after the cluster is created.
 func addInClusterValues(values map[string]any, csp cloudprovider.Provider) error {
-	joinVals, ok := values["join-service"].(map[string]any)
-	if !ok {
-		return errors.New("missing 'join-service' key")
-	}
-
-	joinVals["measurementSalt"] = "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"
-
 	verificationVals, ok := values["verification-service"].(map[string]any)
 	if !ok {
 		return fmt.Errorf("missing 'verification-service' key %v", values)

cli/internal/helm/overrides.go

@@ -54,17 +54,11 @@ func extraCiliumValues(provider cloudprovider.Provider, conformanceMode bool, ou
 // extraConstellationServicesValues extends the given values map by some values depending on user input.
 // Values set inside this function are only applied during init, not during upgrade.
 func extraConstellationServicesValues(
-	cfg *config.Config, masterSecret uri.MasterSecret, measurementSalt []byte, uid, serviceAccURI string, output terraform.ApplyOutput,
+	cfg *config.Config, masterSecret uri.MasterSecret, uid, serviceAccURI string, output terraform.ApplyOutput,
 ) (map[string]any, error) {
-	attestationConfigJSON, err := json.Marshal(cfg.GetAttestationConfig())
-	if err != nil {
-		return nil, fmt.Errorf("marshalling measurements: %w", err)
-	}
 	extraVals := map[string]any{}
 	extraVals["join-service"] = map[string]any{
-		"measurementSalt":    base64.StdEncoding.EncodeToString(measurementSalt),
 		"attestationVariant": cfg.GetAttestationConfig().GetVariant().String(),
-		"attestationConfig":  string(attestationConfigJSON),
 	}
 	extraVals["verification-service"] = map[string]any{
 		"attestationVariant": cfg.GetAttestationConfig().GetVariant().String(),

cli/internal/helm/testdata/AWS/constellation-services/charts/join-service/templates/configmap.yaml

@@ -1,9 +0,0 @@
-apiVersion: v1
-kind: ConfigMap
-metadata:
-  name: join-config
-  namespace: testNamespace
-data:
-  attestationConfig: "{\"measurements\":{\"1\":{\"expected\":\"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\",\"warnOnly\":false}}}"
-binaryData:
-  measurementSalt: AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA

cli/internal/helm/testdata/Azure/constellation-services/charts/join-service/templates/configmap.yaml

@@ -1,9 +0,0 @@
-apiVersion: v1
-kind: ConfigMap
-metadata:
-  name: join-config
-  namespace: testNamespace
-data:
-  attestationConfig: "{\"measurements\":{\"1\":{\"expected\":\"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\",\"warnOnly\":false}},\"bootloaderVersion\":1,\"teeVersion\":2,\"snpVersion\":3,\"microcodeVersion\":4,\"firmwareSignerConfig\":{\"acceptedKeyDigests\":[\"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\"],\"enforcementPolicy\":\"MAAFallback\",\"maaURL\":\"https://192.0.2.1:8080/maa\"},\"amdRootKey\":\"-----BEGIN CERTIFICATE-----\\n-----END CERTIFICATE-----\\n\"}"
-binaryData:
-  measurementSalt: AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA

cli/internal/helm/testdata/GCP/constellation-services/charts/join-service/templates/configmap.yaml

@@ -1,9 +0,0 @@
-apiVersion: v1
-kind: ConfigMap
-metadata:
-  name: join-config
-  namespace: testNamespace
-data:
-  attestationConfig: "{\"measurements\":{\"1\":{\"expected\":\"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\",\"warnOnly\":false}}}"
-binaryData:
-  measurementSalt: AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA

cli/internal/helm/testdata/OpenStack/constellation-services/charts/join-service/templates/configmap.yaml

@@ -1,9 +0,0 @@
-apiVersion: v1
-kind: ConfigMap
-metadata:
-  name: join-config
-  namespace: testNamespace
-data:
-  attestationConfig: "{\"measurements\":{\"1\":{\"expected\":\"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\",\"warnOnly\":false}}}"
-binaryData:
-  measurementSalt: AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA

cli/internal/helm/testdata/QEMU/constellation-services/charts/join-service/templates/configmap.yaml

@@ -1,9 +0,0 @@
-apiVersion: v1
-kind: ConfigMap
-metadata:
-  name: join-config
-  namespace: testNamespace
-data:
-  attestationConfig: "{\"measurements\":{\"1\":{\"expected\":\"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\",\"warnOnly\":false}}}"
-binaryData:
-  measurementSalt: AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA

cli/internal/kubecmd/BUILD.bazel

@@ -31,6 +31,7 @@ go_library(
         "@io_k8s_client_go//util/retry",
         "@io_k8s_kubernetes//cmd/kubeadm/app/apis/kubeadm/v1beta3",
         "@io_k8s_sigs_yaml//:yaml",
+        "@sh_helm_helm_v3//pkg/kube",
     ],
 )
 
@@ -39,6 +40,7 @@ go_test(
     srcs = ["kubecmd_test.go"],
     embed = [":kubecmd"],
     deps = [
+        "//internal/attestation/measurements",
         "//internal/attestation/variant",
         "//internal/cloud/cloudprovider",
         "//internal/compatibility",

cli/internal/kubecmd/kubecmd.go

@@ -37,6 +37,7 @@ import (
 	"github.com/edgelesssys/constellation/v2/internal/versions"
 	"github.com/edgelesssys/constellation/v2/internal/versions/components"
 	updatev1alpha1 "github.com/edgelesssys/constellation/v2/operators/constellation-node-operator/v2/api/v1alpha1"
+	"helm.sh/helm/v3/pkg/kube"
 	corev1 "k8s.io/api/core/v1"
 	k8serrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -193,35 +194,37 @@ func (k *KubeCmd) GetClusterAttestationConfig(ctx context.Context, variant varia
 	return existingAttestationConfig, nil
 }
 
-// UpdateAttestationConfig updates the Constellation cluster's attestation config.
+// ApplyJoinConfig creates or updates the Constellation cluster's join-config ConfigMap.
-// A backup of the previous config is created before updating.
+// This ConfigMap holds the attestation config and measurement salt of the cluster.
-func (k *KubeCmd) UpdateAttestationConfig(ctx context.Context, newAttestConfig config.AttestationCfg) error {
+// A backup of the previous attestation config is created with the suffix `_backup` in the config map data.
-	// backup of previous measurements
+func (k *KubeCmd) ApplyJoinConfig(ctx context.Context, newAttestConfig config.AttestationCfg, measurementSalt []byte) error {
-	joinConfig, err := k.kubectl.GetConfigMap(ctx, constants.ConstellationNamespace, constants.JoinConfigMap)
-	if err != nil {
-		return fmt.Errorf("getting %s ConfigMap: %w", constants.JoinConfigMap, err)
-	}
-
-	// create backup of previous config
-	backup := joinConfig.DeepCopy()
-	backup.ObjectMeta = metav1.ObjectMeta{}
-	backup.Name = fmt.Sprintf("%s-backup", constants.JoinConfigMap)
-	if err := k.applyConfigMap(ctx, backup); err != nil {
-		return fmt.Errorf("creating backup of join-config ConfigMap: %w", err)
-	}
-	k.log.Debugf("Created backup of %s ConfigMap %q in namespace %q", constants.JoinConfigMap, backup.Name, backup.Namespace)
-
 	newConfigJSON, err := json.Marshal(newAttestConfig)
 	if err != nil {
 		return fmt.Errorf("marshaling attestation config: %w", err)
 	}
+
+	joinConfig, err := k.kubectl.GetConfigMap(ctx, constants.ConstellationNamespace, constants.JoinConfigMap)
+	if err != nil {
+		if !k8serrors.IsNotFound(err) {
+			return fmt.Errorf("getting %s ConfigMap: %w", constants.JoinConfigMap, err)
+		}
+
+		k.log.Debugf("ConfigMap %q does not exist in namespace %q, creating it now", constants.JoinConfigMap, constants.ConstellationNamespace)
+		if err := k.kubectl.CreateConfigMap(ctx, joinConfigMap(newConfigJSON, measurementSalt)); err != nil {
+			return fmt.Errorf("creating join-config ConfigMap: %w", err)
+		}
+		k.log.Debugf("Created %q ConfigMap in namespace %q", constants.JoinConfigMap, constants.ConstellationNamespace)
+		return nil
+	}
+
+	// create backup of previous config
+	joinConfig.Data[constants.AttestationConfigFilename+"_backup"] = joinConfig.Data[constants.AttestationConfigFilename]
 	joinConfig.Data[constants.AttestationConfigFilename] = string(newConfigJSON)
 	k.log.Debugf("Triggering attestation config update now")
 	if _, err = k.kubectl.UpdateConfigMap(ctx, joinConfig); err != nil {
 		return fmt.Errorf("setting new attestation config: %w", err)
 	}
 
-	fmt.Fprintln(k.outWriter, "Successfully updated the cluster's attestation config")
 	return nil
 }
 
@@ -398,19 +401,6 @@ func (k *KubeCmd) updateK8s(nodeVersion *updatev1alpha1.NodeVersion, newClusterV
 	return &configMap, nil
 }
 
-// applyConfigMap applies the ConfigMap by creating it if it doesn't exist, or updating it if it does.
-func (k *KubeCmd) applyConfigMap(ctx context.Context, configMap *corev1.ConfigMap) error {
-	if err := k.kubectl.CreateConfigMap(ctx, configMap); err != nil {
-		if !k8serrors.IsAlreadyExists(err) {
-			return fmt.Errorf("creating backup config map: %w", err)
-		}
-		if _, err := k.kubectl.UpdateConfigMap(ctx, configMap); err != nil {
-			return fmt.Errorf("updating backup config map: %w", err)
-		}
-	}
-	return nil
-}
-
 func checkForApplyError(expected, actual updatev1alpha1.NodeVersion) error {
 	var err error
 	switch {
@@ -426,6 +416,87 @@ func checkForApplyError(expected, actual updatev1alpha1.NodeVersion) error {
 	return err
 }
 
+func joinConfigMap(attestationCfgJSON, measurementSalt []byte) *corev1.ConfigMap {
+	return &corev1.ConfigMap{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      constants.JoinConfigMap,
+			Namespace: constants.ConstellationNamespace,
+		},
+		Data: map[string]string{
+			constants.AttestationConfigFilename: string(attestationCfgJSON),
+		},
+		BinaryData: map[string][]byte{
+			constants.MeasurementSaltFilename: measurementSalt,
+		},
+	}
+}
+
+// RemoveAttestationConfigHelmManagement removes labels and annotations from the join-config ConfigMap that are added by Helm.
+// This is to ensure we can cleanly transition from Helm to Constellation's management of the ConfigMap.
+// TODO(v2.11): Remove this function after v2.11 is released.
+func (k *KubeCmd) RemoveAttestationConfigHelmManagement(ctx context.Context) error {
+	const (
+		appManagedByLabel              = "app.kubernetes.io/managed-by"
+		appManagedByHelm               = "Helm"
+		helmReleaseNameAnnotation      = "meta.helm.sh/release-name"
+		helmReleaseNamespaceAnnotation = "meta.helm.sh/release-namespace"
+	)
+
+	k.log.Debugf("Checking if join-config ConfigMap needs to be migrated to remove Helm management")
+	joinConfig, err := k.kubectl.GetConfigMap(ctx, constants.ConstellationNamespace, constants.JoinConfigMap)
+	if err != nil {
+		return fmt.Errorf("getting join config: %w", err)
+	}
+
+	var needUpdate bool
+	if managedBy, ok := joinConfig.Labels[appManagedByLabel]; ok && managedBy == appManagedByHelm {
+		delete(joinConfig.Labels, appManagedByLabel)
+		needUpdate = true
+	}
+	if _, ok := joinConfig.Annotations[helmReleaseNameAnnotation]; ok {
+		delete(joinConfig.Annotations, helmReleaseNameAnnotation)
+		needUpdate = true
+	}
+	if _, ok := joinConfig.Annotations[helmReleaseNamespaceAnnotation]; ok {
+		delete(joinConfig.Annotations, helmReleaseNamespaceAnnotation)
+		needUpdate = true
+	}
+
+	if needUpdate {
+		// Tell Helm to ignore this resource in the future.
+		// TODO(v2.11): Remove this annotation from the ConfigMap.
+		joinConfig.Annotations[kube.ResourcePolicyAnno] = kube.KeepPolicy
+
+		k.log.Debugf("Removing Helm management labels from join-config ConfigMap")
+		if _, err := k.kubectl.UpdateConfigMap(ctx, joinConfig); err != nil {
+			return fmt.Errorf("removing Helm management labels from join-config: %w", err)
+		}
+		k.log.Debugf("Successfully removed Helm management labels from join-config ConfigMap")
+	}
+
+	return nil
+}
+
+// RemoveHelmKeepAnnotation removes the Helm Resource Policy annotation from the join-config ConfigMap.
+// TODO(v2.12): Remove this function after v2.12 is released.
+func (k *KubeCmd) RemoveHelmKeepAnnotation(ctx context.Context) error {
+	k.log.Debugf("Checking if Helm Resource Policy can be removed from join-config ConfigMap")
+	joinConfig, err := k.kubectl.GetConfigMap(ctx, constants.ConstellationNamespace, constants.JoinConfigMap)
+	if err != nil {
+		return fmt.Errorf("getting join config: %w", err)
+	}
+
+	if policy, ok := joinConfig.Annotations[kube.ResourcePolicyAnno]; ok && policy == kube.KeepPolicy {
+		delete(joinConfig.Annotations, kube.ResourcePolicyAnno)
+		if _, err := k.kubectl.UpdateConfigMap(ctx, joinConfig); err != nil {
+			return fmt.Errorf("removing Helm Resource Policy from join-config: %w", err)
+		}
+		k.log.Debugf("Successfully removed Helm Resource Policy from join-config ConfigMap")
+	}
+
+	return nil
+}
+
 // kubectlInterface is provides access to the Kubernetes API.
 type kubectlInterface interface {
 	GetNodes(ctx context.Context) ([]corev1.Node, error)

cli/internal/kubecmd/kubecmd_test.go

@@ -13,8 +13,7 @@ import (
 	"io"
 	"testing"
 
-	kerrors "k8s.io/apimachinery/pkg/api/errors"
+	"github.com/edgelesssys/constellation/v2/internal/attestation/measurements"
-
 	"github.com/edgelesssys/constellation/v2/internal/attestation/variant"
 	"github.com/edgelesssys/constellation/v2/internal/cloud/cloudprovider"
 	"github.com/edgelesssys/constellation/v2/internal/compatibility"
@@ -28,6 +27,7 @@ import (
 	"github.com/stretchr/testify/mock"
 	"github.com/stretchr/testify/require"
 	corev1 "k8s.io/api/core/v1"
+	k8serrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"k8s.io/apimachinery/pkg/runtime"
@@ -281,7 +281,7 @@ func TestUpgradeNodeVersion(t *testing.T) {
 			customClientFn: func(nodeVersion updatev1alpha1.NodeVersion) unstructuredInterface {
 				fakeClient := &fakeUnstructuredClient{}
 				fakeClient.On("GetCR", mock.Anything, mock.Anything).Return(unstructedObjectWithGeneration(nodeVersion, 1), nil)
-				fakeClient.On("UpdateCR", mock.Anything, mock.Anything).Return(nil, kerrors.NewConflict(schema.GroupResource{Resource: nodeVersion.Name}, nodeVersion.Name, nil)).Once()
+				fakeClient.On("UpdateCR", mock.Anything, mock.Anything).Return(nil, k8serrors.NewConflict(schema.GroupResource{Resource: nodeVersion.Name}, nodeVersion.Name, nil)).Once()
 				fakeClient.On("UpdateCR", mock.Anything, mock.Anything).Return(unstructedObjectWithGeneration(nodeVersion, 2), nil).Once()
 				return fakeClient
 			},
@@ -470,7 +470,7 @@ func newJoinConfigMap(data string) *corev1.ConfigMap {
 	}
 }
 
-func TestUpdateAttestationConfig(t *testing.T) {
+func TestApplyAttestationConfig(t *testing.T) {
 	mustMarshal := func(cfg config.AttestationCfg) string {
 		data, err := json.Marshal(cfg)
 		require.NoError(t, err)
@@ -483,25 +483,66 @@ func TestUpdateAttestationConfig(t *testing.T) {
 		wantErr           bool
 	}{
 		"success": {
-			newAttestationCfg: config.DefaultForAzureSEVSNP(),
+			newAttestationCfg: &config.QEMUVTPM{
+				Measurements: measurements.M{
+					0: measurements.WithAllBytes(0x00, measurements.WarnOnly, measurements.PCRMeasurementLength),
+				},
+			},
 			kubectl: &stubKubectl{
 				configMaps: map[string]*corev1.ConfigMap{
-					constants.JoinConfigMap: newJoinConfigMap(mustMarshal(config.DefaultForAzureSEVSNP())),
+					constants.JoinConfigMap: newJoinConfigMap(mustMarshal(&config.QEMUVTPM{
+						Measurements: measurements.M{
+							0: measurements.WithAllBytes(0xFF, measurements.WarnOnly, measurements.PCRMeasurementLength),
+						},
+					})),
 				},
 			},
 		},
-		"error getting ConfigMap": {
+		"Get ConfigMap error": {
-			newAttestationCfg: config.DefaultForAzureSEVSNP(),
+			newAttestationCfg: &config.QEMUVTPM{
+				Measurements: measurements.M{
+					0: measurements.WithAllBytes(0x00, measurements.WarnOnly, measurements.PCRMeasurementLength),
+				},
+			},
 			kubectl: &stubKubectl{
 				getCMErr: assert.AnError,
 			},
 			wantErr: true,
 		},
+		"ConfigMap does not exist yet": {
+			newAttestationCfg: &config.QEMUVTPM{
+				Measurements: measurements.M{
+					0: measurements.WithAllBytes(0x00, measurements.WarnOnly, measurements.PCRMeasurementLength),
+				},
+			},
+			kubectl: &stubKubectl{
+				getCMErr: k8serrors.NewNotFound(schema.GroupResource{}, ""),
+			},
+		},
+		"Update ConfigMap error": {
+			newAttestationCfg: &config.QEMUVTPM{
+				Measurements: measurements.M{
+					0: measurements.WithAllBytes(0x00, measurements.WarnOnly, measurements.PCRMeasurementLength),
+				},
+			},
+			kubectl: &stubKubectl{
+				configMaps: map[string]*corev1.ConfigMap{
+					constants.JoinConfigMap: newJoinConfigMap(mustMarshal(&config.QEMUVTPM{
+						Measurements: measurements.M{
+							0: measurements.WithAllBytes(0xFF, measurements.WarnOnly, measurements.PCRMeasurementLength),
+						},
+					})),
+				},
+				updateCMErr: assert.AnError,
+			},
+			wantErr: true,
+		},
 	}
 
 	for name, tc := range testCases {
 		t.Run(name, func(t *testing.T) {
 			assert := assert.New(t)
+			require := require.New(t)
 
 			cmd := &KubeCmd{
 				kubectl:   tc.kubectl,
@@ -509,12 +550,15 @@ func TestUpdateAttestationConfig(t *testing.T) {
 				outWriter: io.Discard,
 			}
 
-			err := cmd.UpdateAttestationConfig(context.Background(), tc.newAttestationCfg)
+			err := cmd.ApplyJoinConfig(context.Background(), tc.newAttestationCfg, []byte{0x11})
 			if tc.wantErr {
 				assert.Error(err)
 				return
 			}
 			assert.NoError(err)
+			cfg, ok := tc.kubectl.configMaps[constants.JoinConfigMap]
+			require.True(ok)
+			assert.Equal(mustMarshal(tc.newAttestationCfg), cfg.Data[constants.AttestationConfigFilename])
 		})
 	}
 }

docs/docs/workflows/troubleshooting.md

@@ -40,7 +40,7 @@ Or alternatively, for `terminate`:
 ARM_SKIP_PROVIDER_REGISTRATION=true constellation terminate
 ```
 
-### Nodes fail to join with error `untrusted PCR value`
+### Nodes fail to join with error `untrusted measurement value`
 
 This error indicates that a node's [attestation statement](../architecture/attestation.md) contains measurements that don't match the trusted values expected by the [JoinService](../architecture/microservices.md#joinservice).
 This may for example happen if the cloud provider updates the VM's firmware such that it influences the [runtime measurements](../architecture/attestation.md#runtime-measurements) in an unforeseen way.
@@ -55,14 +55,16 @@ When in doubt, check if the encountered [issue is known](https://github.com/edge
 
 :::
 
-
 :::tip
 
-During an upgrade with modified attestation config, a backup of the current configuration is stored in the `join-config-backup` config map in the `kube-system` namespace. To restore the old attestation config after a failed upgrade, you can copy the attestation config from this resource, put it in your configuration file and retry the upgrade.
+During an upgrade with modified attestation config, a backup of the current configuration is stored in the `join-config` config map in the `kube-system` namespace under the `attestationConfig_backup` key. To restore the old attestation config after a failed upgrade, replace the value of `attestationConfig` with the value from `attestationConfig_backup`:
+
+```bash
+kubectl patch configmaps -n kube-system join-config -p "{\"data\":{\"attestationConfig\":\"$(kubectl get configmaps -n kube-system join-config -o "jsonpath={.data.attestationConfig_backup}")\"}}"
+```
 
 :::
 
-
 You can use the `upgrade apply` command to change measurements of a running cluster:
 
 1. Modify the `measurements` key in your local `constellation-conf.yaml` to the expected values.