cli: remove helm management from join-config (#2251)

* Replace UpdateAttestationConfig with ApplyJoinConfig

* Dont set up join-config over Helm, it is now only managed by our CLI directly during init and upgrade

* Remove measurementSalt and attestationConfig parsing from helm, they were only needed for the JoinConfig

* Add migration step to remove join-config from Helm management

* Update attestation config trouble shooting tip

---------

Signed-off-by: Daniel Weiße <dw@edgeless.systems>
This commit is contained in:
Daniel Weiße 2023-08-23 08:14:39 +02:00 committed by GitHub
parent c42e81bf23
commit 053aa60e47
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
21 changed files with 326 additions and 196 deletions

View File

@ -39,6 +39,7 @@ import (
"github.com/edgelesssys/constellation/v2/cli/internal/clusterid"
"github.com/edgelesssys/constellation/v2/cli/internal/cmd/pathprefix"
"github.com/edgelesssys/constellation/v2/cli/internal/helm"
"github.com/edgelesssys/constellation/v2/cli/internal/kubecmd"
"github.com/edgelesssys/constellation/v2/cli/internal/terraform"
"github.com/edgelesssys/constellation/v2/internal/cloud/cloudprovider"
"github.com/edgelesssys/constellation/v2/internal/config"
@ -129,13 +130,20 @@ func runInitialize(cmd *cobra.Command, _ []string) error {
return fmt.Errorf("creating Helm installer: %w", err)
}
i := newInitCmd(tfClient, helmInstaller, fileHandler, spinner, &kubeconfigMerger{log: log}, log)
fetcher := attestationconfigapi.NewFetcher()
return i.initialize(cmd, newDialer, license.NewClient(), fetcher)
newAttestationApplier := func(w io.Writer, kubeConfig string, log debugLog) (attestationConfigApplier, error) {
return kubecmd.New(w, kubeConfig, log)
}
return i.initialize(cmd, newDialer, license.NewClient(), fetcher, newAttestationApplier)
}
// initialize initializes a Constellation.
func (i *initCmd) initialize(cmd *cobra.Command, newDialer func(validator atls.Validator) *dialer.Dialer,
func (i *initCmd) initialize(
cmd *cobra.Command, newDialer func(validator atls.Validator) *dialer.Dialer,
quotaChecker license.QuotaChecker, configFetcher attestationconfigapi.Fetcher,
newAttestationApplier func(io.Writer, string, debugLog) (attestationConfigApplier, error),
) error {
flags, err := i.evalFlagArgs(cmd)
if err != nil {
@ -249,6 +257,14 @@ func (i *initCmd) initialize(cmd *cobra.Command, newDialer func(validator atls.V
return err
}
attestationApplier, err := newAttestationApplier(cmd.OutOrStdout(), constants.AdminConfFilename, i.log)
if err != nil {
return err
}
if err := attestationApplier.ApplyJoinConfig(cmd.Context(), conf.GetAttestationConfig(), measurementSalt); err != nil {
return fmt.Errorf("applying attestation config: %w", err)
}
helmLoader := helm.NewLoader(provider, k8sVersion, clusterName)
i.log.Debugf("Created new Helm loader")
output, err := i.clusterShower.ShowCluster(cmd.Context(), conf.GetProvider())
@ -609,3 +625,7 @@ func (e *nonRetriableError) Unwrap() error {
type initializer interface {
Install(ctx context.Context, releases *helm.Releases) error
}
type attestationConfigApplier interface {
ApplyJoinConfig(ctx context.Context, newAttestConfig config.AttestationCfg, measurementSalt []byte) error
}

View File

@ -187,7 +187,15 @@ func TestInitialize(t *testing.T) {
defer cancel()
cmd.SetContext(ctx)
i := newInitCmd(&stubShowCluster{}, &stubHelmInstaller{}, fileHandler, &nopSpinner{}, nil, logger.NewTest(t))
err := i.initialize(cmd, newDialer, &stubLicenseClient{}, stubAttestationFetcher{})
err := i.initialize(
cmd,
newDialer,
&stubLicenseClient{},
stubAttestationFetcher{},
func(io.Writer, string, debugLog) (attestationConfigApplier, error) {
return &stubAttestationApplier{}, nil
},
)
if tc.wantErr {
assert.Error(err)
@ -486,7 +494,15 @@ func TestAttestation(t *testing.T) {
cmd.SetContext(ctx)
i := newInitCmd(nil, nil, fileHandler, &nopSpinner{}, nil, logger.NewTest(t))
err := i.initialize(cmd, newDialer, &stubLicenseClient{}, stubAttestationFetcher{})
err := i.initialize(
cmd,
newDialer,
&stubLicenseClient{},
stubAttestationFetcher{},
func(io.Writer, string, debugLog) (attestationConfigApplier, error) {
return &stubAttestationApplier{}, nil
},
)
assert.Error(err)
// make sure the error is actually a TLS handshake error
assert.Contains(err.Error(), "transport: authentication handshake failed")
@ -666,3 +682,11 @@ func (s *stubShowCluster) ShowCluster(_ context.Context, csp cloudprovider.Provi
}
return res, nil
}
type stubAttestationApplier struct {
applyErr error
}
func (a *stubAttestationApplier) ApplyJoinConfig(_ context.Context, _ config.AttestationCfg, _ []byte) error {
return a.applyErr
}

View File

@ -10,12 +10,14 @@ import (
"context"
"errors"
"fmt"
"io"
"net"
"github.com/edgelesssys/constellation/v2/cli/internal/cloudcmd"
"github.com/edgelesssys/constellation/v2/cli/internal/cmd/pathprefix"
"github.com/edgelesssys/constellation/v2/cli/internal/featureset"
"github.com/edgelesssys/constellation/v2/cli/internal/helm"
"github.com/edgelesssys/constellation/v2/cli/internal/kubecmd"
"github.com/edgelesssys/constellation/v2/cli/internal/libvirt"
"github.com/edgelesssys/constellation/v2/cli/internal/terraform"
"github.com/edgelesssys/constellation/v2/internal/api/attestationconfigapi"
@ -215,8 +217,12 @@ func (m *miniUpCmd) initializeMiniCluster(cmd *cobra.Command, fileHandler file.H
return fmt.Errorf("creating Terraform client: %w", err)
}
newAttestationApplier := func(w io.Writer, kubeConfig string, log debugLog) (attestationConfigApplier, error) {
return kubecmd.New(w, kubeConfig, log)
}
i := newInitCmd(tfClient, helmInstaller, fileHandler, spinner, &kubeconfigMerger{log: log}, log)
if err := i.initialize(cmd, newDialer, license.NewClient(), m.configFetcher); err != nil {
if err := i.initialize(cmd, newDialer, license.NewClient(), m.configFetcher, newAttestationApplier); err != nil {
return err
}
m.log.Debugf("Initialized mini cluster")

View File

@ -29,6 +29,7 @@ import (
"github.com/edgelesssys/constellation/v2/internal/file"
"github.com/edgelesssys/constellation/v2/internal/kms/uri"
"github.com/edgelesssys/constellation/v2/internal/kubernetes/kubectl"
"github.com/edgelesssys/constellation/v2/internal/semver"
"github.com/edgelesssys/constellation/v2/internal/versions"
"github.com/rogpeppe/go-internal/diff"
"github.com/spf13/afero"
@ -153,7 +154,15 @@ func (u *upgradeApplyCmd) upgradeApply(cmd *cobra.Command) error {
}
conf.UpdateMAAURL(idFile.AttestationURL)
if err := u.confirmAttestationConfigUpgrade(cmd, conf.GetAttestationConfig(), flags); err != nil {
// Apply migrations necessary for the upgrade
if err := migrateFrom2_10(cmd.Context(), u.kubeUpgrader); err != nil {
return fmt.Errorf("applying migration for upgrading from v2.10: %w", err)
}
if err := migrateFrom2_11(cmd.Context(), u.kubeUpgrader); err != nil {
return fmt.Errorf("applying migration for upgrading from v2.11: %w", err)
}
if err := u.confirmAndUpgradeAttestationConfig(cmd, conf.GetAttestationConfig(), idFile.MeasurementSalt, flags); err != nil {
return fmt.Errorf("upgrading measurements: %w", err)
}
@ -177,7 +186,11 @@ func (u *upgradeApplyCmd) upgradeApply(cmd *cobra.Command) error {
return fmt.Errorf("extending cert SANs: %w", err)
}
if conf.GetProvider() == cloudprovider.Azure || conf.GetProvider() == cloudprovider.GCP || conf.GetProvider() == cloudprovider.AWS {
if conf.GetProvider() != cloudprovider.Azure && conf.GetProvider() != cloudprovider.GCP && conf.GetProvider() != cloudprovider.AWS {
cmd.PrintErrln("WARNING: Skipping service and image upgrades, which are currently only supported for AWS, Azure, and GCP.")
return nil
}
var upgradeErr *compatibility.InvalidUpgradeError
err = u.handleServiceUpgrade(cmd, conf, idFile, tfOutput, validK8sVersion, flags)
switch {
@ -198,9 +211,6 @@ func (u *upgradeApplyCmd) upgradeApply(cmd *cobra.Command) error {
case err != nil:
return fmt.Errorf("upgrading NodeVersion: %w", err)
}
} else {
cmd.PrintErrln("WARNING: Skipping service and image upgrades, which are currently only supported for AWS, Azure, and GCP.")
}
return nil
}
@ -338,9 +348,11 @@ func validK8sVersion(cmd *cobra.Command, version string, yes bool) (validVersion
return validVersion, nil
}
// confirmAttestationConfigUpgrade checks if the locally configured measurements are different from the cluster's measurements.
// confirmAndUpgradeAttestationConfig checks if the locally configured measurements are different from the cluster's measurements.
// If so the function will ask the user to confirm (if --yes is not set) and upgrade the cluster's config.
func (u *upgradeApplyCmd) confirmAttestationConfigUpgrade(cmd *cobra.Command, newConfig config.AttestationCfg, flags upgradeApplyFlags) error {
func (u *upgradeApplyCmd) confirmAndUpgradeAttestationConfig(
cmd *cobra.Command, newConfig config.AttestationCfg, measurementSalt []byte, flags upgradeApplyFlags,
) error {
clusterAttestationConfig, err := u.kubeUpgrader.GetClusterAttestationConfig(cmd.Context(), newConfig.GetVariant())
if err != nil {
return fmt.Errorf("getting cluster attestation config: %w", err)
@ -371,9 +383,10 @@ func (u *upgradeApplyCmd) confirmAttestationConfigUpgrade(cmd *cobra.Command, ne
}
}
if err := u.kubeUpgrader.UpdateAttestationConfig(cmd.Context(), newConfig); err != nil {
if err := u.kubeUpgrader.ApplyJoinConfig(cmd.Context(), newConfig, measurementSalt); err != nil {
return fmt.Errorf("updating attestation config: %w", err)
}
cmd.Println("Successfully update the cluster's attestation config")
return nil
}
@ -413,6 +426,34 @@ func (u *upgradeApplyCmd) handleServiceUpgrade(cmd *cobra.Command, conf *config.
return err
}
// migrateFrom2_10 applies migrations necessary for upgrading from v2.10 to v2.11
// TODO(v2.11): Remove this function after v2.11 is released.
func migrateFrom2_10(ctx context.Context, kubeUpgrader kubernetesUpgrader) error {
// Sanity check to make sure we only run migrations on upgrades with CLI version 2.10 < v < 2.12
if !constants.BinaryVersion().MajorMinorEqual(semver.NewFromInt(2, 11, 0, "")) {
return nil
}
if err := kubeUpgrader.RemoveAttestationConfigHelmManagement(ctx); err != nil {
return fmt.Errorf("removing helm management from attestation config: %w", err)
}
return nil
}
// migrateFrom2_11 applies migrations necessary for upgrading from v2.11 to v2.12
// TODO(v2.12): Remove this function after v2.12 is released.
func migrateFrom2_11(ctx context.Context, kubeUpgrader kubernetesUpgrader) error {
// Sanity check to make sure we only run migrations on upgrades with CLI version 2.11 < v < 2.13
if !constants.BinaryVersion().MajorMinorEqual(semver.NewFromInt(2, 12, 0, "")) {
return nil
}
if err := kubeUpgrader.RemoveHelmKeepAnnotation(ctx); err != nil {
return fmt.Errorf("removing helm keep annotation: %w", err)
}
return nil
}
func parseUpgradeApplyFlags(cmd *cobra.Command) (upgradeApplyFlags, error) {
workDir, err := cmd.Flags().GetString("workspace")
if err != nil {
@ -493,7 +534,11 @@ type kubernetesUpgrader interface {
UpgradeNodeVersion(ctx context.Context, conf *config.Config, force bool) error
ExtendClusterConfigCertSANs(ctx context.Context, alternativeNames []string) error
GetClusterAttestationConfig(ctx context.Context, variant variant.Variant) (config.AttestationCfg, error)
UpdateAttestationConfig(ctx context.Context, newAttestConfig config.AttestationCfg) error
ApplyJoinConfig(ctx context.Context, newAttestConfig config.AttestationCfg, measurementSalt []byte) error
// TODO(v2.11): Remove this function after v2.11 is released.
RemoveAttestationConfigHelmManagement(ctx context.Context) error
// TODO(v2.12): Remove this function after v2.12 is released.
RemoveHelmKeepAnnotation(ctx context.Context) error
}
type helmUpgrader interface {

View File

@ -197,7 +197,7 @@ func (u stubKubernetesUpgrader) UpgradeNodeVersion(_ context.Context, _ *config.
return u.nodeVersionErr
}
func (u stubKubernetesUpgrader) UpdateAttestationConfig(_ context.Context, _ config.AttestationCfg) error {
func (u stubKubernetesUpgrader) ApplyJoinConfig(_ context.Context, _ config.AttestationCfg, _ []byte) error {
return nil
}
@ -209,6 +209,16 @@ func (u stubKubernetesUpgrader) ExtendClusterConfigCertSANs(_ context.Context, _
return nil
}
// TODO(v2.11): Remove this function.
func (u stubKubernetesUpgrader) RemoveAttestationConfigHelmManagement(_ context.Context) error {
return nil
}
// TODO(v2.12): Remove this function.
func (u stubKubernetesUpgrader) RemoveHelmKeepAnnotation(_ context.Context) error {
return nil
}
type stubTerraformUpgrader struct {
terraformDiff bool
planTerraformErr error

View File

@ -226,7 +226,6 @@ go_library(
"charts/edgeless/constellation-services/charts/join-service/Chart.yaml",
"charts/edgeless/constellation-services/charts/join-service/templates/clusterrole.yaml",
"charts/edgeless/constellation-services/charts/join-service/templates/clusterrolebinding.yaml",
"charts/edgeless/constellation-services/charts/join-service/templates/configmap.yaml",
"charts/edgeless/constellation-services/charts/join-service/templates/daemonset.yaml",
"charts/edgeless/constellation-services/charts/join-service/templates/service.yaml",
"charts/edgeless/constellation-services/charts/join-service/templates/serviceaccount.yaml",
@ -468,7 +467,6 @@ go_test(
deps = [
"//cli/internal/clusterid",
"//cli/internal/terraform",
"//internal/attestation/idkeydigest",
"//internal/attestation/measurements",
"//internal/cloud/azureshared",
"//internal/cloud/cloudprovider",

View File

@ -1,10 +0,0 @@
apiVersion: v1
kind: ConfigMap
metadata:
name: join-config
namespace: {{ .Release.Namespace }}
data:
{{/* mustToJson is required so the json-strings passed from go are of type string in the rendered yaml. */}}
attestationConfig: {{ .Values.attestationConfig | mustToJson }}
binaryData:
measurementSalt: {{ .Values.measurementSalt }}

View File

@ -3,33 +3,33 @@
"properties": {
"csp": {
"description": "CSP to which the chart is deployed.",
"enum": ["AWS", "Azure", "GCP", "OpenStack", "QEMU"]
},
"attestationConfig": {
"description": "JSON-string to describe the config to use for attestation validation.",
"type": "string",
"examples": ["{'measurements':{'1':{'expected':'AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA','warnOnly':true},'15':{'expected':'AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=','warnOnly':true}}}"]
"enum": [
"AWS",
"Azure",
"GCP",
"OpenStack",
"QEMU"
]
},
"image": {
"description": "Container image to use for the spawned pods.",
"type": "string",
"examples": ["ghcr.io/edgelesssys/constellation/join-service:latest"]
},
"measurementSalt": {
"description": "Salt used to generate node measurements",
"type": "string",
"examples": ["AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"]
"examples": [
"ghcr.io/edgelesssys/constellation/join-service:latest"
]
},
"attestationVariant": {
"description": "Attestation variant to use for aTLS connections.",
"type": "string",
"examples": ["azure-sev-snp", "azure-trusted-launch", "gcp-sev-es"]
"examples": [
"azure-sev-snp",
"azure-trusted-launch",
"gcp-sev-es"
]
}
},
"required": [
"csp",
"attestationConfig",
"measurementSalt",
"image",
"attestationVariant"
],

View File

@ -1,5 +1,4 @@
csp: "gcp"
attestationVariant: ""
measurementSalt: ""
joinServicePort: 9090
joinServiceNodePort: 30090

View File

@ -134,10 +134,7 @@ func (i *ChartLoader) LoadReleases(
return nil, fmt.Errorf("loading constellation-services: %w", err)
}
if idFile.MeasurementSalt == nil {
return nil, errors.New("missing measurement salt in idFile")
}
svcVals, err := extraConstellationServicesValues(config, masterSecret, idFile.MeasurementSalt, idFile.UID, serviceAccURI, output)
svcVals, err := extraConstellationServicesValues(config, masterSecret, idFile.UID, serviceAccURI, output)
if err != nil {
return nil, fmt.Errorf("extending constellation-services values: %w", err)
}

View File

@ -7,7 +7,6 @@ SPDX-License-Identifier: AGPL-3.0-only
package helm
import (
"bytes"
"fmt"
"io/fs"
"os"
@ -25,7 +24,6 @@ import (
"github.com/edgelesssys/constellation/v2/cli/internal/clusterid"
"github.com/edgelesssys/constellation/v2/cli/internal/terraform"
"github.com/edgelesssys/constellation/v2/internal/attestation/idkeydigest"
"github.com/edgelesssys/constellation/v2/internal/attestation/measurements"
"github.com/edgelesssys/constellation/v2/internal/cloud/azureshared"
"github.com/edgelesssys/constellation/v2/internal/cloud/cloudprovider"
@ -71,7 +69,7 @@ func TestLoadReleases(t *testing.T) {
helmReleases, err := chartLoader.LoadReleases(
config, true, WaitModeAtomic,
uri.MasterSecret{Key: []byte("secret"), Salt: []byte("masterSalt")},
fakeServiceAccURI(cloudprovider.GCP), clusterid.File{UID: "testuid", MeasurementSalt: []byte("measurementSalt")}, terraform.ApplyOutput{GCP: &terraform.GCPApplyOutput{}},
fakeServiceAccURI(cloudprovider.GCP), clusterid.File{UID: "testuid"}, terraform.ApplyOutput{GCP: &terraform.GCPApplyOutput{}},
)
require.NoError(err)
chart := helmReleases.ConstellationServices.Chart
@ -113,18 +111,7 @@ func TestConstellationServices(t *testing.T) {
Provider: config.ProviderConfig{Azure: &config.AzureConfig{
DeployCSIDriver: toPtr(true),
}},
Attestation: config.AttestationConfig{AzureSEVSNP: &config.AzureSEVSNP{
Measurements: measurements.M{1: measurements.WithAllBytes(0xAA, measurements.Enforce, measurements.PCRMeasurementLength)},
FirmwareSignerConfig: config.SNPFirmwareSignerConfig{
AcceptedKeyDigests: idkeydigest.List{bytes.Repeat([]byte{0xAA}, 32)},
EnforcementPolicy: idkeydigest.MAAFallback,
MAAURL: "https://192.0.2.1:8080/maa",
},
BootloaderVersion: config.AttestationVersion{Value: 1, WantLatest: true},
TEEVersion: config.AttestationVersion{Value: 2, WantLatest: true},
SNPVersion: config.AttestationVersion{Value: 3, WantLatest: true},
MicrocodeVersion: config.AttestationVersion{Value: 4, WantLatest: true},
}},
Attestation: config.AttestationConfig{AzureSEVSNP: &config.AzureSEVSNP{}},
},
enforceIDKeyDigest: true,
ccmImage: "ccmImageForAzure",
@ -135,27 +122,21 @@ func TestConstellationServices(t *testing.T) {
Provider: config.ProviderConfig{GCP: &config.GCPConfig{
DeployCSIDriver: toPtr(true),
}},
Attestation: config.AttestationConfig{GCPSEVES: &config.GCPSEVES{
Measurements: measurements.M{1: measurements.WithAllBytes(0xAA, measurements.Enforce, measurements.PCRMeasurementLength)},
}},
Attestation: config.AttestationConfig{GCPSEVES: &config.GCPSEVES{}},
},
ccmImage: "ccmImageForGCP",
},
"OpenStack": {
config: &config.Config{
Provider: config.ProviderConfig{OpenStack: &config.OpenStackConfig{}},
Attestation: config.AttestationConfig{QEMUVTPM: &config.QEMUVTPM{
Measurements: measurements.M{1: measurements.WithAllBytes(0xAA, measurements.Enforce, measurements.PCRMeasurementLength)},
}},
Attestation: config.AttestationConfig{QEMUVTPM: &config.QEMUVTPM{}},
},
ccmImage: "ccmImageForOpenStack",
},
"QEMU": {
config: &config.Config{
Provider: config.ProviderConfig{QEMU: &config.QEMUConfig{}},
Attestation: config.AttestationConfig{QEMUVTPM: &config.QEMUVTPM{
Measurements: measurements.M{1: measurements.WithAllBytes(0xAA, measurements.Enforce, measurements.PCRMeasurementLength)},
}},
Attestation: config.AttestationConfig{QEMUVTPM: &config.QEMUVTPM{}},
},
},
}
@ -186,7 +167,6 @@ func TestConstellationServices(t *testing.T) {
Key: []byte("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"),
Salt: []byte("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"),
},
[]byte("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"),
"uid", serviceAccURI, terraform.ApplyOutput{
Azure: &terraform.AzureApplyOutput{},
GCP: &terraform.GCPApplyOutput{},
@ -381,13 +361,6 @@ func buildTestdataMap(csp string, expectedData map[string]string, require *requi
// addInClusterValues adds values that are only known after the cluster is created.
func addInClusterValues(values map[string]any, csp cloudprovider.Provider) error {
joinVals, ok := values["join-service"].(map[string]any)
if !ok {
return errors.New("missing 'join-service' key")
}
joinVals["measurementSalt"] = "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"
verificationVals, ok := values["verification-service"].(map[string]any)
if !ok {
return fmt.Errorf("missing 'verification-service' key %v", values)

View File

@ -54,17 +54,11 @@ func extraCiliumValues(provider cloudprovider.Provider, conformanceMode bool, ou
// extraConstellationServicesValues extends the given values map by some values depending on user input.
// Values set inside this function are only applied during init, not during upgrade.
func extraConstellationServicesValues(
cfg *config.Config, masterSecret uri.MasterSecret, measurementSalt []byte, uid, serviceAccURI string, output terraform.ApplyOutput,
cfg *config.Config, masterSecret uri.MasterSecret, uid, serviceAccURI string, output terraform.ApplyOutput,
) (map[string]any, error) {
attestationConfigJSON, err := json.Marshal(cfg.GetAttestationConfig())
if err != nil {
return nil, fmt.Errorf("marshalling measurements: %w", err)
}
extraVals := map[string]any{}
extraVals["join-service"] = map[string]any{
"measurementSalt": base64.StdEncoding.EncodeToString(measurementSalt),
"attestationVariant": cfg.GetAttestationConfig().GetVariant().String(),
"attestationConfig": string(attestationConfigJSON),
}
extraVals["verification-service"] = map[string]any{
"attestationVariant": cfg.GetAttestationConfig().GetVariant().String(),

View File

@ -1,9 +0,0 @@
apiVersion: v1
kind: ConfigMap
metadata:
name: join-config
namespace: testNamespace
data:
attestationConfig: "{\"measurements\":{\"1\":{\"expected\":\"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\",\"warnOnly\":false}}}"
binaryData:
measurementSalt: AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA

View File

@ -1,9 +0,0 @@
apiVersion: v1
kind: ConfigMap
metadata:
name: join-config
namespace: testNamespace
data:
attestationConfig: "{\"measurements\":{\"1\":{\"expected\":\"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\",\"warnOnly\":false}},\"bootloaderVersion\":1,\"teeVersion\":2,\"snpVersion\":3,\"microcodeVersion\":4,\"firmwareSignerConfig\":{\"acceptedKeyDigests\":[\"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\"],\"enforcementPolicy\":\"MAAFallback\",\"maaURL\":\"https://192.0.2.1:8080/maa\"},\"amdRootKey\":\"-----BEGIN CERTIFICATE-----\\n-----END CERTIFICATE-----\\n\"}"
binaryData:
measurementSalt: AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA

View File

@ -1,9 +0,0 @@
apiVersion: v1
kind: ConfigMap
metadata:
name: join-config
namespace: testNamespace
data:
attestationConfig: "{\"measurements\":{\"1\":{\"expected\":\"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\",\"warnOnly\":false}}}"
binaryData:
measurementSalt: AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA

View File

@ -1,9 +0,0 @@
apiVersion: v1
kind: ConfigMap
metadata:
name: join-config
namespace: testNamespace
data:
attestationConfig: "{\"measurements\":{\"1\":{\"expected\":\"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\",\"warnOnly\":false}}}"
binaryData:
measurementSalt: AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA

View File

@ -1,9 +0,0 @@
apiVersion: v1
kind: ConfigMap
metadata:
name: join-config
namespace: testNamespace
data:
attestationConfig: "{\"measurements\":{\"1\":{\"expected\":\"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\",\"warnOnly\":false}}}"
binaryData:
measurementSalt: AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA

View File

@ -31,6 +31,7 @@ go_library(
"@io_k8s_client_go//util/retry",
"@io_k8s_kubernetes//cmd/kubeadm/app/apis/kubeadm/v1beta3",
"@io_k8s_sigs_yaml//:yaml",
"@sh_helm_helm_v3//pkg/kube",
],
)
@ -39,6 +40,7 @@ go_test(
srcs = ["kubecmd_test.go"],
embed = [":kubecmd"],
deps = [
"//internal/attestation/measurements",
"//internal/attestation/variant",
"//internal/cloud/cloudprovider",
"//internal/compatibility",

View File

@ -37,6 +37,7 @@ import (
"github.com/edgelesssys/constellation/v2/internal/versions"
"github.com/edgelesssys/constellation/v2/internal/versions/components"
updatev1alpha1 "github.com/edgelesssys/constellation/v2/operators/constellation-node-operator/v2/api/v1alpha1"
"helm.sh/helm/v3/pkg/kube"
corev1 "k8s.io/api/core/v1"
k8serrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@ -193,35 +194,37 @@ func (k *KubeCmd) GetClusterAttestationConfig(ctx context.Context, variant varia
return existingAttestationConfig, nil
}
// UpdateAttestationConfig updates the Constellation cluster's attestation config.
// A backup of the previous config is created before updating.
func (k *KubeCmd) UpdateAttestationConfig(ctx context.Context, newAttestConfig config.AttestationCfg) error {
// backup of previous measurements
joinConfig, err := k.kubectl.GetConfigMap(ctx, constants.ConstellationNamespace, constants.JoinConfigMap)
if err != nil {
return fmt.Errorf("getting %s ConfigMap: %w", constants.JoinConfigMap, err)
}
// create backup of previous config
backup := joinConfig.DeepCopy()
backup.ObjectMeta = metav1.ObjectMeta{}
backup.Name = fmt.Sprintf("%s-backup", constants.JoinConfigMap)
if err := k.applyConfigMap(ctx, backup); err != nil {
return fmt.Errorf("creating backup of join-config ConfigMap: %w", err)
}
k.log.Debugf("Created backup of %s ConfigMap %q in namespace %q", constants.JoinConfigMap, backup.Name, backup.Namespace)
// ApplyJoinConfig creates or updates the Constellation cluster's join-config ConfigMap.
// This ConfigMap holds the attestation config and measurement salt of the cluster.
// A backup of the previous attestation config is created with the suffix `_backup` in the config map data.
func (k *KubeCmd) ApplyJoinConfig(ctx context.Context, newAttestConfig config.AttestationCfg, measurementSalt []byte) error {
newConfigJSON, err := json.Marshal(newAttestConfig)
if err != nil {
return fmt.Errorf("marshaling attestation config: %w", err)
}
joinConfig, err := k.kubectl.GetConfigMap(ctx, constants.ConstellationNamespace, constants.JoinConfigMap)
if err != nil {
if !k8serrors.IsNotFound(err) {
return fmt.Errorf("getting %s ConfigMap: %w", constants.JoinConfigMap, err)
}
k.log.Debugf("ConfigMap %q does not exist in namespace %q, creating it now", constants.JoinConfigMap, constants.ConstellationNamespace)
if err := k.kubectl.CreateConfigMap(ctx, joinConfigMap(newConfigJSON, measurementSalt)); err != nil {
return fmt.Errorf("creating join-config ConfigMap: %w", err)
}
k.log.Debugf("Created %q ConfigMap in namespace %q", constants.JoinConfigMap, constants.ConstellationNamespace)
return nil
}
// create backup of previous config
joinConfig.Data[constants.AttestationConfigFilename+"_backup"] = joinConfig.Data[constants.AttestationConfigFilename]
joinConfig.Data[constants.AttestationConfigFilename] = string(newConfigJSON)
k.log.Debugf("Triggering attestation config update now")
if _, err = k.kubectl.UpdateConfigMap(ctx, joinConfig); err != nil {
return fmt.Errorf("setting new attestation config: %w", err)
}
fmt.Fprintln(k.outWriter, "Successfully updated the cluster's attestation config")
return nil
}
@ -398,19 +401,6 @@ func (k *KubeCmd) updateK8s(nodeVersion *updatev1alpha1.NodeVersion, newClusterV
return &configMap, nil
}
// applyConfigMap applies the ConfigMap by creating it if it doesn't exist, or updating it if it does.
func (k *KubeCmd) applyConfigMap(ctx context.Context, configMap *corev1.ConfigMap) error {
if err := k.kubectl.CreateConfigMap(ctx, configMap); err != nil {
if !k8serrors.IsAlreadyExists(err) {
return fmt.Errorf("creating backup config map: %w", err)
}
if _, err := k.kubectl.UpdateConfigMap(ctx, configMap); err != nil {
return fmt.Errorf("updating backup config map: %w", err)
}
}
return nil
}
func checkForApplyError(expected, actual updatev1alpha1.NodeVersion) error {
var err error
switch {
@ -426,6 +416,87 @@ func checkForApplyError(expected, actual updatev1alpha1.NodeVersion) error {
return err
}
func joinConfigMap(attestationCfgJSON, measurementSalt []byte) *corev1.ConfigMap {
return &corev1.ConfigMap{
ObjectMeta: metav1.ObjectMeta{
Name: constants.JoinConfigMap,
Namespace: constants.ConstellationNamespace,
},
Data: map[string]string{
constants.AttestationConfigFilename: string(attestationCfgJSON),
},
BinaryData: map[string][]byte{
constants.MeasurementSaltFilename: measurementSalt,
},
}
}
// RemoveAttestationConfigHelmManagement removes labels and annotations from the join-config ConfigMap that are added by Helm.
// This is to ensure we can cleanly transition from Helm to Constellation's management of the ConfigMap.
// TODO(v2.11): Remove this function after v2.11 is released.
func (k *KubeCmd) RemoveAttestationConfigHelmManagement(ctx context.Context) error {
const (
appManagedByLabel = "app.kubernetes.io/managed-by"
appManagedByHelm = "Helm"
helmReleaseNameAnnotation = "meta.helm.sh/release-name"
helmReleaseNamespaceAnnotation = "meta.helm.sh/release-namespace"
)
k.log.Debugf("Checking if join-config ConfigMap needs to be migrated to remove Helm management")
joinConfig, err := k.kubectl.GetConfigMap(ctx, constants.ConstellationNamespace, constants.JoinConfigMap)
if err != nil {
return fmt.Errorf("getting join config: %w", err)
}
var needUpdate bool
if managedBy, ok := joinConfig.Labels[appManagedByLabel]; ok && managedBy == appManagedByHelm {
delete(joinConfig.Labels, appManagedByLabel)
needUpdate = true
}
if _, ok := joinConfig.Annotations[helmReleaseNameAnnotation]; ok {
delete(joinConfig.Annotations, helmReleaseNameAnnotation)
needUpdate = true
}
if _, ok := joinConfig.Annotations[helmReleaseNamespaceAnnotation]; ok {
delete(joinConfig.Annotations, helmReleaseNamespaceAnnotation)
needUpdate = true
}
if needUpdate {
// Tell Helm to ignore this resource in the future.
// TODO(v2.11): Remove this annotation from the ConfigMap.
joinConfig.Annotations[kube.ResourcePolicyAnno] = kube.KeepPolicy
k.log.Debugf("Removing Helm management labels from join-config ConfigMap")
if _, err := k.kubectl.UpdateConfigMap(ctx, joinConfig); err != nil {
return fmt.Errorf("removing Helm management labels from join-config: %w", err)
}
k.log.Debugf("Successfully removed Helm management labels from join-config ConfigMap")
}
return nil
}
// RemoveHelmKeepAnnotation removes the Helm Resource Policy annotation from the join-config ConfigMap.
// TODO(v2.12): Remove this function after v2.12 is released.
func (k *KubeCmd) RemoveHelmKeepAnnotation(ctx context.Context) error {
k.log.Debugf("Checking if Helm Resource Policy can be removed from join-config ConfigMap")
joinConfig, err := k.kubectl.GetConfigMap(ctx, constants.ConstellationNamespace, constants.JoinConfigMap)
if err != nil {
return fmt.Errorf("getting join config: %w", err)
}
if policy, ok := joinConfig.Annotations[kube.ResourcePolicyAnno]; ok && policy == kube.KeepPolicy {
delete(joinConfig.Annotations, kube.ResourcePolicyAnno)
if _, err := k.kubectl.UpdateConfigMap(ctx, joinConfig); err != nil {
return fmt.Errorf("removing Helm Resource Policy from join-config: %w", err)
}
k.log.Debugf("Successfully removed Helm Resource Policy from join-config ConfigMap")
}
return nil
}
// kubectlInterface is provides access to the Kubernetes API.
type kubectlInterface interface {
GetNodes(ctx context.Context) ([]corev1.Node, error)

View File

@ -13,8 +13,7 @@ import (
"io"
"testing"
kerrors "k8s.io/apimachinery/pkg/api/errors"
"github.com/edgelesssys/constellation/v2/internal/attestation/measurements"
"github.com/edgelesssys/constellation/v2/internal/attestation/variant"
"github.com/edgelesssys/constellation/v2/internal/cloud/cloudprovider"
"github.com/edgelesssys/constellation/v2/internal/compatibility"
@ -28,6 +27,7 @@ import (
"github.com/stretchr/testify/mock"
"github.com/stretchr/testify/require"
corev1 "k8s.io/api/core/v1"
k8serrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/runtime"
@ -281,7 +281,7 @@ func TestUpgradeNodeVersion(t *testing.T) {
customClientFn: func(nodeVersion updatev1alpha1.NodeVersion) unstructuredInterface {
fakeClient := &fakeUnstructuredClient{}
fakeClient.On("GetCR", mock.Anything, mock.Anything).Return(unstructedObjectWithGeneration(nodeVersion, 1), nil)
fakeClient.On("UpdateCR", mock.Anything, mock.Anything).Return(nil, kerrors.NewConflict(schema.GroupResource{Resource: nodeVersion.Name}, nodeVersion.Name, nil)).Once()
fakeClient.On("UpdateCR", mock.Anything, mock.Anything).Return(nil, k8serrors.NewConflict(schema.GroupResource{Resource: nodeVersion.Name}, nodeVersion.Name, nil)).Once()
fakeClient.On("UpdateCR", mock.Anything, mock.Anything).Return(unstructedObjectWithGeneration(nodeVersion, 2), nil).Once()
return fakeClient
},
@ -470,7 +470,7 @@ func newJoinConfigMap(data string) *corev1.ConfigMap {
}
}
func TestUpdateAttestationConfig(t *testing.T) {
func TestApplyAttestationConfig(t *testing.T) {
mustMarshal := func(cfg config.AttestationCfg) string {
data, err := json.Marshal(cfg)
require.NoError(t, err)
@ -483,25 +483,66 @@ func TestUpdateAttestationConfig(t *testing.T) {
wantErr bool
}{
"success": {
newAttestationCfg: config.DefaultForAzureSEVSNP(),
newAttestationCfg: &config.QEMUVTPM{
Measurements: measurements.M{
0: measurements.WithAllBytes(0x00, measurements.WarnOnly, measurements.PCRMeasurementLength),
},
},
kubectl: &stubKubectl{
configMaps: map[string]*corev1.ConfigMap{
constants.JoinConfigMap: newJoinConfigMap(mustMarshal(config.DefaultForAzureSEVSNP())),
constants.JoinConfigMap: newJoinConfigMap(mustMarshal(&config.QEMUVTPM{
Measurements: measurements.M{
0: measurements.WithAllBytes(0xFF, measurements.WarnOnly, measurements.PCRMeasurementLength),
},
})),
},
},
},
"error getting ConfigMap": {
newAttestationCfg: config.DefaultForAzureSEVSNP(),
"Get ConfigMap error": {
newAttestationCfg: &config.QEMUVTPM{
Measurements: measurements.M{
0: measurements.WithAllBytes(0x00, measurements.WarnOnly, measurements.PCRMeasurementLength),
},
},
kubectl: &stubKubectl{
getCMErr: assert.AnError,
},
wantErr: true,
},
"ConfigMap does not exist yet": {
newAttestationCfg: &config.QEMUVTPM{
Measurements: measurements.M{
0: measurements.WithAllBytes(0x00, measurements.WarnOnly, measurements.PCRMeasurementLength),
},
},
kubectl: &stubKubectl{
getCMErr: k8serrors.NewNotFound(schema.GroupResource{}, ""),
},
},
"Update ConfigMap error": {
newAttestationCfg: &config.QEMUVTPM{
Measurements: measurements.M{
0: measurements.WithAllBytes(0x00, measurements.WarnOnly, measurements.PCRMeasurementLength),
},
},
kubectl: &stubKubectl{
configMaps: map[string]*corev1.ConfigMap{
constants.JoinConfigMap: newJoinConfigMap(mustMarshal(&config.QEMUVTPM{
Measurements: measurements.M{
0: measurements.WithAllBytes(0xFF, measurements.WarnOnly, measurements.PCRMeasurementLength),
},
})),
},
updateCMErr: assert.AnError,
},
wantErr: true,
},
}
for name, tc := range testCases {
t.Run(name, func(t *testing.T) {
assert := assert.New(t)
require := require.New(t)
cmd := &KubeCmd{
kubectl: tc.kubectl,
@ -509,12 +550,15 @@ func TestUpdateAttestationConfig(t *testing.T) {
outWriter: io.Discard,
}
err := cmd.UpdateAttestationConfig(context.Background(), tc.newAttestationCfg)
err := cmd.ApplyJoinConfig(context.Background(), tc.newAttestationCfg, []byte{0x11})
if tc.wantErr {
assert.Error(err)
return
}
assert.NoError(err)
cfg, ok := tc.kubectl.configMaps[constants.JoinConfigMap]
require.True(ok)
assert.Equal(mustMarshal(tc.newAttestationCfg), cfg.Data[constants.AttestationConfigFilename])
})
}
}

View File

@ -40,7 +40,7 @@ Or alternatively, for `terminate`:
ARM_SKIP_PROVIDER_REGISTRATION=true constellation terminate
```
### Nodes fail to join with error `untrusted PCR value`
### Nodes fail to join with error `untrusted measurement value`
This error indicates that a node's [attestation statement](../architecture/attestation.md) contains measurements that don't match the trusted values expected by the [JoinService](../architecture/microservices.md#joinservice).
This may for example happen if the cloud provider updates the VM's firmware such that it influences the [runtime measurements](../architecture/attestation.md#runtime-measurements) in an unforeseen way.
@ -55,14 +55,16 @@ When in doubt, check if the encountered [issue is known](https://github.com/edge
:::
:::tip
During an upgrade with modified attestation config, a backup of the current configuration is stored in the `join-config-backup` config map in the `kube-system` namespace. To restore the old attestation config after a failed upgrade, you can copy the attestation config from this resource, put it in your configuration file and retry the upgrade.
During an upgrade with modified attestation config, a backup of the current configuration is stored in the `join-config` config map in the `kube-system` namespace under the `attestationConfig_backup` key. To restore the old attestation config after a failed upgrade, replace the value of `attestationConfig` with the value from `attestationConfig_backup`:
```bash
kubectl patch configmaps -n kube-system join-config -p "{\"data\":{\"attestationConfig\":\"$(kubectl get configmaps -n kube-system join-config -o "jsonpath={.data.attestationConfig_backup}")\"}}"
```
:::
You can use the `upgrade apply` command to change measurements of a running cluster:
1. Modify the `measurements` key in your local `constellation-conf.yaml` to the expected values.