Refactor Helm deployments (#341)

* Wrap KMS deployment in one main chart that
deploys all other services. Other services will follow.
* Use .tgz via helm-package as serialization format
* Change Release type to carry chart as byte slice
* Remove KMSConfig
* Use json-schema to validate values
* Extend release.md to mention updating helm charts
This commit is contained in:
Otto Bittner 2022-10-21 12:01:28 +02:00 committed by GitHub
parent 10a207c7ec
commit 07f02a442c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
31 changed files with 261 additions and 119 deletions

View File

@ -51,15 +51,16 @@ This checklist will prepare `v1.3.0` from `v1.2.0`. Adjust your version numbers
1. Rename the "Unreleased" heading to "[v1.3.0] - YYYY-MM-DD" and link the version to the upcoming release tag.
2. Create a new block for unreleased changes
5. Update project version in [CMakeLists.txt](/CMakeLists.txt) to `1.3.0` (without v).
6. When the microservice builds are finished update versions in [versions.go](../../internal/versions/versions.go#L33-L39) to `v1.3.0`, **add the container hashes** and **push your changes**.
7. Create a [production OS image](/.github/workflows/build-coreos.yml)
6. Update the `version` key in [constellation-services/Chart.yaml](/cli/internal/helm/charts/edgeless/constellation-services/Chart.yaml). Also update the `version` key for all subcharts, e.g. [Chart.yaml](/cli/internal/helm/charts/edgeless/constellation-services/charts/kms/Chart.yaml). Lastly, update the `dependencies.*.version` key for all dependencies in the main chart [constellation-services/Chart.yaml](/cli/internal/helm/charts/edgeless/constellation-services/Chart.yaml).
7. When the microservice builds are finished update versions in [versions.go](../../internal/versions/versions.go#L33-L39) to `v1.3.0`, **add the container hashes** and **push your changes**.
8. Create a [production coreOS image](/.github/workflows/build-coreos.yml)
```sh
gh workflow run build-os-image.yml --ref release/v$minor -F debug=false -F imageVersion=v$ver
```
8. Update [default images in config](/internal/config/images_enterprise.go)
9. Run manual E2E tests using [Linux](/.github/workflows/e2e-test-manual.yml) and [macOS](/.github/workflows/e2e-test-manual-macos.yml) to confirm functionality and stability.
9. Update [default images in config](/internal/config/images_enterprise.go)
10. Run manual E2E tests using [Linux](/.github/workflows/e2e-test-manual.yml) and [macOS](/.github/workflows/e2e-test-manual-macos.yml) to confirm functionality and stability.
```sh
sono='--plugin e2e --plugin-env e2e.E2E_FOCUS="\[Conformance\]" --plugin-env e2e.E2E_SKIP="for service with type clusterIP|HostPort validates that there is no conflict between pods with same hostPort but different hostIP and protocol" --plugin https://raw.githubusercontent.com/vmware-tanzu/sonobuoy-plugins/master/cis-benchmarks/kube-bench-plugin.yaml --plugin https://raw.githubusercontent.com/vmware-tanzu/sonobuoy-plugins/master/cis-benchmarks/kube-bench-master-plugin.yaml'
@ -69,14 +70,14 @@ This checklist will prepare `v1.3.0` from `v1.2.0`. Adjust your version numbers
gh workflow run e2e-test-manual-macos.yml --ref release/v$minor -F cloudProvider=gcp -F machineType=n2d-standard-4 -F sonobuoyTestSuiteCmd="$sono" -F osImage=projects/constellation-images/global/images/constellation-v$gcpVer -F isDebugImage=false
```
10. [Generate measurements](/.github/workflows/generate-measurements.yml) for the images on each CSP.
11. [Generate measurements](/.github/workflows/generate-measurements.yml) for the images on each CSP.
```sh
gh workflow run generate-measurements.yml --ref release/v$minor -F cloudProvider=azure -F osImage=/CommunityGalleries/ConstellationCVM-b3782fa0-0df7-4f2f-963e-fc7fc42663df/Images/constellation/Versions/$ver -F isDebugImage=false
gh workflow run generate-measurements.yml --ref release/v$minor -F cloudProvider=gcp -F osImage=projects/constellation-images/global/images/constellation-v$gcpVer -F isDebugImage=false
```
11. Create a new tag on this release branch
12. Create a new tag on this release branch
```sh
git tag v$ver
git tags --push

View File

@ -63,4 +63,4 @@ add_test(NAME integration-node-operator COMMAND make test WORKING_DIRECTORY ${CM
add_test(NAME integration-csi COMMAND bash -c "go test -tags integration -c ./test/ && sudo ./test.test -test.v" WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}/csi)
add_test(NAME integration-dm COMMAND bash -c "go test -tags integration -c ./test/ && sudo ./test.test -test.v" WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}/disk-mapper/internal)
add_test(NAME integration-license COMMAND bash -c "go test -tags integration" WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}/internal/license)
add_test(NAME helm-lint COMMAND bash -c "helm lint --strict *" WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}/cli/internal/helm/charts/edgeless/)
add_test(NAME helm-lint COMMAND bash -c "helm lint * --set kms.image='ghcr.io/edgelesssys/constellation/kms:latest' --set kms.salt='aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa' --set kms.masterSecret='aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa'" WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}/cli/internal/helm/charts/edgeless/)

View File

@ -9,7 +9,6 @@ package main
import (
"context"
"github.com/edgelesssys/constellation/v2/bootstrapper/internal/helm"
"github.com/edgelesssys/constellation/v2/internal/cloud/metadata"
"github.com/edgelesssys/constellation/v2/internal/logger"
"github.com/edgelesssys/constellation/v2/internal/role"
@ -22,7 +21,7 @@ type clusterFake struct{}
// InitCluster fakes bootstrapping a new cluster with the current node being the master, returning the arguments required to join the cluster.
func (c *clusterFake) InitCluster(
context.Context, string, string, []byte, []uint32, bool, []byte, bool,
helm.KMSConfig, map[string]string, []byte, bool, *logger.Logger,
map[string]string, []byte, bool, *logger.Logger,
) ([]byte, error) {
return []byte{}, nil
}

View File

@ -7,8 +7,8 @@ SPDX-License-Identifier: AGPL-3.0-only
package helm
import (
"bytes"
"context"
"encoding/base64"
"errors"
"fmt"
"net"
@ -21,6 +21,7 @@ import (
"github.com/edgelesssys/constellation/v2/internal/deploy/helm"
"github.com/edgelesssys/constellation/v2/internal/logger"
"helm.sh/helm/v3/pkg/action"
"helm.sh/helm/v3/pkg/chart/loader"
"helm.sh/helm/v3/pkg/cli"
corev1 "k8s.io/api/core/v1"
)
@ -51,6 +52,31 @@ func New(log *logger.Logger) (*Client, error) {
}, nil
}
// InstallConstellationServices installs the constellation-services chart. In the future this chart should bundle all microservices.
func (h *Client) InstallConstellationServices(ctx context.Context, release helm.Release) error {
h.Namespace = constants.HelmNamespace
h.ReleaseName = release.ReleaseName
h.Wait = release.Wait
h.Timeout = timeout
// update dependencies - unsure if necessary for local deps.
h.DependencyUpdate = true
// TODO: Possibly fetch metadata to extend values here.
reader := bytes.NewReader(release.Chart)
chart, err := loader.LoadArchive(reader)
if err != nil {
return fmt.Errorf("helm load archive: %w", err)
}
_, err = h.RunWithContext(ctx, chart, release.Values)
if err != nil {
return fmt.Errorf("helm install services: %w", err)
}
return nil
}
// InstallCilium sets up the cilium pod network.
func (h *Client) InstallCilium(ctx context.Context, kubectl k8sapi.Client, release helm.Release, in k8sapi.SetupPodNetworkInput) error {
h.Namespace = constants.HelmNamespace
@ -75,7 +101,13 @@ func (h *Client) installCiliumAzure(ctx context.Context, release helm.Release, k
release.Values["k8sServiceHost"] = host
release.Values["k8sServicePort"] = strconv.Itoa(constants.KubernetesPort)
_, err := h.RunWithContext(ctx, release.Chart, release.Values)
reader := bytes.NewReader(release.Chart)
chart, err := loader.LoadArchive(reader)
if err != nil {
return fmt.Errorf("helm load archive: %w", err)
}
_, err = h.RunWithContext(ctx, chart, release.Values)
if err != nil {
return fmt.Errorf("installing cilium: %w", err)
}
@ -127,7 +159,13 @@ func (h *Client) installlCiliumGCP(ctx context.Context, kubectl k8sapi.Client, r
release.Values["k8sServicePort"] = port
}
_, err = h.RunWithContext(ctx, release.Chart, release.Values)
reader := bytes.NewReader(release.Chart)
chart, err := loader.LoadArchive(reader)
if err != nil {
return fmt.Errorf("helm load archive: %w", err)
}
_, err = h.RunWithContext(ctx, chart, release.Values)
if err != nil {
return fmt.Errorf("helm install cilium: %w", err)
}
@ -148,27 +186,15 @@ func (h *Client) installCiliumQEMU(ctx context.Context, release helm.Release, su
release.Values["k8sServiceHost"] = kubeAPIEndpoint
release.Values["k8sServicePort"] = strconv.Itoa(constants.KubernetesPort)
_, err := h.RunWithContext(ctx, release.Chart, release.Values)
reader := bytes.NewReader(release.Chart)
chart, err := loader.LoadArchive(reader)
if err != nil {
return fmt.Errorf("helm load archive: %w", err)
}
_, err = h.RunWithContext(ctx, chart, release.Values)
if err != nil {
return fmt.Errorf("helm install cilium: %w", err)
}
return nil
}
// InstallKMS deploys the KMS deployment.
func (h *Client) InstallKMS(ctx context.Context, release helm.Release, kmsConfig KMSConfig) error {
h.Namespace = constants.HelmNamespace
h.ReleaseName = release.ReleaseName
h.Wait = release.Wait
h.Timeout = timeout
release.Values["masterSecret"] = base64.StdEncoding.EncodeToString(kmsConfig.MasterSecret[:])
release.Values["salt"] = base64.StdEncoding.EncodeToString(kmsConfig.Salt[:])
_, err := h.RunWithContext(ctx, release.Chart, release.Values)
if err != nil {
return fmt.Errorf("helm install kms: %w", err)
}
return nil
}

View File

@ -1,13 +0,0 @@
/*
Copyright (c) Edgeless Systems GmbH
SPDX-License-Identifier: AGPL-3.0-only
*/
package helm
// KMSConfig is the configuration needed to set up Constellation's key management service.
type KMSConfig struct {
MasterSecret []byte
Salt []byte
}

View File

@ -15,7 +15,6 @@ import (
"github.com/edgelesssys/constellation/v2/bootstrapper/initproto"
"github.com/edgelesssys/constellation/v2/bootstrapper/internal/diskencryption"
"github.com/edgelesssys/constellation/v2/bootstrapper/internal/helm"
"github.com/edgelesssys/constellation/v2/internal/atls"
"github.com/edgelesssys/constellation/v2/internal/attestation"
"github.com/edgelesssys/constellation/v2/internal/cloud/vmtype"
@ -132,10 +131,6 @@ func (s *Server) Init(ctx context.Context, req *initproto.InitRequest) (*initpro
req.EnforceIdkeydigest,
s.issuerWrapper.IDKeyDigest(),
s.issuerWrapper.VMType() == vmtype.AzureCVM,
helm.KMSConfig{
MasterSecret: req.MasterSecret,
Salt: req.Salt,
},
sshProtoKeysToMap(req.SshUserKeys),
req.HelmDeployments,
req.ConformanceMode,
@ -236,7 +231,6 @@ type ClusterInitializer interface {
enforceIDKeyDigest bool,
idKeyDigest []byte,
azureCVM bool,
kmsConfig helm.KMSConfig,
sshUserKeys map[string]string,
helmDeployments []byte,
conformanceMode bool,

View File

@ -16,7 +16,6 @@ import (
"time"
"github.com/edgelesssys/constellation/v2/bootstrapper/initproto"
"github.com/edgelesssys/constellation/v2/bootstrapper/internal/helm"
"github.com/edgelesssys/constellation/v2/internal/crypto/testvector"
"github.com/edgelesssys/constellation/v2/internal/file"
"github.com/edgelesssys/constellation/v2/internal/logger"
@ -290,7 +289,7 @@ type stubClusterInitializer struct {
func (i *stubClusterInitializer) InitCluster(
context.Context, string, string, []byte, []uint32, bool, []byte, bool,
helm.KMSConfig, map[string]string, []byte, bool, *logger.Logger,
map[string]string, []byte, bool, *logger.Logger,
) ([]byte, error) {
return i.initClusterKubeconfig, i.initClusterErr
}

View File

@ -10,7 +10,6 @@ import (
"context"
"net"
helmClient "github.com/edgelesssys/constellation/v2/bootstrapper/internal/helm"
"github.com/edgelesssys/constellation/v2/bootstrapper/internal/kubernetes/k8sapi"
"github.com/edgelesssys/constellation/v2/internal/deploy/helm"
"github.com/edgelesssys/constellation/v2/internal/kubernetes"
@ -38,10 +37,10 @@ type clusterUtil interface {
StartKubelet() error
}
// HelmUtil bundles functions related to microservice deployment. Only microservices that can be deployed purely via Helm are deployed with this interface.
// helmClient bundles functions related to microservice deployment. Only microservices that can be deployed purely via Helm are deployed with this interface.
// Currently only a subset of microservices is deployed via Helm.
// Naming is inspired by Helm.
type HelmUtil interface {
type helmClient interface {
InstallCilium(context.Context, k8sapi.Client, helm.Release, k8sapi.SetupPodNetworkInput) error
InstallKMS(context.Context, helm.Release, helmClient.KMSConfig) error
InstallConstellationServices(ctx context.Context, release helm.Release) error
}

View File

@ -15,7 +15,6 @@ import (
"strconv"
"strings"
helmClient "github.com/edgelesssys/constellation/v2/bootstrapper/internal/helm"
"github.com/edgelesssys/constellation/v2/bootstrapper/internal/kubernetes/k8sapi"
"github.com/edgelesssys/constellation/v2/bootstrapper/internal/kubernetes/k8sapi/resources"
"github.com/edgelesssys/constellation/v2/internal/cloud/metadata"
@ -46,7 +45,7 @@ type configurationProvider interface {
type KubeWrapper struct {
cloudProvider string
clusterUtil clusterUtil
helmUtil HelmUtil
helmClient helmClient
configProvider configurationProvider
client k8sapi.Client
kubeconfigReader configReader
@ -60,12 +59,12 @@ type KubeWrapper struct {
// New creates a new KubeWrapper with real values.
func New(cloudProvider string, clusterUtil clusterUtil, configProvider configurationProvider, client k8sapi.Client, cloudControllerManager CloudControllerManager,
cloudNodeManager CloudNodeManager, clusterAutoscaler ClusterAutoscaler, providerMetadata ProviderMetadata, initialMeasurementsJSON []byte, helmUtil HelmUtil,
cloudNodeManager CloudNodeManager, clusterAutoscaler ClusterAutoscaler, providerMetadata ProviderMetadata, initialMeasurementsJSON []byte, helmClient helmClient,
) *KubeWrapper {
return &KubeWrapper{
cloudProvider: cloudProvider,
clusterUtil: clusterUtil,
helmUtil: helmUtil,
helmClient: helmClient,
configProvider: configProvider,
client: client,
kubeconfigReader: &KubeconfigReader{fs: afero.Afero{Fs: afero.NewOsFs()}},
@ -81,7 +80,7 @@ func New(cloudProvider string, clusterUtil clusterUtil, configProvider configura
// InitCluster initializes a new Kubernetes cluster and applies pod network provider.
func (k *KubeWrapper) InitCluster(
ctx context.Context, cloudServiceAccountURI, versionString string, measurementSalt []byte, enforcedPCRs []uint32,
enforceIDKeyDigest bool, idKeyDigest []byte, azureCVM bool, kmsConfig helmClient.KMSConfig, sshUsers map[string]string,
enforceIDKeyDigest bool, idKeyDigest []byte, azureCVM bool, sshUsers map[string]string,
helmReleasesRaw []byte, conformanceMode bool, log *logger.Logger,
) ([]byte, error) {
k8sVersion, err := versions.NewValidK8sVersion(versionString)
@ -184,7 +183,7 @@ func (k *KubeWrapper) InitCluster(
return nil, fmt.Errorf("unmarshalling helm releases: %w", err)
}
if err = k.helmUtil.InstallCilium(ctx, k.client, helmReleases.Cilium, setupPodNetworkInput); err != nil {
if err = k.helmClient.InstallCilium(ctx, k.client, helmReleases.Cilium, setupPodNetworkInput); err != nil {
return nil, fmt.Errorf("installing pod network: %w", err)
}
@ -201,7 +200,7 @@ func (k *KubeWrapper) InitCluster(
return nil, fmt.Errorf("setting up konnectivity: %w", err)
}
if err = k.helmUtil.InstallKMS(ctx, helmReleases.KMS, kmsConfig); err != nil {
if err = k.helmClient.InstallConstellationServices(ctx, helmReleases.ConstellationServices); err != nil {
return nil, fmt.Errorf("installing kms: %w", err)
}

View File

@ -14,7 +14,6 @@ import (
"strconv"
"testing"
helmClient "github.com/edgelesssys/constellation/v2/bootstrapper/internal/helm"
"github.com/edgelesssys/constellation/v2/bootstrapper/internal/kubernetes/k8sapi"
"github.com/edgelesssys/constellation/v2/internal/cloud/metadata"
"github.com/edgelesssys/constellation/v2/internal/constants"
@ -37,7 +36,6 @@ func TestMain(m *testing.M) {
func TestInitCluster(t *testing.T) {
someErr := errors.New("failed")
serviceAccountURI := "some-service-account-uri"
masterSecret := []byte("some-master-secret")
nodeName := "node-name"
providerID := "provider-id"
@ -48,7 +46,7 @@ func TestInitCluster(t *testing.T) {
testCases := map[string]struct {
clusterUtil stubClusterUtil
helmUtil stubHelmClient
helmClient stubHelmClient
kubectl stubKubectl
providerMetadata ProviderMetadata
CloudControllerManager CloudControllerManager
@ -182,7 +180,7 @@ func TestInitCluster(t *testing.T) {
},
"kubeadm init fails when deploying cilium": {
clusterUtil: stubClusterUtil{},
helmUtil: stubHelmClient{ciliumError: someErr},
helmClient: stubHelmClient{ciliumError: someErr},
kubeconfigReader: &stubKubeconfigReader{
Kubeconfig: []byte("someKubeconfig"),
},
@ -255,7 +253,7 @@ func TestInitCluster(t *testing.T) {
},
"kubeadm init fails when setting up the kms": {
clusterUtil: stubClusterUtil{},
helmUtil: stubHelmClient{kmsError: someErr},
helmClient: stubHelmClient{servicesError: someErr},
kubeconfigReader: &stubKubeconfigReader{
Kubeconfig: []byte("someKubeconfig"),
},
@ -311,7 +309,7 @@ func TestInitCluster(t *testing.T) {
kube := KubeWrapper{
clusterUtil: &tc.clusterUtil,
helmUtil: &tc.helmUtil,
helmClient: &tc.helmClient,
providerMetadata: tc.providerMetadata,
cloudControllerManager: tc.CloudControllerManager,
cloudNodeManager: tc.CloudNodeManager,
@ -324,7 +322,7 @@ func TestInitCluster(t *testing.T) {
_, err := kube.InitCluster(
context.Background(), serviceAccountURI, string(tc.k8sVersion),
nil, nil, false, nil, true, helmClient.KMSConfig{MasterSecret: masterSecret}, nil, []byte("{}"), false, logger.NewTest(t),
nil, nil, false, nil, true, nil, []byte("{}"), false, logger.NewTest(t),
)
if tc.wantErr {
@ -682,14 +680,14 @@ func (s *stubKubeconfigReader) ReadKubeconfig() ([]byte, error) {
}
type stubHelmClient struct {
ciliumError error
kmsError error
ciliumError error
servicesError error
}
func (s *stubHelmClient) InstallCilium(ctx context.Context, kubectl k8sapi.Client, release helm.Release, in k8sapi.SetupPodNetworkInput) error {
return s.ciliumError
}
func (s *stubHelmClient) InstallKMS(ctx context.Context, release helm.Release, kmsConfig helmClient.KMSConfig) error {
return s.kmsError
func (s *stubHelmClient) InstallConstellationServices(ctx context.Context, release helm.Release) error {
return s.servicesError
}

View File

@ -9,13 +9,13 @@ package cmd
import "github.com/edgelesssys/constellation/v2/internal/cloud/cloudprovider"
type helmLoader interface {
Load(csp cloudprovider.Provider, conformanceMode bool) ([]byte, error)
Load(csp cloudprovider.Provider, conformanceMode bool, masterSecret []byte, salt []byte) ([]byte, error)
}
type stubHelmLoader struct {
loadErr error
}
func (d *stubHelmLoader) Load(csp cloudprovider.Provider, conformanceMode bool) ([]byte, error) {
func (d *stubHelmLoader) Load(csp cloudprovider.Provider, conformanceMode bool, masterSecret []byte, salt []byte) ([]byte, error) {
return nil, d.loadErr
}

View File

@ -121,16 +121,16 @@ func initialize(cmd *cobra.Command, newDialer func(validator *cloudcmd.Validator
return err
}
helmDeployments, err := helmLoader.Load(provider, flags.conformance)
if err != nil {
return fmt.Errorf("loading Helm charts: %w", err)
}
masterSecret, err := readOrGenerateMasterSecret(cmd.OutOrStdout(), fileHandler, flags.masterSecretPath)
if err != nil {
return fmt.Errorf("parsing or generating master secret from file %s: %w", flags.masterSecretPath, err)
}
helmDeployments, err := helmLoader.Load(provider, flags.conformance, masterSecret.Key, masterSecret.Salt)
if err != nil {
return fmt.Errorf("loading Helm charts: %w", err)
}
spinner.Start("Initializing cluster ", false)
req := &initproto.InitRequest{
MasterSecret: masterSecret.Key,

View File

@ -0,0 +1,9 @@
apiVersion: v2
name: constellation-services
description: A chart to deploy all microservices that are part of a valid constellation cluster
type: application
version: 2.2.0-pre
dependencies:
- name: kms
version: 2.2.0-pre

View File

@ -0,0 +1,23 @@
# Patterns to ignore when building packages.
# This supports shell glob matching, relative path matching, and
# negation (prefixed with !). Only one pattern per line.
.DS_Store
# Common VCS dirs
.git/
.gitignore
.bzr/
.bzrignore
.hg/
.hgignore
.svn/
# Common backup files
*.swp
*.bak
*.tmp
*.orig
*~
# Various IDEs
.project
.idea/
*.tmproj
.vscode/

View File

@ -2,5 +2,4 @@ apiVersion: v2
name: kms
description: A Helm chart to deploy the Constellation Key Management Service
type: application
version: 0.1.0
appVersion: "2.1.0"
version: 2.2.0-pre

View File

@ -18,8 +18,8 @@ spec:
spec:
containers:
- args:
- --port={{ .Values.kmsPort }}
image: {{ .Values.kmsImage }}
- --port={{ .Values.port }}
image: {{ .Values.image }}
name: kms
resources: {}
volumeMounts:

View File

@ -6,9 +6,9 @@ metadata:
spec:
ports:
- name: grpc
port: {{ .Values.kmsPort }}
port: {{ .Values.port }}
protocol: TCP
targetPort: {{ .Values.kmsPort }}
targetPort: {{ .Values.port }}
selector:
k8s-app: kms
type: ClusterIP

View File

@ -0,0 +1,30 @@
{
"$schema": "https://json-schema.org/draft-07/schema#",
"properties": {
"image": {
"description": "Container image to use for the spawned pods.",
"type": "string",
"examples": ["ghcr.io/edgelesssys/constellation/kms:latest"],
"pattern": "ghcr.io/edgelesssys/constellation/kms:*"
},
"masterSecret": {
"description": "Secret used to derive key material within the cluster",
"type": "string",
"examples": ["h1ydxM+1LKhL6kfj3XJnCYvTPnQGUgU0stk91ebEVqM="],
"minLength": 44
},
"salt": {
"description": "Salt for key derivation within the cluster",
"type": "string",
"examples": ["loC4hhWwFH5rHAKq5/EshSWk1jwkrf22VuHc2SGsWdc="],
"minLength": 44
}
},
"required": [
"image",
"salt",
"masterSecret"
],
"title": "Values",
"type": "object"
}

View File

@ -1,19 +1,13 @@
# Namespace to which KMS will be deployed.
namespace: "kube-system"
# Port on which the service will listen.
kmsPort: 9000
port: 9000
# Name of the ConfigMap that holds measurements and other info.
joinConfigCMName: join-config
# Path to which secrets/CMs are mounted.
serviceBasePath: /var/config
# Container image.
kmsImage: setFullImagePathHere
# Salt for key derivation.
salt: ""
# Name of the key within the respective secret that holds the salt.
saltKeyName: salt
# MasterSecret for the cluster.
masterSecret: ""
# Name of the secret that contains the master secret.
masterSecretName: constellation-mastersecret
# Name of the key within the respective secret that holds the master secret.

View File

@ -9,9 +9,11 @@ package helm
import (
"bytes"
"embed"
"encoding/base64"
"encoding/json"
"fmt"
"io/fs"
"os"
"path/filepath"
"strings"
@ -23,6 +25,7 @@ import (
"helm.sh/helm/pkg/ignore"
"helm.sh/helm/v3/pkg/chart"
"helm.sh/helm/v3/pkg/chart/loader"
"helm.sh/helm/v3/pkg/chartutil"
)
// Run `go generate` to deterministically create the patched Helm deployment for cilium
@ -33,17 +36,18 @@ var HelmFS embed.FS
type ChartLoader struct{}
func (i *ChartLoader) Load(csp cloudprovider.Provider, conformanceMode bool) ([]byte, error) {
func (i *ChartLoader) Load(csp cloudprovider.Provider, conformanceMode bool, masterSecret []byte, salt []byte) ([]byte, error) {
ciliumRelease, err := i.loadCilium(csp, conformanceMode)
if err != nil {
return nil, err
}
kmsRelease, err := i.loadKMS()
conServicesRelease, err := i.loadConstellationServices(masterSecret, salt)
if err != nil {
return nil, err
}
releases := helm.Releases{Cilium: ciliumRelease, KMS: kmsRelease}
releases := helm.Releases{Cilium: ciliumRelease, ConstellationServices: conServicesRelease}
rel, err := json.Marshal(releases)
if err != nil {
return nil, err
@ -54,8 +58,14 @@ func (i *ChartLoader) Load(csp cloudprovider.Provider, conformanceMode bool) ([]
func (i *ChartLoader) loadCilium(csp cloudprovider.Provider, conformanceMode bool) (helm.Release, error) {
chart, err := loadChartsDir(HelmFS, "charts/cilium")
if err != nil {
return helm.Release{}, err
return helm.Release{}, fmt.Errorf("loading cilium chart: %w", err)
}
chartRaw, err := i.marshalChart(chart)
if err != nil {
return helm.Release{}, fmt.Errorf("packaging chart: %w", err)
}
var ciliumVals map[string]interface{}
switch csp {
case cloudprovider.GCP:
@ -77,27 +87,54 @@ func (i *ChartLoader) loadCilium(csp cloudprovider.Provider, conformanceMode boo
}
return helm.Release{Chart: chart, Values: ciliumVals, ReleaseName: "cilium", Wait: true}, nil
return helm.Release{Chart: chartRaw, Values: ciliumVals, ReleaseName: "cilium", Wait: true}, nil
}
func (i *ChartLoader) loadKMS() (helm.Release, error) {
chart, err := loadChartsDir(HelmFS, "charts/edgeless/kms")
func (i *ChartLoader) loadConstellationServices(masterSecret []byte, salt []byte) (helm.Release, error) {
chart, err := loadChartsDir(HelmFS, "charts/edgeless/constellation-services")
if err != nil {
return helm.Release{}, err
}
kmsVals := map[string]interface{}{
"namespace": constants.ConstellationNamespace,
"kmsPort": constants.KMSPort,
"joinConfigCMName": constants.JoinConfigMap,
"serviceBasePath": constants.ServiceBasePath,
"kmsImage": versions.KmsImage,
"masterSecretName": constants.ConstellationMasterSecretStoreName,
"masterSecretKeyName": constants.ConstellationMasterSecretKey,
"saltKeyName": constants.ConstellationSaltKey,
"measurementsFilename": constants.MeasurementsFilename,
return helm.Release{}, fmt.Errorf("loading constellation-services chart: %w", err)
}
return helm.Release{Chart: chart, Values: kmsVals, ReleaseName: "kms", Wait: true}, nil
chartRaw, err := i.marshalChart(chart)
if err != nil {
return helm.Release{}, fmt.Errorf("packaging chart: %w", err)
}
vals := map[string]interface{}{
"kms": map[string]interface{}{
"namespace": constants.ConstellationNamespace,
"port": constants.KMSPort,
"joinConfigCMName": constants.JoinConfigMap,
"serviceBasePath": constants.ServiceBasePath,
"image": versions.KmsImage,
"masterSecretName": constants.ConstellationMasterSecretStoreName,
"masterSecretKeyName": constants.ConstellationMasterSecretKey,
"saltKeyName": constants.ConstellationSaltKey,
"measurementsFilename": constants.MeasurementsFilename,
"masterSecret": base64.StdEncoding.EncodeToString(masterSecret),
"salt": base64.StdEncoding.EncodeToString(salt),
},
}
return helm.Release{Chart: chartRaw, Values: vals, ReleaseName: "constellation-services", Wait: true}, nil
}
// marshalChart takes a Chart object, packages it to a temporary file and returns the content of that file.
// We currently need to take this approach of marshaling as dependencies are not marshaled correctly with json.Marshal.
// This stems from the fact that chart.Chart does not export the dependencies property.
// See: https://github.com/helm/helm/issues/11454
func (i *ChartLoader) marshalChart(chart *chart.Chart) ([]byte, error) {
path, err := chartutil.Save(chart, os.TempDir())
defer os.Remove(path)
if err != nil {
return nil, fmt.Errorf("packaging chart: %w", err)
}
chartRaw, err := os.ReadFile(path)
if err != nil {
return nil, fmt.Errorf("reading packaged chart: %w", err)
}
return chartRaw, nil
}
// taken from loader.LoadDir from the helm go module

View File

@ -0,0 +1,34 @@
/*
Copyright (c) Edgeless Systems GmbH
SPDX-License-Identifier: AGPL-3.0-only
*/
package helm
import (
"bytes"
"encoding/json"
"testing"
"github.com/edgelesssys/constellation/v2/internal/cloud/cloudprovider"
"github.com/edgelesssys/constellation/v2/internal/deploy/helm"
"github.com/stretchr/testify/assert"
"helm.sh/helm/v3/pkg/chart/loader"
)
func TestLoad(t *testing.T) {
assert := assert.New(t)
chartLoader := ChartLoader{}
release, err := chartLoader.Load(cloudprovider.GCP, true, []byte("secret"), []byte("salt"))
assert.NoError(err)
var helmReleases helm.Releases
err = json.Unmarshal(release, &helmReleases)
assert.NoError(err)
reader := bytes.NewReader(helmReleases.ConstellationServices.Chart)
chart, err := loader.LoadArchive(reader)
assert.NoError(err)
assert.NotNil(chart.Dependencies())
}

View File

@ -99,6 +99,7 @@ require (
github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e // indirect
github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect
github.com/cyberphone/json-canonicalization v0.0.0-20210303052042-6bc126869bf4 // indirect
github.com/cyphar/filepath-securejoin v0.2.3 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/docker/distribution v2.8.1+incompatible // indirect
github.com/docker/docker v20.10.19+incompatible // indirect
@ -165,8 +166,10 @@ require (
github.com/manifoldco/promptui v0.9.0 // indirect
github.com/mattn/go-runewidth v0.0.13 // indirect
github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect
github.com/mitchellh/copystructure v1.2.0 // indirect
github.com/mitchellh/go-homedir v1.1.0 // indirect
github.com/mitchellh/mapstructure v1.5.0 // indirect
github.com/mitchellh/reflectwalk v1.0.2 // indirect
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
github.com/modern-go/reflect2 v1.0.2 // indirect
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
@ -199,6 +202,9 @@ require (
github.com/titanous/rocacheck v0.0.0-20171023193734-afe73141d399 // indirect
github.com/transparency-dev/merkle v0.0.1 // indirect
github.com/xanzy/ssh-agent v0.3.0 // indirect
github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f // indirect
github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect
github.com/xeipuuv/gojsonschema v1.2.0 // indirect
github.com/zclconf/go-cty v1.11.0 // indirect
go.mongodb.org/mongo-driver v1.10.0 // indirect
go.opencensus.io v0.23.0 // indirect
@ -222,6 +228,7 @@ require (
helm.sh/helm v2.17.0+incompatible // indirect
helm.sh/helm/v3 v3.10.1 // indirect
k8s.io/api v0.25.3 // indirect
k8s.io/apiextensions-apiserver v0.25.3 // indirect
k8s.io/apimachinery v0.25.3 // indirect
k8s.io/client-go v0.25.3 // indirect
k8s.io/klog/v2 v2.80.1 // indirect

View File

@ -336,6 +336,8 @@ github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ3
github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
github.com/cyberphone/json-canonicalization v0.0.0-20210303052042-6bc126869bf4 h1:7AjYfmq7AmviXsuZjV5DcE7PuhJ4dWMi8gLllpLVDQY=
github.com/cyberphone/json-canonicalization v0.0.0-20210303052042-6bc126869bf4/go.mod h1:uzvlm1mxhHkdfqitSA92i7Se+S9ksOn3a3qmv/kyOCw=
github.com/cyphar/filepath-securejoin v0.2.3 h1:YX6ebbZCZP7VkM3scTTokDgBL2TY741X51MTk3ycuNI=
github.com/cyphar/filepath-securejoin v0.2.3/go.mod h1:aPGpWjXOXUn2NCNjFvBE6aRxGGx79pTxQpKOJNYHHl4=
github.com/daixiang0/gci v0.2.9/go.mod h1:+4dZ7TISfSmqfAGv59ePaHfNzgGtIkHAhhdKggP1JAc=
github.com/danieljoos/wincred v1.0.2/go.mod h1:SnuYRW9lp1oJrZX/dXJqr0cPK5gYXqx3EJbmjhLdK9U=
github.com/davecgh/go-spew v0.0.0-20161028175848-04cdfd42973b/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@ -991,6 +993,7 @@ github.com/mitchellh/cli v1.1.0/go.mod h1:xcISNoH86gajksDmfB23e/pu+B+GeFRMYmoHXx
github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db h1:62I3jR2EmQ4l5rM/4FEfDWcRD+abF5XlKShorW5LRoQ=
github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db/go.mod h1:l0dey0ia/Uv7NcFFVbCLtqEBQbrT4OCwCSKTEv6enCw=
github.com/mitchellh/copystructure v1.0.0/go.mod h1:SNtv71yrdKgLRyLFxmLdkAbkKEFWgYaq1OVrnRcwhnw=
github.com/mitchellh/copystructure v1.2.0 h1:vpKXTN4ewci03Vljg/q9QvCGUDttBOGBIa15WveJJGw=
github.com/mitchellh/copystructure v1.2.0/go.mod h1:qLl+cE2AmVv+CoeAwDPye/v+N2HKCj9FbZEVFJRxO9s=
github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
@ -1009,6 +1012,7 @@ github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyua
github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
github.com/mitchellh/reflectwalk v1.0.0/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw=
github.com/mitchellh/reflectwalk v1.0.1/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw=
github.com/mitchellh/reflectwalk v1.0.2 h1:G2LzWKi524PWgd3mLHV8Y5k7s6XUvT0Gef6zxSIeXaQ=
github.com/mitchellh/reflectwalk v1.0.2/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw=
github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6 h1:dcztxKSvZ4Id8iPpHERQBbIJfabdt4wUm5qy3wOL2Zc=
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
@ -1365,8 +1369,11 @@ github.com/xdg-go/scram v1.0.2/go.mod h1:1WAq6h33pAW+iRreB34OORO2Nf7qel3VV3fjBj+
github.com/xdg-go/scram v1.1.1/go.mod h1:RaEWvsqvNKKvBPvcKeFjrG2cJqOkHTiyTpzz23ni57g=
github.com/xdg-go/stringprep v1.0.2/go.mod h1:8F9zXuvzgwmyT5DUm4GUfZGDdT3W+LCvS6+da4O5kxM=
github.com/xdg-go/stringprep v1.0.3/go.mod h1:W3f5j4i+9rC0kuIEJL0ky1VpHXQU3ocBgklLGvcBnW8=
github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f h1:J9EGpcZtP0E/raorCMxlFGSTBrsSlaDGf3jU/qvAE2c=
github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU=
github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 h1:EzJWgHovont7NscjpAxXsDA8S8BMYve8Y5+7cuRE7R0=
github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ=
github.com/xeipuuv/gojsonschema v1.2.0 h1:LhYJRs+L4fBtjZUfuSZIKGeVu0QRy8e5Xi7D17UxZ74=
github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y=
github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8/go.mod h1:HUYIGzjTL3rfEspMxjDjgmT5uz5wzYJKVo23qUhYTos=
github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
@ -2225,6 +2232,8 @@ honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9
honnef.co/go/tools v0.2.1/go.mod h1:lPVVZ2BS5TfnjLyizF7o7hv7j9/L+8cZY2hLyjP9cGY=
k8s.io/api v0.25.3 h1:Q1v5UFfYe87vi5H7NU0p4RXC26PPMT8KOpr1TLQbCMQ=
k8s.io/api v0.25.3/go.mod h1:o42gKscFrEVjHdQnyRenACrMtbuJsVdP+WVjqejfzmI=
k8s.io/apiextensions-apiserver v0.25.3 h1:bfI4KS31w2f9WM1KLGwnwuVlW3RSRPuIsfNF/3HzR0k=
k8s.io/apiextensions-apiserver v0.25.3/go.mod h1:ZJqwpCkxIx9itilmZek7JgfUAM0dnTsA48I4krPqRmo=
k8s.io/apimachinery v0.25.3 h1:7o9ium4uyUOM76t6aunP0nZuex7gDf8VGwkR5RcJnQc=
k8s.io/apimachinery v0.25.3/go.mod h1:jaF9C/iPNM1FuLl7Zuy5b9v+n35HGSh6AQ4HYRkCqwo=
k8s.io/client-go v0.25.3 h1:oB4Dyl8d6UbfDHD8Bv8evKylzs3BXzzufLiO27xuPs0=

View File

@ -6,17 +6,15 @@ SPDX-License-Identifier: AGPL-3.0-only
package helm
import "helm.sh/helm/v3/pkg/chart"
// Release bundles all information necessary to create a helm release.
type Release struct {
Chart *chart.Chart
Chart []byte
Values map[string]interface{}
ReleaseName string
Wait bool
}
type Releases struct {
Cilium Release
KMS Release
Cilium Release
ConstellationServices Release
}