mirror of
https://github.com/edgelesssys/constellation.git
synced 2024-12-26 16:09:45 -05:00
577 lines
16 KiB
Go
577 lines
16 KiB
Go
/*
|
|
Copyright (c) Edgeless Systems GmbH
|
|
|
|
SPDX-License-Identifier: AGPL-3.0-only
|
|
*/
|
|
|
|
package helm
|
|
|
|
import (
|
|
"bytes"
|
|
"embed"
|
|
"encoding/base64"
|
|
"encoding/json"
|
|
"fmt"
|
|
"io/fs"
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
|
|
"github.com/edgelesssys/constellation/v2/internal/cloud/cloudprovider"
|
|
"github.com/edgelesssys/constellation/v2/internal/config"
|
|
"github.com/edgelesssys/constellation/v2/internal/constants"
|
|
"github.com/edgelesssys/constellation/v2/internal/deploy/helm"
|
|
"github.com/edgelesssys/constellation/v2/internal/versions"
|
|
"github.com/pkg/errors"
|
|
"helm.sh/helm/pkg/ignore"
|
|
"helm.sh/helm/v3/pkg/chart"
|
|
"helm.sh/helm/v3/pkg/chart/loader"
|
|
"helm.sh/helm/v3/pkg/chartutil"
|
|
)
|
|
|
|
// Run `go generate` to download (and patch) upstream helm charts.
|
|
//go:generate ./generateCilium.sh
|
|
//go:generate ./update-csi-charts.sh
|
|
//go:generate ./generateCertManager.sh
|
|
|
|
//go:embed all:charts/*
|
|
var helmFS embed.FS
|
|
|
|
const (
|
|
ciliumReleaseName = "cilium"
|
|
conServicesReleaseName = "constellation-services"
|
|
conOperatorsReleaseName = "constellation-operators"
|
|
certManagerReleaseName = "cert-manager"
|
|
|
|
conServicesPath = "charts/edgeless/constellation-services"
|
|
conOperatorsPath = "charts/edgeless/operators"
|
|
certManagerPath = "charts/cert-manager"
|
|
ciliumPath = "charts/cilium"
|
|
)
|
|
|
|
// ChartLoader loads embedded helm charts.
|
|
type ChartLoader struct {
|
|
joinServiceImage string
|
|
keyServiceImage string
|
|
ccmImage string
|
|
cnmImage string
|
|
autoscalerImage string
|
|
verificationServiceImage string
|
|
gcpGuestAgentImage string
|
|
konnectivityImage string
|
|
constellationOperatorImage string
|
|
nodeMaintenanceOperatorImage string
|
|
}
|
|
|
|
// NewLoader creates a new ChartLoader.
|
|
func NewLoader(csp cloudprovider.Provider, k8sVersion versions.ValidK8sVersion) *ChartLoader {
|
|
var ccmImage, cnmImage string
|
|
switch csp {
|
|
case cloudprovider.AWS:
|
|
ccmImage = versions.VersionConfigs[k8sVersion].CloudControllerManagerImageAWS
|
|
case cloudprovider.Azure:
|
|
ccmImage = versions.VersionConfigs[k8sVersion].CloudControllerManagerImageAzure
|
|
cnmImage = versions.VersionConfigs[k8sVersion].CloudNodeManagerImageAzure
|
|
case cloudprovider.GCP:
|
|
ccmImage = versions.VersionConfigs[k8sVersion].CloudControllerManagerImageGCP
|
|
}
|
|
|
|
return &ChartLoader{
|
|
joinServiceImage: versions.JoinImage,
|
|
keyServiceImage: versions.KeyServiceImage,
|
|
ccmImage: ccmImage,
|
|
cnmImage: cnmImage,
|
|
autoscalerImage: versions.VersionConfigs[k8sVersion].ClusterAutoscalerImage,
|
|
verificationServiceImage: versions.VerificationImage,
|
|
gcpGuestAgentImage: versions.GcpGuestImage,
|
|
konnectivityImage: versions.KonnectivityAgentImage,
|
|
constellationOperatorImage: versions.ConstellationOperatorImage,
|
|
nodeMaintenanceOperatorImage: versions.NodeMaintenanceOperatorImage,
|
|
}
|
|
}
|
|
|
|
// Load the embedded helm charts.
|
|
func (i *ChartLoader) Load(config *config.Config, conformanceMode bool, masterSecret, salt []byte) ([]byte, error) {
|
|
ciliumRelease, err := i.loadCilium(config.GetProvider(), conformanceMode)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("loading cilium: %w", err)
|
|
}
|
|
|
|
certManagerRelease, err := i.loadCertManager()
|
|
if err != nil {
|
|
return nil, fmt.Errorf("loading cilium: %w", err)
|
|
}
|
|
|
|
operatorRelease, err := i.loadOperators(config.GetProvider())
|
|
if err != nil {
|
|
return nil, fmt.Errorf("loading operators: %w", err)
|
|
}
|
|
|
|
conServicesRelease, err := i.loadConstellationServices(config, masterSecret, salt)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("loading constellation-services: %w", err)
|
|
}
|
|
releases := helm.Releases{Cilium: ciliumRelease, CertManager: certManagerRelease, Operators: operatorRelease, ConstellationServices: conServicesRelease}
|
|
|
|
rel, err := json.Marshal(releases)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return rel, nil
|
|
}
|
|
|
|
func (i *ChartLoader) loadCilium(csp cloudprovider.Provider, conformanceMode bool) (helm.Release, error) {
|
|
chart, err := loadChartsDir(helmFS, ciliumPath)
|
|
if err != nil {
|
|
return helm.Release{}, fmt.Errorf("loading cilium chart: %w", err)
|
|
}
|
|
values, err := i.loadCiliumValues(csp, conformanceMode)
|
|
if err != nil {
|
|
return helm.Release{}, err
|
|
}
|
|
|
|
chartRaw, err := i.marshalChart(chart)
|
|
if err != nil {
|
|
return helm.Release{}, fmt.Errorf("packaging cilium chart: %w", err)
|
|
}
|
|
|
|
return helm.Release{Chart: chartRaw, Values: values, ReleaseName: ciliumReleaseName, Wait: false}, nil
|
|
}
|
|
|
|
// loadCiliumHelper is used to separate the marshalling step from the loading step.
|
|
// This reduces the time unit tests take to execute.
|
|
func (i *ChartLoader) loadCiliumValues(csp cloudprovider.Provider, conformanceMode bool) (map[string]any, error) {
|
|
var values map[string]any
|
|
switch csp {
|
|
case cloudprovider.AWS:
|
|
values = awsVals
|
|
case cloudprovider.Azure:
|
|
values = azureVals
|
|
case cloudprovider.GCP:
|
|
values = gcpVals
|
|
case cloudprovider.QEMU:
|
|
values = qemuVals
|
|
default:
|
|
return nil, fmt.Errorf("unknown csp: %s", csp)
|
|
}
|
|
if conformanceMode {
|
|
values["kubeProxyReplacementHealthzBindAddr"] = ""
|
|
values["kubeProxyReplacement"] = "partial"
|
|
values["sessionAffinity"] = true
|
|
values["cni"] = map[string]any{
|
|
"chainingMode": "portmap",
|
|
}
|
|
|
|
}
|
|
return values, nil
|
|
}
|
|
|
|
func (i *ChartLoader) loadCertManager() (helm.Release, error) {
|
|
chart, err := loadChartsDir(helmFS, certManagerPath)
|
|
if err != nil {
|
|
return helm.Release{}, fmt.Errorf("loading cert-manager chart: %w", err)
|
|
}
|
|
values := i.loadCertManagerValues()
|
|
|
|
chartRaw, err := i.marshalChart(chart)
|
|
if err != nil {
|
|
return helm.Release{}, fmt.Errorf("packaging cert-manager chart: %w", err)
|
|
}
|
|
|
|
return helm.Release{Chart: chartRaw, Values: values, ReleaseName: certManagerReleaseName, Wait: false}, nil
|
|
}
|
|
|
|
// loadCertManagerHelper is used to separate the marshalling step from the loading step.
|
|
// This reduces the time unit tests take to execute.
|
|
func (i *ChartLoader) loadCertManagerValues() map[string]any {
|
|
return map[string]any{
|
|
"installCRDs": true,
|
|
"prometheus": map[string]any{
|
|
"enabled": false,
|
|
},
|
|
"tolerations": []map[string]any{
|
|
{
|
|
"key": "node-role.kubernetes.io/control-plane",
|
|
"effect": "NoSchedule",
|
|
"operator": "Exists",
|
|
},
|
|
{
|
|
"key": "node-role.kubernetes.io/master",
|
|
"effect": "NoSchedule",
|
|
"operator": "Exists",
|
|
},
|
|
},
|
|
"webhook": map[string]any{
|
|
"tolerations": []map[string]any{
|
|
{
|
|
"key": "node-role.kubernetes.io/control-plane",
|
|
"effect": "NoSchedule",
|
|
"operator": "Exists",
|
|
},
|
|
{
|
|
"key": "node-role.kubernetes.io/master",
|
|
"effect": "NoSchedule",
|
|
"operator": "Exists",
|
|
},
|
|
},
|
|
},
|
|
"cainjector": map[string]any{
|
|
"tolerations": []map[string]any{
|
|
{
|
|
"key": "node-role.kubernetes.io/control-plane",
|
|
"effect": "NoSchedule",
|
|
"operator": "Exists",
|
|
},
|
|
{
|
|
"key": "node-role.kubernetes.io/master",
|
|
"effect": "NoSchedule",
|
|
"operator": "Exists",
|
|
},
|
|
},
|
|
},
|
|
"startupapicheck": map[string]any{
|
|
"timeout": "5m",
|
|
"extraArgs": []string{
|
|
"--verbose",
|
|
},
|
|
"tolerations": []map[string]any{
|
|
{
|
|
"key": "node-role.kubernetes.io/control-plane",
|
|
"effect": "NoSchedule",
|
|
"operator": "Exists",
|
|
},
|
|
{
|
|
"key": "node-role.kubernetes.io/master",
|
|
"effect": "NoSchedule",
|
|
"operator": "Exists",
|
|
},
|
|
},
|
|
},
|
|
}
|
|
}
|
|
|
|
func (i *ChartLoader) loadOperators(csp cloudprovider.Provider) (helm.Release, error) {
|
|
chart, err := loadChartsDir(helmFS, conOperatorsPath)
|
|
if err != nil {
|
|
return helm.Release{}, fmt.Errorf("loading operators chart: %w", err)
|
|
}
|
|
values, err := i.loadOperatorsValues(csp)
|
|
if err != nil {
|
|
return helm.Release{}, err
|
|
}
|
|
|
|
chartRaw, err := i.marshalChart(chart)
|
|
if err != nil {
|
|
return helm.Release{}, fmt.Errorf("packaging operators chart: %w", err)
|
|
}
|
|
|
|
return helm.Release{Chart: chartRaw, Values: values, ReleaseName: conOperatorsReleaseName, Wait: false}, nil
|
|
}
|
|
|
|
// loadOperatorsHelper is used to separate the marshalling step from the loading step.
|
|
// This reduces the time unit tests take to execute.
|
|
func (i *ChartLoader) loadOperatorsValues(csp cloudprovider.Provider) (map[string]any, error) {
|
|
values := map[string]any{
|
|
"constellation-operator": map[string]any{
|
|
"controllerManager": map[string]any{
|
|
"manager": map[string]any{
|
|
"image": i.constellationOperatorImage,
|
|
},
|
|
},
|
|
},
|
|
"node-maintenance-operator": map[string]any{
|
|
"controllerManager": map[string]any{
|
|
"manager": map[string]any{
|
|
"image": i.nodeMaintenanceOperatorImage,
|
|
},
|
|
},
|
|
},
|
|
}
|
|
switch csp {
|
|
case cloudprovider.Azure:
|
|
conOpVals, ok := values["constellation-operator"].(map[string]any)
|
|
if !ok {
|
|
return nil, errors.New("invalid constellation-operator values")
|
|
}
|
|
conOpVals["csp"] = "Azure"
|
|
|
|
values["tags"] = map[string]any{
|
|
"Azure": true,
|
|
}
|
|
case cloudprovider.GCP:
|
|
conOpVals, ok := values["constellation-operator"].(map[string]any)
|
|
if !ok {
|
|
return nil, errors.New("invalid constellation-operator values")
|
|
}
|
|
conOpVals["csp"] = "GCP"
|
|
|
|
values["tags"] = map[string]any{
|
|
"GCP": true,
|
|
}
|
|
case cloudprovider.QEMU:
|
|
conOpVals, ok := values["constellation-operator"].(map[string]any)
|
|
if !ok {
|
|
return nil, errors.New("invalid constellation-operator values")
|
|
}
|
|
conOpVals["csp"] = "QEMU"
|
|
|
|
values["tags"] = map[string]any{
|
|
"QEMU": true,
|
|
}
|
|
case cloudprovider.AWS:
|
|
conOpVals, ok := values["constellation-operator"].(map[string]any)
|
|
if !ok {
|
|
return nil, errors.New("invalid constellation-operator values")
|
|
}
|
|
conOpVals["csp"] = "AWS"
|
|
|
|
values["tags"] = map[string]any{
|
|
"AWS": true,
|
|
}
|
|
}
|
|
|
|
return values, nil
|
|
}
|
|
|
|
// loadConstellationServices loads the constellation-services chart from the embed.FS,
|
|
// marshals it into a helm-package .tgz and sets the values that can be set in the CLI.
|
|
func (i *ChartLoader) loadConstellationServices(config *config.Config, masterSecret, salt []byte) (helm.Release, error) {
|
|
chart, err := loadChartsDir(helmFS, conServicesPath)
|
|
if err != nil {
|
|
return helm.Release{}, fmt.Errorf("loading constellation-services chart: %w", err)
|
|
}
|
|
values, err := i.loadConstellationServicesValues(config, masterSecret, salt)
|
|
if err != nil {
|
|
return helm.Release{}, err
|
|
}
|
|
|
|
chartRaw, err := i.marshalChart(chart)
|
|
if err != nil {
|
|
return helm.Release{}, fmt.Errorf("packaging constellation-services chart: %w", err)
|
|
}
|
|
|
|
return helm.Release{Chart: chartRaw, Values: values, ReleaseName: conServicesReleaseName, Wait: false}, nil
|
|
}
|
|
|
|
// loadConstellationServicesHelper is used to separate the marshalling step from the loading step.
|
|
// This reduces the time unit tests take to execute.
|
|
func (i *ChartLoader) loadConstellationServicesValues(config *config.Config, masterSecret, salt []byte) (map[string]any, error) {
|
|
csp := config.GetProvider()
|
|
values := map[string]any{
|
|
"global": map[string]any{
|
|
"keyServicePort": constants.KeyServicePort,
|
|
"keyServiceNamespace": "", // empty namespace means we use the release namespace
|
|
"serviceBasePath": constants.ServiceBasePath,
|
|
"joinConfigCMName": constants.JoinConfigMap,
|
|
"internalCMName": constants.InternalConfigMap,
|
|
},
|
|
"key-service": map[string]any{
|
|
"image": i.keyServiceImage,
|
|
"masterSecret": base64.StdEncoding.EncodeToString(masterSecret),
|
|
"salt": base64.StdEncoding.EncodeToString(salt),
|
|
"saltKeyName": constants.ConstellationSaltKey,
|
|
"masterSecretKeyName": constants.ConstellationMasterSecretKey,
|
|
"masterSecretName": constants.ConstellationMasterSecretStoreName,
|
|
"measurementsFilename": constants.MeasurementsFilename,
|
|
},
|
|
"join-service": map[string]any{
|
|
"csp": csp.String(),
|
|
"image": i.joinServiceImage,
|
|
},
|
|
"ccm": map[string]any{
|
|
"csp": csp.String(),
|
|
},
|
|
"autoscaler": map[string]any{
|
|
"csp": csp.String(),
|
|
"image": i.autoscalerImage,
|
|
},
|
|
"verification-service": map[string]any{
|
|
"csp": csp.String(),
|
|
"image": i.verificationServiceImage,
|
|
},
|
|
"gcp-guest-agent": map[string]any{
|
|
"image": i.gcpGuestAgentImage,
|
|
},
|
|
"konnectivity": map[string]any{
|
|
"image": i.konnectivityImage,
|
|
},
|
|
}
|
|
|
|
switch csp {
|
|
case cloudprovider.Azure:
|
|
joinServiceVals, ok := values["join-service"].(map[string]any)
|
|
if !ok {
|
|
return nil, errors.New("invalid join-service values")
|
|
}
|
|
joinServiceVals["enforceIdKeyDigest"] = config.EnforcesIDKeyDigest()
|
|
|
|
marshalledDigests, err := json.Marshal(config.IDKeyDigests())
|
|
if err != nil {
|
|
return nil, fmt.Errorf("marshalling id key digests: %w", err)
|
|
}
|
|
joinServiceVals["idkeydigests"] = string(marshalledDigests)
|
|
|
|
ccmVals, ok := values["ccm"].(map[string]any)
|
|
if !ok {
|
|
return nil, errors.New("invalid ccm values")
|
|
}
|
|
ccmVals["Azure"] = map[string]any{
|
|
"image": i.ccmImage,
|
|
}
|
|
|
|
values["cnm"] = map[string]any{
|
|
"image": i.cnmImage,
|
|
}
|
|
|
|
values["azure"] = map[string]any{
|
|
"deployCSIDriver": config.DeployCSIDriver(),
|
|
}
|
|
|
|
values["tags"] = map[string]any{
|
|
"Azure": true,
|
|
}
|
|
|
|
case cloudprovider.GCP:
|
|
ccmVals, ok := values["ccm"].(map[string]any)
|
|
if !ok {
|
|
return nil, errors.New("invalid ccm values")
|
|
}
|
|
ccmVals["GCP"] = map[string]any{
|
|
"image": i.ccmImage,
|
|
}
|
|
|
|
values["gcp"] = map[string]any{
|
|
"deployCSIDriver": config.DeployCSIDriver(),
|
|
}
|
|
|
|
values["tags"] = map[string]any{
|
|
"GCP": true,
|
|
}
|
|
|
|
case cloudprovider.QEMU:
|
|
values["tags"] = map[string]interface{}{
|
|
"QEMU": true,
|
|
}
|
|
|
|
case cloudprovider.AWS:
|
|
ccmVals, ok := values["ccm"].(map[string]any)
|
|
if !ok {
|
|
return nil, errors.New("invalid ccm values")
|
|
}
|
|
ccmVals["AWS"] = map[string]any{
|
|
"image": i.ccmImage,
|
|
}
|
|
|
|
values["tags"] = map[string]any{
|
|
"AWS": true,
|
|
}
|
|
}
|
|
return values, nil
|
|
}
|
|
|
|
// marshalChart takes a Chart object, packages it to a temporary file and returns the content of that file.
|
|
// We currently need to take this approach of marshaling as dependencies are not marshaled correctly with json.Marshal.
|
|
// This stems from the fact that chart.Chart does not export the dependencies property.
|
|
func (i *ChartLoader) marshalChart(chart *chart.Chart) ([]byte, error) {
|
|
// A separate tmpdir path is necessary since during unit testing multiple go routines are accessing the same path, possibly deleting files for other routines.
|
|
tmpDirPath, err := os.MkdirTemp("", "*")
|
|
defer os.Remove(tmpDirPath)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("creating tmp dir: %w", err)
|
|
}
|
|
|
|
path, err := chartutil.Save(chart, tmpDirPath)
|
|
defer os.Remove(path)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("chartutil save: %w", err)
|
|
}
|
|
chartRaw, err := os.ReadFile(path)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("reading packaged chart: %w", err)
|
|
}
|
|
return chartRaw, nil
|
|
}
|
|
|
|
// taken from loader.LoadDir from the helm go module
|
|
// loadChartsDir loads from a directory.
|
|
//
|
|
// This loads charts only from directories.
|
|
func loadChartsDir(efs embed.FS, dir string) (*chart.Chart, error) {
|
|
utf8bom := []byte{0xEF, 0xBB, 0xBF}
|
|
// Just used for errors.
|
|
c := &chart.Chart{}
|
|
|
|
rules := ignore.Empty()
|
|
ifile, err := efs.ReadFile(filepath.Join(dir, ignore.HelmIgnore))
|
|
if err == nil {
|
|
r, err := ignore.Parse(bytes.NewReader(ifile))
|
|
if err != nil {
|
|
return c, err
|
|
}
|
|
rules = r
|
|
}
|
|
rules.AddDefaults()
|
|
|
|
files := []*loader.BufferedFile{}
|
|
|
|
walk := func(path string, d fs.DirEntry, err error) error {
|
|
n := strings.TrimPrefix(path, dir)
|
|
if n == "" {
|
|
// No need to process top level. Avoid bug with helmignore .* matching
|
|
// empty names. See issue https://github.com/kubernetes/helm/issues/1776.
|
|
return nil
|
|
}
|
|
|
|
// Normalize to / since it will also work on Windows
|
|
n = filepath.ToSlash(n)
|
|
|
|
// Check input err
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
fi, err := d.Info()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
if d.IsDir() {
|
|
// Directory-based ignore rules should involve skipping the entire
|
|
// contents of that directory.
|
|
if rules.Ignore(n, fi) {
|
|
return filepath.SkipDir
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// If a .helmignore file matches, skip this file.
|
|
if rules.Ignore(n, fi) {
|
|
return nil
|
|
}
|
|
|
|
// Irregular files include devices, sockets, and other uses of files that
|
|
// are not regular files. In Go they have a file mode type bit set.
|
|
// See https://golang.org/pkg/os/#FileMode for examples.
|
|
if !fi.Mode().IsRegular() {
|
|
return fmt.Errorf("cannot load irregular file %s as it has file mode type bits set", path)
|
|
}
|
|
|
|
data, err := efs.ReadFile(path)
|
|
if err != nil {
|
|
return errors.Wrapf(err, "error reading %s", n)
|
|
}
|
|
|
|
data = bytes.TrimPrefix(data, utf8bom)
|
|
n = strings.TrimPrefix(n, "/")
|
|
|
|
files = append(files, &loader.BufferedFile{Name: n, Data: data})
|
|
return nil
|
|
}
|
|
|
|
if err := fs.WalkDir(efs, dir, walk); err != nil {
|
|
return c, err
|
|
}
|
|
|
|
return loader.LoadFiles(files)
|
|
}
|