constellation/cli/internal/helm/loader.go
Otto Bittner cac43a1dd0 ci: add e2e-upgrade test
The test is implemented as a go test.
It can be executed as a bazel target.
The general workflow is to setup a cluster,
point the test to the workspace in which to
find the kubeconfig and the constellation config
and specify a target image, k8s and
service version. The test will succeed
if it detects all target versions in the cluster
within the configured timeout.
The CI automates the above steps.
A separate workflow is introduced as there
are multiple input fields to the test.
Adding all of these to the manual e2e test
seemed confusing.

Co-authored-by: Fabian Kammel <fk@edgeless.systems>
2023-03-23 14:57:38 +01:00

645 lines
19 KiB
Go

/*
Copyright (c) Edgeless Systems GmbH
SPDX-License-Identifier: AGPL-3.0-only
*/
package helm
import (
"bytes"
"embed"
"encoding/base64"
"encoding/json"
"fmt"
"io/fs"
"os"
"path/filepath"
"strings"
"github.com/edgelesssys/constellation/v2/internal/attestation/idkeydigest"
"github.com/edgelesssys/constellation/v2/internal/cloud/cloudprovider"
"github.com/edgelesssys/constellation/v2/internal/compatibility"
"github.com/edgelesssys/constellation/v2/internal/config"
"github.com/edgelesssys/constellation/v2/internal/constants"
"github.com/edgelesssys/constellation/v2/internal/deploy/helm"
"github.com/edgelesssys/constellation/v2/internal/versions"
"github.com/pkg/errors"
"helm.sh/helm/pkg/ignore"
"helm.sh/helm/v3/pkg/chart"
"helm.sh/helm/v3/pkg/chart/loader"
"helm.sh/helm/v3/pkg/chartutil"
)
// Run `go generate` to download (and patch) upstream helm charts.
//go:generate ./generateCilium.sh
//go:generate ./update-csi-charts.sh
//go:generate ./generateCertManager.sh
//go:embed all:charts/*
var helmFS embed.FS
type chartInfo struct {
releaseName string
chartName string
path string
}
var (
ciliumInfo = chartInfo{releaseName: "cilium", chartName: "cilium", path: "charts/cilium"}
certManagerInfo = chartInfo{releaseName: "cert-manager", chartName: "cert-manager", path: "charts/cert-manager"}
constellationOperatorsInfo = chartInfo{releaseName: "constellation-operators", chartName: "constellation-operators", path: "charts/edgeless/operators"}
constellationServicesInfo = chartInfo{releaseName: "constellation-services", chartName: "constellation-services", path: "charts/edgeless/constellation-services"}
)
// ChartLoader loads embedded helm charts.
type ChartLoader struct {
csp cloudprovider.Provider
joinServiceImage string
keyServiceImage string
ccmImage string
cnmImage string
autoscalerImage string
verificationServiceImage string
gcpGuestAgentImage string
konnectivityImage string
constellationOperatorImage string
nodeMaintenanceOperatorImage string
}
// NewLoader creates a new ChartLoader.
func NewLoader(csp cloudprovider.Provider, k8sVersion versions.ValidK8sVersion) *ChartLoader {
var ccmImage, cnmImage string
switch csp {
case cloudprovider.AWS:
ccmImage = versions.VersionConfigs[k8sVersion].CloudControllerManagerImageAWS
case cloudprovider.Azure:
ccmImage = versions.VersionConfigs[k8sVersion].CloudControllerManagerImageAzure
cnmImage = versions.VersionConfigs[k8sVersion].CloudNodeManagerImageAzure
case cloudprovider.GCP:
ccmImage = versions.VersionConfigs[k8sVersion].CloudControllerManagerImageGCP
case cloudprovider.OpenStack:
ccmImage = versions.VersionConfigs[k8sVersion].CloudControllerManagerImageOpenStack
}
return &ChartLoader{
csp: csp,
joinServiceImage: versions.JoinImage,
keyServiceImage: versions.KeyServiceImage,
ccmImage: ccmImage,
cnmImage: cnmImage,
autoscalerImage: versions.VersionConfigs[k8sVersion].ClusterAutoscalerImage,
verificationServiceImage: versions.VerificationImage,
gcpGuestAgentImage: versions.GcpGuestImage,
konnectivityImage: versions.KonnectivityAgentImage,
constellationOperatorImage: versions.ConstellationOperatorImage,
nodeMaintenanceOperatorImage: versions.NodeMaintenanceOperatorImage,
}
}
// AvailableServiceVersions returns the chart version number of the bundled service versions.
func AvailableServiceVersions() (string, error) {
servicesChart, err := loadChartsDir(helmFS, constellationServicesInfo.path)
if err != nil {
return "", fmt.Errorf("loading constellation-services chart: %w", err)
}
return compatibility.EnsurePrefixV(servicesChart.Metadata.Version), nil
}
// Load the embedded helm charts.
func (i *ChartLoader) Load(config *config.Config, conformanceMode bool, masterSecret, salt []byte, maaURL string) ([]byte, error) {
ciliumRelease, err := i.loadRelease(ciliumInfo)
if err != nil {
return nil, fmt.Errorf("loading cilium: %w", err)
}
extendCiliumValues(ciliumRelease.Values, conformanceMode)
certManagerRelease, err := i.loadRelease(certManagerInfo)
if err != nil {
return nil, fmt.Errorf("loading cert-manager: %w", err)
}
operatorRelease, err := i.loadRelease(constellationOperatorsInfo)
if err != nil {
return nil, fmt.Errorf("loading operators: %w", err)
}
conServicesRelease, err := i.loadRelease(constellationServicesInfo)
if err != nil {
return nil, fmt.Errorf("loading constellation-services: %w", err)
}
if err := extendConstellationServicesValues(conServicesRelease.Values, config, masterSecret, salt, maaURL); err != nil {
return nil, fmt.Errorf("extending constellation-services values: %w", err)
}
releases := helm.Releases{Cilium: ciliumRelease, CertManager: certManagerRelease, Operators: operatorRelease, ConstellationServices: conServicesRelease}
rel, err := json.Marshal(releases)
if err != nil {
return nil, err
}
return rel, nil
}
// loadRelease loads the embedded chart and values depending on the given info argument.
func (i *ChartLoader) loadRelease(info chartInfo) (helm.Release, error) {
chart, err := loadChartsDir(helmFS, info.path)
if err != nil {
return helm.Release{}, fmt.Errorf("loading %s chart: %w", info.releaseName, err)
}
var values map[string]any
switch info.releaseName {
case ciliumInfo.releaseName:
values, err = i.loadCiliumValues()
case certManagerInfo.releaseName:
values = i.loadCertManagerValues()
case constellationOperatorsInfo.releaseName:
updateVersions(chart, compatibility.EnsurePrefixV(constants.VersionInfo()))
values, err = i.loadOperatorsValues()
case constellationServicesInfo.releaseName:
updateVersions(chart, compatibility.EnsurePrefixV(constants.VersionInfo()))
values, err = i.loadConstellationServicesValues()
}
if err != nil {
return helm.Release{}, fmt.Errorf("loading %s values: %w", info.releaseName, err)
}
chartRaw, err := i.marshalChart(chart)
if err != nil {
return helm.Release{}, fmt.Errorf("packaging %s chart: %w", info.releaseName, err)
}
return helm.Release{Chart: chartRaw, Values: values, ReleaseName: info.releaseName, Wait: false}, nil
}
// loadCiliumValues is used to separate the marshalling step from the loading step.
// This reduces the time unit tests take to execute.
func (i *ChartLoader) loadCiliumValues() (map[string]any, error) {
var values map[string]any
switch i.csp {
case cloudprovider.AWS:
values = awsVals
case cloudprovider.Azure:
values = azureVals
case cloudprovider.GCP:
values = gcpVals
case cloudprovider.OpenStack:
values = openStackVals
case cloudprovider.QEMU:
values = qemuVals
default:
return nil, fmt.Errorf("unknown csp: %s", i.csp)
}
return values, nil
}
// extendCiliumValues extends the given values map by some values depending on user input.
// This extra step of separating the application of user input is necessary since service upgrades should
// reuse user input from the init step. However, we can't rely on reuse-values, because
// during upgrades we all values need to be set locally as they might have changed.
// Also, the charts are not rendered correctly without all of these values.
func extendCiliumValues(in map[string]any, conformanceMode bool) {
if conformanceMode {
in["kubeProxyReplacementHealthzBindAddr"] = ""
in["kubeProxyReplacement"] = "partial"
in["sessionAffinity"] = true
in["cni"] = map[string]any{
"chainingMode": "portmap",
}
}
}
// loadCertManagerHelper is used to separate the marshalling step from the loading step.
// This reduces the time unit tests take to execute.
func (i *ChartLoader) loadCertManagerValues() map[string]any {
return map[string]any{
"installCRDs": true,
"prometheus": map[string]any{
"enabled": false,
},
"tolerations": []map[string]any{
{
"key": "node-role.kubernetes.io/control-plane",
"effect": "NoSchedule",
"operator": "Exists",
},
{
"key": "node-role.kubernetes.io/master",
"effect": "NoSchedule",
"operator": "Exists",
},
},
"webhook": map[string]any{
"tolerations": []map[string]any{
{
"key": "node-role.kubernetes.io/control-plane",
"effect": "NoSchedule",
"operator": "Exists",
},
{
"key": "node-role.kubernetes.io/master",
"effect": "NoSchedule",
"operator": "Exists",
},
},
},
"cainjector": map[string]any{
"tolerations": []map[string]any{
{
"key": "node-role.kubernetes.io/control-plane",
"effect": "NoSchedule",
"operator": "Exists",
},
{
"key": "node-role.kubernetes.io/master",
"effect": "NoSchedule",
"operator": "Exists",
},
},
},
"startupapicheck": map[string]any{
"timeout": "5m",
"extraArgs": []string{
"--verbose",
},
"tolerations": []map[string]any{
{
"key": "node-role.kubernetes.io/control-plane",
"effect": "NoSchedule",
"operator": "Exists",
},
{
"key": "node-role.kubernetes.io/master",
"effect": "NoSchedule",
"operator": "Exists",
},
},
},
}
}
// loadOperatorsHelper is used to separate the marshalling step from the loading step.
// This reduces the time unit tests take to execute.
func (i *ChartLoader) loadOperatorsValues() (map[string]any, error) {
values := map[string]any{
"constellation-operator": map[string]any{
"controllerManager": map[string]any{
"manager": map[string]any{
"image": i.constellationOperatorImage,
},
},
},
"node-maintenance-operator": map[string]any{
"controllerManager": map[string]any{
"manager": map[string]any{
"image": i.nodeMaintenanceOperatorImage,
},
},
},
}
switch i.csp {
case cloudprovider.AWS:
conOpVals, ok := values["constellation-operator"].(map[string]any)
if !ok {
return nil, errors.New("invalid constellation-operator values")
}
conOpVals["csp"] = "AWS"
values["tags"] = map[string]any{
"AWS": true,
}
case cloudprovider.Azure:
conOpVals, ok := values["constellation-operator"].(map[string]any)
if !ok {
return nil, errors.New("invalid constellation-operator values")
}
conOpVals["csp"] = "Azure"
values["tags"] = map[string]any{
"Azure": true,
}
case cloudprovider.GCP:
conOpVals, ok := values["constellation-operator"].(map[string]any)
if !ok {
return nil, errors.New("invalid constellation-operator values")
}
conOpVals["csp"] = "GCP"
values["tags"] = map[string]any{
"GCP": true,
}
case cloudprovider.OpenStack:
conOpVals, ok := values["constellation-operator"].(map[string]any)
if !ok {
return nil, errors.New("invalid constellation-operator values")
}
conOpVals["csp"] = "OpenStack"
values["tags"] = map[string]any{
"OpenStack": true,
}
case cloudprovider.QEMU:
conOpVals, ok := values["constellation-operator"].(map[string]any)
if !ok {
return nil, errors.New("invalid constellation-operator values")
}
conOpVals["csp"] = "QEMU"
values["tags"] = map[string]any{
"QEMU": true,
}
}
return values, nil
}
// loadConstellationServicesHelper is used to separate the marshalling step from the loading step.
// This reduces the time unit tests take to execute.
func (i *ChartLoader) loadConstellationServicesValues() (map[string]any, error) {
values := map[string]any{
"global": map[string]any{
"keyServicePort": constants.KeyServicePort,
"keyServiceNamespace": "", // empty namespace means we use the release namespace
"serviceBasePath": constants.ServiceBasePath,
"joinConfigCMName": constants.JoinConfigMap,
"internalCMName": constants.InternalConfigMap,
},
"key-service": map[string]any{
"image": i.keyServiceImage,
"saltKeyName": constants.ConstellationSaltKey,
"masterSecretKeyName": constants.ConstellationMasterSecretKey,
"masterSecretName": constants.ConstellationMasterSecretStoreName,
"measurementsFilename": constants.MeasurementsFilename,
},
"join-service": map[string]any{
"csp": i.csp.String(),
"image": i.joinServiceImage,
},
"ccm": map[string]any{
"csp": i.csp.String(),
},
"autoscaler": map[string]any{
"csp": i.csp.String(),
"image": i.autoscalerImage,
},
"verification-service": map[string]any{
"image": i.verificationServiceImage,
},
"gcp-guest-agent": map[string]any{
"image": i.gcpGuestAgentImage,
},
"konnectivity": map[string]any{
"image": i.konnectivityImage,
},
}
switch i.csp {
case cloudprovider.AWS:
ccmVals, ok := values["ccm"].(map[string]any)
if !ok {
return nil, errors.New("invalid ccm values")
}
ccmVals["AWS"] = map[string]any{
"image": i.ccmImage,
}
values["tags"] = map[string]any{
"AWS": true,
}
case cloudprovider.Azure:
ccmVals, ok := values["ccm"].(map[string]any)
if !ok {
return nil, errors.New("invalid ccm values")
}
ccmVals["Azure"] = map[string]any{
"image": i.ccmImage,
}
values["cnm"] = map[string]any{
"image": i.cnmImage,
}
values["tags"] = map[string]any{
"Azure": true,
}
case cloudprovider.GCP:
ccmVals, ok := values["ccm"].(map[string]any)
if !ok {
return nil, errors.New("invalid ccm values")
}
ccmVals["GCP"] = map[string]any{
"image": i.ccmImage,
}
values["tags"] = map[string]any{
"GCP": true,
}
case cloudprovider.OpenStack:
ccmVals, ok := values["ccm"].(map[string]any)
if !ok {
return nil, errors.New("invalid ccm values")
}
ccmVals["OpenStack"] = map[string]any{
"image": i.ccmImage,
}
values["tags"] = map[string]any{
"OpenStack": true,
}
case cloudprovider.QEMU:
values["tags"] = map[string]any{
"QEMU": true,
}
}
return values, nil
}
// extendConstellationServicesValues extends the given values map by some values depending on user input.
// Values set inside this function are only applied during init, not during upgrade.
func extendConstellationServicesValues(
in map[string]any, config *config.Config, masterSecret, salt []byte, maaURL string,
) error {
keyServiceValues, ok := in["key-service"].(map[string]any)
if !ok {
return errors.New("missing 'key-service' key")
}
keyServiceValues["masterSecret"] = base64.StdEncoding.EncodeToString(masterSecret)
keyServiceValues["salt"] = base64.StdEncoding.EncodeToString(salt)
joinServiceVals, ok := in["join-service"].(map[string]any)
if !ok {
return errors.New("invalid join-service values")
}
joinServiceVals["attestationVariant"] = config.AttestationVariant
verifyServiceVals, ok := in["verification-service"].(map[string]any)
if !ok {
return errors.New("invalid verification-service values")
}
verifyServiceVals["attestationVariant"] = config.AttestationVariant
csp := config.GetProvider()
switch csp {
case cloudprovider.Azure:
joinServiceVals, ok := in["join-service"].(map[string]any)
if !ok {
return errors.New("invalid join-service values")
}
idKeyCfg := idkeydigest.Config{
IDKeyDigests: config.IDKeyDigests(),
EnforcementPolicy: config.IDKeyDigestPolicy(),
MAAURL: maaURL,
}
marshalledCfg, err := json.Marshal(idKeyCfg)
if err != nil {
return fmt.Errorf("marshalling id key digest config: %w", err)
}
joinServiceVals["idKeyConfig"] = string(marshalledCfg)
in["azure"] = map[string]any{
"deployCSIDriver": config.DeployCSIDriver(),
}
case cloudprovider.GCP:
in["gcp"] = map[string]any{
"deployCSIDriver": config.DeployCSIDriver(),
}
}
return nil
}
// updateVersions changes all versions of direct dependencies that are set to "0.0.0" to newVersion.
func updateVersions(chart *chart.Chart, newVersion string) {
chart.Metadata.Version = newVersion
selectedDeps := chart.Metadata.Dependencies
for i := range selectedDeps {
if selectedDeps[i].Version == "0.0.0" {
selectedDeps[i].Version = newVersion
}
}
deps := chart.Dependencies()
for i := range deps {
if deps[i].Metadata.Version == "0.0.0" {
deps[i].Metadata.Version = newVersion
}
}
}
// marshalChart takes a Chart object, packages it to a temporary file and returns the content of that file.
// We currently need to take this approach of marshaling as dependencies are not marshaled correctly with json.Marshal.
// This stems from the fact that chart.Chart does not export the dependencies property.
func (i *ChartLoader) marshalChart(chart *chart.Chart) ([]byte, error) {
// A separate tmpdir path is necessary since during unit testing multiple go routines are accessing the same path, possibly deleting files for other routines.
tmpDirPath, err := os.MkdirTemp("", "*")
defer os.Remove(tmpDirPath)
if err != nil {
return nil, fmt.Errorf("creating tmp dir: %w", err)
}
path, err := chartutil.Save(chart, tmpDirPath)
defer os.Remove(path)
if err != nil {
return nil, fmt.Errorf("chartutil save: %w", err)
}
chartRaw, err := os.ReadFile(path)
if err != nil {
return nil, fmt.Errorf("reading packaged chart: %w", err)
}
return chartRaw, nil
}
// taken from loader.LoadDir from the helm go module
// loadChartsDir loads from a directory.
//
// This loads charts only from directories.
func loadChartsDir(efs embed.FS, dir string) (*chart.Chart, error) {
utf8bom := []byte{0xEF, 0xBB, 0xBF}
// Just used for errors.
c := &chart.Chart{}
rules := ignore.Empty()
ifile, err := efs.ReadFile(filepath.Join(dir, ignore.HelmIgnore))
if err == nil {
r, err := ignore.Parse(bytes.NewReader(ifile))
if err != nil {
return c, err
}
rules = r
}
rules.AddDefaults()
files := []*loader.BufferedFile{}
walk := func(path string, d fs.DirEntry, err error) error {
n := strings.TrimPrefix(path, dir)
if n == "" {
// No need to process top level. Avoid bug with helmignore .* matching
// empty names. See issue https://github.com/kubernetes/helm/issues/1776.
return nil
}
// Normalize to / since it will also work on Windows
n = filepath.ToSlash(n)
// Check input err
if err != nil {
return err
}
fi, err := d.Info()
if err != nil {
return err
}
if d.IsDir() {
// Directory-based ignore rules should involve skipping the entire
// contents of that directory.
if rules.Ignore(n, fi) {
return filepath.SkipDir
}
return nil
}
// If a .helmignore file matches, skip this file.
if rules.Ignore(n, fi) {
return nil
}
// Irregular files include devices, sockets, and other uses of files that
// are not regular files. In Go they have a file mode type bit set.
// See https://golang.org/pkg/os/#FileMode for examples.
if !fi.Mode().IsRegular() {
return fmt.Errorf("cannot load irregular file %s as it has file mode type bits set", path)
}
data, err := efs.ReadFile(path)
if err != nil {
return errors.Wrapf(err, "error reading %s", n)
}
data = bytes.TrimPrefix(data, utf8bom)
n = strings.TrimPrefix(n, "/")
files = append(files, &loader.BufferedFile{Name: n, Data: data})
return nil
}
if err := fs.WalkDir(efs, dir, walk); err != nil {
return c, err
}
return loader.LoadFiles(files)
}