Git-Mirrors/constellation
1
0
Fork 0
mirror of https://github.com/edgelesssys/constellation.git synced 2025-11-11 16:25:05 -05:00
Code Issues Projects Releases Packages Wiki Activity

cli: install cilium in cli instead of bootstrapper (#2146)

Browse source 
* add wait and restartDS

* cilium working (tested on azure + gcp)

* clean helm code from bootstrapper

* fixup! clean helm code from bootstrapper

* fixup! clean helm code from bootstrapper

* fixup! clean helm code from bootstrapper

* add patchnode for gcp

* fix gcp

* patch node inside bootstrapper

* apply renaming of client

* fixup! apply renaming of client

* otto feedback
This commit is contained in:
Adrian Stobbe 2023-08-02 15:49:40 +02:00 • committed by GitHub
parent da1376cd90
commit 13eea1ca31
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
36 changed files with 519 additions and 575 deletions
Download patch file Download diff file Expand all files Collapse all files

28
internal/deploy/helm/BUILD.bazel
View file

@ -1,28 +0,0 @@
load("@io_bazel_rules_go//go:def.bzl", "go_library")
load("//bazel/go:go_test.bzl", "go_test")
go_library(
name = "helm",
srcs = [
"helm.go",
"install.go",
],
importpath = "github.com/edgelesssys/constellation/v2/internal/deploy/helm",
visibility = ["//:__subpackages__"],
deps = [
"//internal/constants",
"//internal/retry",
"@io_k8s_apimachinery//pkg/util/wait",
"@sh_helm_helm_v3//pkg/action",
"@sh_helm_helm_v3//pkg/chart",
"@sh_helm_helm_v3//pkg/chart/loader",
"@sh_helm_helm_v3//pkg/cli",
],
)
go_test(
name = "helm_test",
srcs = ["helm_test.go"],
embed = [":helm"],
deps = ["@com_github_stretchr_testify//assert"],
)

61
internal/deploy/helm/helm.go
View file

@ -1,61 +0,0 @@
/*
Copyright (c) Edgeless Systems GmbH
SPDX-License-Identifier: AGPL-3.0-only
*/
// Package helm provides types and functions shared across services.
package helm
// Release bundles all information necessary to create a helm release.
type Release struct {
Chart []byte
Values map[string]any
ReleaseName string
WaitMode WaitMode
}
// Releases bundles all helm releases to be deployed to Constellation.
type Releases struct {
AWSLoadBalancerController *Release
CSI *Release
Cilium Release
CertManager Release
ConstellationOperators Release
ConstellationServices Release
}
// MergeMaps returns a new map that is the merger of it's inputs.
// Key collisions are resolved by taking the value of the second argument (map b).
// Taken from: https://github.com/helm/helm/blob/dbc6d8e20fe1d58d50e6ed30f09a04a77e4c68db/pkg/cli/values/options.go#L91-L108.
func MergeMaps(a, b map[string]any) map[string]any {
out := make(map[string]any, len(a))
for k, v := range a {
out[k] = v
}
for k, v := range b {
if v, ok := v.(map[string]any); ok {
if bv, ok := out[k]; ok {
if bv, ok := bv.(map[string]any); ok {
out[k] = MergeMaps(bv, v)
continue
}
}
}
out[k] = v
}
return out
}
// WaitMode specifies the wait mode for a helm release.
type WaitMode string
const (
// WaitModeNone specifies that the helm release should not wait for the resources to be ready.
WaitModeNone WaitMode = ""
// WaitModeWait specifies that the helm release should wait for the resources to be ready.
WaitModeWait WaitMode = "wait"
// WaitModeAtomic specifies that the helm release should
// wait for the resources to be ready and roll back atomically on failure.
WaitModeAtomic WaitMode = "atomic"
)

109
internal/deploy/helm/helm_test.go
View file

@ -1,109 +0,0 @@
/*
Copyright (c) Edgeless Systems GmbH
SPDX-License-Identifier: AGPL-3.0-only
*/
package helm
import (
"testing"
"github.com/stretchr/testify/assert"
)
func TestMergeMaps(t *testing.T) {
testCases := map[string]struct {
vals map[string]any
extraVals map[string]any
expected map[string]any
}{
"equal": {
vals: map[string]any{
"join-service": map[string]any{
"key1": "foo",
"key2": "bar",
},
},
extraVals: map[string]any{
"join-service": map[string]any{
"extraKey1": "extraFoo",
"extraKey2": "extraBar",
},
},
expected: map[string]any{
"join-service": map[string]any{
"key1": "foo",
"key2": "bar",
"extraKey1": "extraFoo",
"extraKey2": "extraBar",
},
},
},
"missing join-service extraVals": {
vals: map[string]any{
"join-service": map[string]any{
"key1": "foo",
"key2": "bar",
},
},
extraVals: map[string]any{
"extraKey1": "extraFoo",
"extraKey2": "extraBar",
},
expected: map[string]any{
"join-service": map[string]any{
"key1": "foo",
"key2": "bar",
},
"extraKey1": "extraFoo",
"extraKey2": "extraBar",
},
},
"missing join-service vals": {
vals: map[string]any{
"key1": "foo",
"key2": "bar",
},
extraVals: map[string]any{
"join-service": map[string]any{
"extraKey1": "extraFoo",
"extraKey2": "extraBar",
},
},
expected: map[string]any{
"key1": "foo",
"key2": "bar",
"join-service": map[string]any{
"extraKey1": "extraFoo",
"extraKey2": "extraBar",
},
},
},
"key collision": {
vals: map[string]any{
"join-service": map[string]any{
"key1": "foo",
},
},
extraVals: map[string]any{
"join-service": map[string]any{
"key1": "bar",
},
},
expected: map[string]any{
"join-service": map[string]any{
"key1": "bar",
},
},
},
}
for name, tc := range testCases {
t.Run(name, func(t *testing.T) {
assert := assert.New(t)
newVals := MergeMaps(tc.vals, tc.extraVals)
assert.Equal(tc.expected, newVals)
})
}
}

160
internal/deploy/helm/install.go
View file

@ -1,160 +0,0 @@
/*
Copyright (c) Edgeless Systems GmbH
SPDX-License-Identifier: AGPL-3.0-only
*/
package helm
import (
"bytes"
"context"
"fmt"
"strings"
"time"
"github.com/edgelesssys/constellation/v2/internal/constants"
"github.com/edgelesssys/constellation/v2/internal/retry"
"helm.sh/helm/v3/pkg/action"
"helm.sh/helm/v3/pkg/chart"
"helm.sh/helm/v3/pkg/chart/loader"
"helm.sh/helm/v3/pkg/cli"
"k8s.io/apimachinery/pkg/util/wait"
)
const (
// timeout is the maximum time given to the helm Installer.
timeout = 10 * time.Minute
// maximumRetryAttempts is the maximum number of attempts to retry a helm install.
maximumRetryAttempts = 3
)
type debugLog interface {
Debugf(format string, args ...any)
Sync()
}
// Installer is a wrapper for a helm install action.
type Installer struct {
*action.Install
log debugLog
}
// NewInstaller creates a new Installer with the given logger.
func NewInstaller(kubeconfig string, logger debugLog) (*Installer, error) {
settings := cli.New()
settings.KubeConfig = kubeconfig
actionConfig := &action.Configuration{}
if err := actionConfig.Init(settings.RESTClientGetter(), constants.HelmNamespace,
"secret", logger.Debugf); err != nil {
return nil, err
}
action := action.NewInstall(actionConfig)
action.Namespace = constants.HelmNamespace
action.Timeout = timeout
return &Installer{
Install: action,
log: logger,
}, nil
}
// InstallChart is the generic install function for helm charts.
func (h *Installer) InstallChart(ctx context.Context, release Release) error {
return h.InstallChartWithValues(ctx, release, nil)
}
// InstallChartWithValues is the generic install function for helm charts with custom values.
func (h *Installer) InstallChartWithValues(ctx context.Context, release Release, extraValues map[string]any) error {
mergedVals := MergeMaps(release.Values, extraValues)
h.ReleaseName = release.ReleaseName
if err := h.SetWaitMode(release.WaitMode); err != nil {
return err
}
return h.install(ctx, release.Chart, mergedVals)
}
// install tries to install the given chart and aborts after ~5 tries.
// The function will wait 30 seconds before retrying a failed installation attempt.
// After 3 tries, the retrier will be canceled and the function returns with an error.
func (h *Installer) install(ctx context.Context, chartRaw []byte, values map[string]any) error {
var retries int
retriable := func(err error) bool {
// abort after maximumRetryAttempts tries.
if retries >= maximumRetryAttempts {
return false
}
retries++
// only retry if atomic is set
// otherwise helm doesn't uninstall
// the release on failure
if !h.Atomic {
return false
}
// check if error is retriable
return wait.Interrupted(err) ||
strings.Contains(err.Error(), "connection refused")
}
reader := bytes.NewReader(chartRaw)
chart, err := loader.LoadArchive(reader)
if err != nil {
return fmt.Errorf("helm load archive: %w", err)
}
doer := installDoer{
h,
chart,
values,
h.log,
}
retrier := retry.NewIntervalRetrier(doer, 30*time.Second, retriable)
retryLoopStartTime := time.Now()
if err := retrier.Do(ctx); err != nil {
return fmt.Errorf("helm install: %w", err)
}
retryLoopFinishDuration := time.Since(retryLoopStartTime)
h.log.Debugf("Helm chart %q installation finished after %s", chart.Name(), retryLoopFinishDuration)
return nil
}
// SetWaitMode sets the wait mode of the installer.
func (h *Installer) SetWaitMode(waitMode WaitMode) error {
switch waitMode {
case WaitModeNone:
h.Wait = false
h.Atomic = false
case WaitModeWait:
h.Wait = true
h.Atomic = false
case WaitModeAtomic:
h.Wait = true
h.Atomic = true
default:
return fmt.Errorf("unknown wait mode %q", waitMode)
}
return nil
}
// installDoer is a help struct to enable retrying helm's install action.
type installDoer struct {
Installer *Installer
chart *chart.Chart
values map[string]any
log debugLog
}
// Do logs which chart is installed and tries to install it.
func (i installDoer) Do(ctx context.Context) error {
i.log.Debugf("Trying to install Helm chart %s", i.chart.Name())
if _, err := i.Installer.RunWithContext(ctx, i.chart, i.values); err != nil {
i.log.Debugf("Helm chart installation % failed: %v", i.chart.Name(), err)
return err
}
return nil
}

2
internal/kubernetes/kubectl/BUILD.bazel
View file

@ -12,9 +12,11 @@ go_library(
"@io_k8s_apiextensions_apiserver//pkg/client/clientset/clientset/typed/apiextensions/v1:apiextensions",
"@io_k8s_apimachinery//pkg/apis/meta/v1:meta",
"@io_k8s_apimachinery//pkg/apis/meta/v1/unstructured",
"@io_k8s_apimachinery//pkg/labels",
"@io_k8s_apimachinery//pkg/runtime",
"@io_k8s_apimachinery//pkg/runtime/schema",
"@io_k8s_apimachinery//pkg/runtime/serializer",
"@io_k8s_apimachinery//pkg/types",
"@io_k8s_client_go//dynamic",
"@io_k8s_client_go//kubernetes",
"@io_k8s_client_go//scale/scheme",

18
internal/kubernetes/kubectl/kubectl.go
View file

@ -19,9 +19,11 @@ import (
apiextensionsclientv1 "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset/typed/apiextensions/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/labels"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apimachinery/pkg/runtime/serializer"
"k8s.io/apimachinery/pkg/types"
"k8s.io/client-go/dynamic"
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/scale/scheme"
@ -130,6 +132,22 @@ func (k *Kubectl) AnnotateNode(ctx context.Context, nodeName, annotationKey, ann
})
}
// PatchFirstNodePodCIDR patches the firstNodePodCIDR of the first control-plane node for Cilium.
func (k *Kubectl) PatchFirstNodePodCIDR(ctx context.Context, firstNodePodCIDR string) error {
selector := labels.Set{"node-role.kubernetes.io/control-plane": ""}.AsSelector()
controlPlaneList, err := k.CoreV1().Nodes().List(ctx, metav1.ListOptions{LabelSelector: selector.String()})
if err != nil {
return err
}
if len(controlPlaneList.Items) != 1 {
return fmt.Errorf("expected 1 control-plane node, got %d", len(controlPlaneList.Items))
}
nodeName := controlPlaneList.Items[0].Name
// Update the node's spec
_, err = k.CoreV1().Nodes().Patch(context.Background(), nodeName, types.MergePatchType, []byte(fmt.Sprintf(`{"spec":{"podCIDR":"%s"}}`, firstNodePodCIDR)), metav1.PatchOptions{})
return err
}
// ListAllNamespaces returns all namespaces in the cluster.
func (k *Kubectl) ListAllNamespaces(ctx context.Context) (*corev1.NamespaceList, error) {
return k.CoreV1().Namespaces().List(ctx, metav1.ListOptions{})