cli: install cilium in cli instead of bootstrapper (#2146)

* add wait and restartDS

* cilium working (tested on azure + gcp)

* clean helm code from bootstrapper

* fixup! clean helm code from bootstrapper

* fixup! clean helm code from bootstrapper

* fixup! clean helm code from bootstrapper

* add patchnode for gcp

* fix gcp

* patch node inside bootstrapper

* apply renaming of client

* fixup! apply renaming of client

* otto feedback
This commit is contained in:
Adrian Stobbe 2023-08-02 15:49:40 +02:00 committed by GitHub
parent da1376cd90
commit 13eea1ca31
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
36 changed files with 519 additions and 575 deletions

View File

@ -35,7 +35,6 @@ go_library(
"//internal/cloud/openstack", "//internal/cloud/openstack",
"//internal/cloud/qemu", "//internal/cloud/qemu",
"//internal/constants", "//internal/constants",
"//internal/deploy/helm",
"//internal/file", "//internal/file",
"//internal/grpc/dialer", "//internal/grpc/dialer",
"//internal/kubernetes/kubectl", "//internal/kubernetes/kubectl",

View File

@ -32,7 +32,6 @@ import (
openstackcloud "github.com/edgelesssys/constellation/v2/internal/cloud/openstack" openstackcloud "github.com/edgelesssys/constellation/v2/internal/cloud/openstack"
qemucloud "github.com/edgelesssys/constellation/v2/internal/cloud/qemu" qemucloud "github.com/edgelesssys/constellation/v2/internal/cloud/qemu"
"github.com/edgelesssys/constellation/v2/internal/constants" "github.com/edgelesssys/constellation/v2/internal/constants"
"github.com/edgelesssys/constellation/v2/internal/deploy/helm"
"github.com/edgelesssys/constellation/v2/internal/file" "github.com/edgelesssys/constellation/v2/internal/file"
"github.com/edgelesssys/constellation/v2/internal/kubernetes/kubectl" "github.com/edgelesssys/constellation/v2/internal/kubernetes/kubectl"
"github.com/edgelesssys/constellation/v2/internal/logger" "github.com/edgelesssys/constellation/v2/internal/logger"
@ -67,11 +66,6 @@ func main() {
var openDevice vtpm.TPMOpenFunc var openDevice vtpm.TPMOpenFunc
var fs afero.Fs var fs afero.Fs
helmClient, err := helm.NewInstaller(constants.ControlPlaneAdminConfFilename, log)
if err != nil {
log.With(zap.Error(err)).Fatalf("Helm client could not be initialized")
}
attestVariant, err := variant.FromString(os.Getenv(constants.AttestationVariant)) attestVariant, err := variant.FromString(os.Getenv(constants.AttestationVariant))
if err != nil { if err != nil {
log.With(zap.Error(err)).Fatalf("Failed to parse attestation variant") log.With(zap.Error(err)).Fatalf("Failed to parse attestation variant")
@ -96,7 +90,7 @@ func main() {
clusterInitJoiner = kubernetes.New( clusterInitJoiner = kubernetes.New(
"aws", k8sapi.NewKubernetesUtil(), &k8sapi.KubdeadmConfiguration{}, kubectl.New(), "aws", k8sapi.NewKubernetesUtil(), &k8sapi.KubdeadmConfiguration{}, kubectl.New(),
metadata, helmClient, &kubewaiter.CloudKubeAPIWaiter{}, metadata, &kubewaiter.CloudKubeAPIWaiter{},
) )
openDevice = vtpm.OpenVTPM openDevice = vtpm.OpenVTPM
fs = afero.NewOsFs() fs = afero.NewOsFs()
@ -116,7 +110,7 @@ func main() {
metadataAPI = metadata metadataAPI = metadata
clusterInitJoiner = kubernetes.New( clusterInitJoiner = kubernetes.New(
"gcp", k8sapi.NewKubernetesUtil(), &k8sapi.KubdeadmConfiguration{}, kubectl.New(), "gcp", k8sapi.NewKubernetesUtil(), &k8sapi.KubdeadmConfiguration{}, kubectl.New(),
metadata, helmClient, &kubewaiter.CloudKubeAPIWaiter{}, metadata, &kubewaiter.CloudKubeAPIWaiter{},
) )
openDevice = vtpm.OpenVTPM openDevice = vtpm.OpenVTPM
fs = afero.NewOsFs() fs = afero.NewOsFs()
@ -134,7 +128,7 @@ func main() {
metadataAPI = metadata metadataAPI = metadata
clusterInitJoiner = kubernetes.New( clusterInitJoiner = kubernetes.New(
"azure", k8sapi.NewKubernetesUtil(), &k8sapi.KubdeadmConfiguration{}, kubectl.New(), "azure", k8sapi.NewKubernetesUtil(), &k8sapi.KubdeadmConfiguration{}, kubectl.New(),
metadata, helmClient, &kubewaiter.CloudKubeAPIWaiter{}, metadata, &kubewaiter.CloudKubeAPIWaiter{},
) )
openDevice = vtpm.OpenVTPM openDevice = vtpm.OpenVTPM
@ -145,7 +139,7 @@ func main() {
metadata := qemucloud.New() metadata := qemucloud.New()
clusterInitJoiner = kubernetes.New( clusterInitJoiner = kubernetes.New(
"qemu", k8sapi.NewKubernetesUtil(), &k8sapi.KubdeadmConfiguration{}, kubectl.New(), "qemu", k8sapi.NewKubernetesUtil(), &k8sapi.KubdeadmConfiguration{}, kubectl.New(),
metadata, helmClient, &kubewaiter.CloudKubeAPIWaiter{}, metadata, &kubewaiter.CloudKubeAPIWaiter{},
) )
metadataAPI = metadata metadataAPI = metadata
@ -168,7 +162,7 @@ func main() {
} }
clusterInitJoiner = kubernetes.New( clusterInitJoiner = kubernetes.New(
"openstack", k8sapi.NewKubernetesUtil(), &k8sapi.KubdeadmConfiguration{}, kubectl.New(), "openstack", k8sapi.NewKubernetesUtil(), &k8sapi.KubdeadmConfiguration{}, kubectl.New(),
metadata, helmClient, &kubewaiter.CloudKubeAPIWaiter{}, metadata, &kubewaiter.CloudKubeAPIWaiter{},
) )
metadataAPI = metadata metadataAPI = metadata
openDevice = vtpm.OpenVTPM openDevice = vtpm.OpenVTPM

View File

@ -22,7 +22,7 @@ type clusterFake struct{}
// InitCluster fakes bootstrapping a new cluster with the current node being the master, returning the arguments required to join the cluster. // InitCluster fakes bootstrapping a new cluster with the current node being the master, returning the arguments required to join the cluster.
func (c *clusterFake) InitCluster( func (c *clusterFake) InitCluster(
context.Context, string, string, context.Context, string, string,
[]byte, bool, components.Components, []string, *logger.Logger, bool, components.Components, []string, *logger.Logger,
) ([]byte, error) { ) ([]byte, error) {
return []byte{}, nil return []byte{}, nil
} }

View File

@ -33,7 +33,6 @@ type InitRequest struct {
StorageUri string `protobuf:"bytes,4,opt,name=storage_uri,json=storageUri,proto3" json:"storage_uri,omitempty"` StorageUri string `protobuf:"bytes,4,opt,name=storage_uri,json=storageUri,proto3" json:"storage_uri,omitempty"`
CloudServiceAccountUri string `protobuf:"bytes,7,opt,name=cloud_service_account_uri,json=cloudServiceAccountUri,proto3" json:"cloud_service_account_uri,omitempty"` CloudServiceAccountUri string `protobuf:"bytes,7,opt,name=cloud_service_account_uri,json=cloudServiceAccountUri,proto3" json:"cloud_service_account_uri,omitempty"`
KubernetesVersion string `protobuf:"bytes,8,opt,name=kubernetes_version,json=kubernetesVersion,proto3" json:"kubernetes_version,omitempty"` KubernetesVersion string `protobuf:"bytes,8,opt,name=kubernetes_version,json=kubernetesVersion,proto3" json:"kubernetes_version,omitempty"`
HelmDeployments []byte `protobuf:"bytes,11,opt,name=helm_deployments,json=helmDeployments,proto3" json:"helm_deployments,omitempty"`
ConformanceMode bool `protobuf:"varint,14,opt,name=conformance_mode,json=conformanceMode,proto3" json:"conformance_mode,omitempty"` ConformanceMode bool `protobuf:"varint,14,opt,name=conformance_mode,json=conformanceMode,proto3" json:"conformance_mode,omitempty"`
KubernetesComponents []*KubernetesComponent `protobuf:"bytes,15,rep,name=kubernetes_components,json=kubernetesComponents,proto3" json:"kubernetes_components,omitempty"` KubernetesComponents []*KubernetesComponent `protobuf:"bytes,15,rep,name=kubernetes_components,json=kubernetesComponents,proto3" json:"kubernetes_components,omitempty"`
InitSecret []byte `protobuf:"bytes,16,opt,name=init_secret,json=initSecret,proto3" json:"init_secret,omitempty"` InitSecret []byte `protobuf:"bytes,16,opt,name=init_secret,json=initSecret,proto3" json:"init_secret,omitempty"`
@ -101,13 +100,6 @@ func (x *InitRequest) GetKubernetesVersion() string {
return "" return ""
} }
func (x *InitRequest) GetHelmDeployments() []byte {
if x != nil {
return x.HelmDeployments
}
return nil
}
func (x *InitRequest) GetConformanceMode() bool { func (x *InitRequest) GetConformanceMode() bool {
if x != nil { if x != nil {
return x.ConformanceMode return x.ConformanceMode
@ -471,7 +463,7 @@ var File_bootstrapper_initproto_init_proto protoreflect.FileDescriptor
var file_bootstrapper_initproto_init_proto_rawDesc = []byte{ var file_bootstrapper_initproto_init_proto_rawDesc = []byte{
0x0a, 0x21, 0x62, 0x6f, 0x6f, 0x74, 0x73, 0x74, 0x72, 0x61, 0x70, 0x70, 0x65, 0x72, 0x2f, 0x69, 0x0a, 0x21, 0x62, 0x6f, 0x6f, 0x74, 0x73, 0x74, 0x72, 0x61, 0x70, 0x70, 0x65, 0x72, 0x2f, 0x69,
0x6e, 0x69, 0x74, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x69, 0x6e, 0x69, 0x74, 0x2e, 0x70, 0x72, 0x6e, 0x69, 0x74, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x69, 0x6e, 0x69, 0x74, 0x2e, 0x70, 0x72,
0x6f, 0x74, 0x6f, 0x12, 0x04, 0x69, 0x6e, 0x69, 0x74, 0x22, 0xcb, 0x03, 0x0a, 0x0b, 0x49, 0x6e, 0x6f, 0x74, 0x6f, 0x12, 0x04, 0x69, 0x6e, 0x69, 0x74, 0x22, 0xa0, 0x03, 0x0a, 0x0b, 0x49, 0x6e,
0x69, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x17, 0x0a, 0x07, 0x6b, 0x6d, 0x73, 0x69, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x17, 0x0a, 0x07, 0x6b, 0x6d, 0x73,
0x5f, 0x75, 0x72, 0x69, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x6b, 0x6d, 0x73, 0x55, 0x5f, 0x75, 0x72, 0x69, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x6b, 0x6d, 0x73, 0x55,
0x72, 0x69, 0x12, 0x1f, 0x0a, 0x0b, 0x73, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x5f, 0x75, 0x72, 0x72, 0x69, 0x12, 0x1f, 0x0a, 0x0b, 0x73, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x5f, 0x75, 0x72,
@ -483,64 +475,61 @@ var file_bootstrapper_initproto_init_proto_rawDesc = []byte{
0x0a, 0x12, 0x6b, 0x75, 0x62, 0x65, 0x72, 0x6e, 0x65, 0x74, 0x65, 0x73, 0x5f, 0x76, 0x65, 0x72, 0x0a, 0x12, 0x6b, 0x75, 0x62, 0x65, 0x72, 0x6e, 0x65, 0x74, 0x65, 0x73, 0x5f, 0x76, 0x65, 0x72,
0x73, 0x69, 0x6f, 0x6e, 0x18, 0x08, 0x20, 0x01, 0x28, 0x09, 0x52, 0x11, 0x6b, 0x75, 0x62, 0x65, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x08, 0x20, 0x01, 0x28, 0x09, 0x52, 0x11, 0x6b, 0x75, 0x62, 0x65,
0x72, 0x6e, 0x65, 0x74, 0x65, 0x73, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x29, 0x0a, 0x72, 0x6e, 0x65, 0x74, 0x65, 0x73, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x29, 0x0a,
0x10, 0x68, 0x65, 0x6c, 0x6d, 0x5f, 0x64, 0x65, 0x70, 0x6c, 0x6f, 0x79, 0x6d, 0x65, 0x6e, 0x74, 0x10, 0x63, 0x6f, 0x6e, 0x66, 0x6f, 0x72, 0x6d, 0x61, 0x6e, 0x63, 0x65, 0x5f, 0x6d, 0x6f, 0x64,
0x73, 0x18, 0x0b, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0f, 0x68, 0x65, 0x6c, 0x6d, 0x44, 0x65, 0x70, 0x65, 0x18, 0x0e, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0f, 0x63, 0x6f, 0x6e, 0x66, 0x6f, 0x72, 0x6d,
0x6c, 0x6f, 0x79, 0x6d, 0x65, 0x6e, 0x74, 0x73, 0x12, 0x29, 0x0a, 0x10, 0x63, 0x6f, 0x6e, 0x66, 0x61, 0x6e, 0x63, 0x65, 0x4d, 0x6f, 0x64, 0x65, 0x12, 0x4e, 0x0a, 0x15, 0x6b, 0x75, 0x62, 0x65,
0x6f, 0x72, 0x6d, 0x61, 0x6e, 0x63, 0x65, 0x5f, 0x6d, 0x6f, 0x64, 0x65, 0x18, 0x0e, 0x20, 0x01, 0x72, 0x6e, 0x65, 0x74, 0x65, 0x73, 0x5f, 0x63, 0x6f, 0x6d, 0x70, 0x6f, 0x6e, 0x65, 0x6e, 0x74,
0x28, 0x08, 0x52, 0x0f, 0x63, 0x6f, 0x6e, 0x66, 0x6f, 0x72, 0x6d, 0x61, 0x6e, 0x63, 0x65, 0x4d, 0x73, 0x18, 0x0f, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x69, 0x6e, 0x69, 0x74, 0x2e, 0x4b,
0x6f, 0x64, 0x65, 0x12, 0x4e, 0x0a, 0x15, 0x6b, 0x75, 0x62, 0x65, 0x72, 0x6e, 0x65, 0x74, 0x65,
0x73, 0x5f, 0x63, 0x6f, 0x6d, 0x70, 0x6f, 0x6e, 0x65, 0x6e, 0x74, 0x73, 0x18, 0x0f, 0x20, 0x03,
0x28, 0x0b, 0x32, 0x19, 0x2e, 0x69, 0x6e, 0x69, 0x74, 0x2e, 0x4b, 0x75, 0x62, 0x65, 0x72, 0x6e,
0x65, 0x74, 0x65, 0x73, 0x43, 0x6f, 0x6d, 0x70, 0x6f, 0x6e, 0x65, 0x6e, 0x74, 0x52, 0x14, 0x6b,
0x75, 0x62, 0x65, 0x72, 0x6e, 0x65, 0x74, 0x65, 0x73, 0x43, 0x6f, 0x6d, 0x70, 0x6f, 0x6e, 0x65, 0x75, 0x62, 0x65, 0x72, 0x6e, 0x65, 0x74, 0x65, 0x73, 0x43, 0x6f, 0x6d, 0x70, 0x6f, 0x6e, 0x65,
0x6e, 0x74, 0x73, 0x12, 0x1f, 0x0a, 0x0b, 0x69, 0x6e, 0x69, 0x74, 0x5f, 0x73, 0x65, 0x63, 0x72, 0x6e, 0x74, 0x52, 0x14, 0x6b, 0x75, 0x62, 0x65, 0x72, 0x6e, 0x65, 0x74, 0x65, 0x73, 0x43, 0x6f,
0x65, 0x74, 0x18, 0x10, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0a, 0x69, 0x6e, 0x69, 0x74, 0x53, 0x65, 0x6d, 0x70, 0x6f, 0x6e, 0x65, 0x6e, 0x74, 0x73, 0x12, 0x1f, 0x0a, 0x0b, 0x69, 0x6e, 0x69, 0x74,
0x63, 0x72, 0x65, 0x74, 0x12, 0x21, 0x0a, 0x0c, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x5f, 0x73, 0x65, 0x63, 0x72, 0x65, 0x74, 0x18, 0x10, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0a, 0x69,
0x6e, 0x61, 0x6d, 0x65, 0x18, 0x11, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x63, 0x6c, 0x75, 0x73, 0x6e, 0x69, 0x74, 0x53, 0x65, 0x63, 0x72, 0x65, 0x74, 0x12, 0x21, 0x0a, 0x0c, 0x63, 0x6c, 0x75,
0x74, 0x65, 0x72, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x2e, 0x0a, 0x13, 0x61, 0x70, 0x69, 0x73, 0x65, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x11, 0x20, 0x01, 0x28, 0x09, 0x52,
0x72, 0x76, 0x65, 0x72, 0x5f, 0x63, 0x65, 0x72, 0x74, 0x5f, 0x73, 0x61, 0x6e, 0x73, 0x18, 0x12, 0x0b, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x2e, 0x0a, 0x13,
0x20, 0x03, 0x28, 0x09, 0x52, 0x11, 0x61, 0x70, 0x69, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x43, 0x61, 0x70, 0x69, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x5f, 0x63, 0x65, 0x72, 0x74, 0x5f, 0x73,
0x65, 0x72, 0x74, 0x53, 0x61, 0x6e, 0x73, 0x22, 0xc1, 0x01, 0x0a, 0x0c, 0x49, 0x6e, 0x69, 0x74, 0x61, 0x6e, 0x73, 0x18, 0x12, 0x20, 0x03, 0x28, 0x09, 0x52, 0x11, 0x61, 0x70, 0x69, 0x73, 0x65,
0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x3e, 0x0a, 0x0c, 0x69, 0x6e, 0x69, 0x74, 0x72, 0x76, 0x65, 0x72, 0x43, 0x65, 0x72, 0x74, 0x53, 0x61, 0x6e, 0x73, 0x22, 0xc1, 0x01, 0x0a,
0x5f, 0x73, 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x19, 0x0c, 0x49, 0x6e, 0x69, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x3e, 0x0a,
0x2e, 0x69, 0x6e, 0x69, 0x74, 0x2e, 0x49, 0x6e, 0x69, 0x74, 0x53, 0x75, 0x63, 0x63, 0x65, 0x73, 0x0c, 0x69, 0x6e, 0x69, 0x74, 0x5f, 0x73, 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, 0x18, 0x01, 0x20,
0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x48, 0x00, 0x52, 0x0b, 0x69, 0x6e, 0x69, 0x01, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x69, 0x6e, 0x69, 0x74, 0x2e, 0x49, 0x6e, 0x69, 0x74, 0x53,
0x74, 0x53, 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, 0x12, 0x3e, 0x0a, 0x0c, 0x69, 0x6e, 0x69, 0x74, 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x48, 0x00,
0x5f, 0x66, 0x61, 0x69, 0x6c, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x19, 0x52, 0x0b, 0x69, 0x6e, 0x69, 0x74, 0x53, 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, 0x12, 0x3e, 0x0a,
0x2e, 0x69, 0x6e, 0x69, 0x74, 0x2e, 0x49, 0x6e, 0x69, 0x74, 0x46, 0x61, 0x69, 0x6c, 0x75, 0x72, 0x0c, 0x69, 0x6e, 0x69, 0x74, 0x5f, 0x66, 0x61, 0x69, 0x6c, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20,
0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x48, 0x00, 0x52, 0x0b, 0x69, 0x6e, 0x69, 0x01, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x69, 0x6e, 0x69, 0x74, 0x2e, 0x49, 0x6e, 0x69, 0x74, 0x46,
0x74, 0x46, 0x61, 0x69, 0x6c, 0x75, 0x72, 0x65, 0x12, 0x29, 0x0a, 0x03, 0x6c, 0x6f, 0x67, 0x18, 0x61, 0x69, 0x6c, 0x75, 0x72, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x48, 0x00,
0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x15, 0x2e, 0x69, 0x6e, 0x69, 0x74, 0x2e, 0x4c, 0x6f, 0x67, 0x52, 0x0b, 0x69, 0x6e, 0x69, 0x74, 0x46, 0x61, 0x69, 0x6c, 0x75, 0x72, 0x65, 0x12, 0x29, 0x0a,
0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x54, 0x79, 0x70, 0x65, 0x48, 0x00, 0x52, 0x03, 0x03, 0x6c, 0x6f, 0x67, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x15, 0x2e, 0x69, 0x6e, 0x69,
0x6c, 0x6f, 0x67, 0x42, 0x06, 0x0a, 0x04, 0x6b, 0x69, 0x6e, 0x64, 0x22, 0x6f, 0x0a, 0x13, 0x49, 0x74, 0x2e, 0x4c, 0x6f, 0x67, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x54, 0x79, 0x70,
0x6e, 0x69, 0x74, 0x53, 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x65, 0x48, 0x00, 0x52, 0x03, 0x6c, 0x6f, 0x67, 0x42, 0x06, 0x0a, 0x04, 0x6b, 0x69, 0x6e, 0x64,
0x73, 0x65, 0x12, 0x1e, 0x0a, 0x0a, 0x6b, 0x75, 0x62, 0x65, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x22, 0x6f, 0x0a, 0x13, 0x49, 0x6e, 0x69, 0x74, 0x53, 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, 0x52,
0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0a, 0x6b, 0x75, 0x62, 0x65, 0x63, 0x6f, 0x6e, 0x66, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x1e, 0x0a, 0x0a, 0x6b, 0x75, 0x62, 0x65, 0x63,
0x69, 0x67, 0x12, 0x19, 0x0a, 0x08, 0x6f, 0x77, 0x6e, 0x65, 0x72, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0a, 0x6b, 0x75, 0x62,
0x20, 0x01, 0x28, 0x0c, 0x52, 0x07, 0x6f, 0x77, 0x6e, 0x65, 0x72, 0x49, 0x64, 0x12, 0x1d, 0x0a, 0x65, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x19, 0x0a, 0x08, 0x6f, 0x77, 0x6e, 0x65, 0x72,
0x0a, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x69, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x07, 0x6f, 0x77, 0x6e, 0x65, 0x72,
0x0c, 0x52, 0x09, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x49, 0x64, 0x22, 0x2b, 0x0a, 0x13, 0x49, 0x64, 0x12, 0x1d, 0x0a, 0x0a, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x69, 0x64,
0x49, 0x6e, 0x69, 0x74, 0x46, 0x61, 0x69, 0x6c, 0x75, 0x72, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x09, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x49,
0x6e, 0x73, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x18, 0x01, 0x20, 0x01, 0x64, 0x22, 0x2b, 0x0a, 0x13, 0x49, 0x6e, 0x69, 0x74, 0x46, 0x61, 0x69, 0x6c, 0x75, 0x72, 0x65,
0x28, 0x09, 0x52, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x22, 0x23, 0x0a, 0x0f, 0x4c, 0x6f, 0x67, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x65, 0x72, 0x72, 0x6f,
0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x54, 0x79, 0x70, 0x65, 0x12, 0x10, 0x0a, 0x03, 0x72, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x22, 0x23,
0x6c, 0x6f, 0x67, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x03, 0x6c, 0x6f, 0x67, 0x22, 0x78, 0x0a, 0x0f, 0x4c, 0x6f, 0x67, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x54, 0x79, 0x70,
0x0a, 0x13, 0x4b, 0x75, 0x62, 0x65, 0x72, 0x6e, 0x65, 0x74, 0x65, 0x73, 0x43, 0x6f, 0x6d, 0x70, 0x65, 0x12, 0x10, 0x0a, 0x03, 0x6c, 0x6f, 0x67, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x03,
0x6f, 0x6e, 0x65, 0x6e, 0x74, 0x12, 0x10, 0x0a, 0x03, 0x75, 0x72, 0x6c, 0x18, 0x01, 0x20, 0x01, 0x6c, 0x6f, 0x67, 0x22, 0x78, 0x0a, 0x13, 0x4b, 0x75, 0x62, 0x65, 0x72, 0x6e, 0x65, 0x74, 0x65,
0x28, 0x09, 0x52, 0x03, 0x75, 0x72, 0x6c, 0x12, 0x12, 0x0a, 0x04, 0x68, 0x61, 0x73, 0x68, 0x18, 0x73, 0x43, 0x6f, 0x6d, 0x70, 0x6f, 0x6e, 0x65, 0x6e, 0x74, 0x12, 0x10, 0x0a, 0x03, 0x75, 0x72,
0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x68, 0x61, 0x73, 0x68, 0x12, 0x21, 0x0a, 0x0c, 0x69, 0x6c, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x75, 0x72, 0x6c, 0x12, 0x12, 0x0a, 0x04,
0x6e, 0x73, 0x74, 0x61, 0x6c, 0x6c, 0x5f, 0x70, 0x61, 0x74, 0x68, 0x18, 0x03, 0x20, 0x01, 0x28, 0x68, 0x61, 0x73, 0x68, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x68, 0x61, 0x73, 0x68,
0x09, 0x52, 0x0b, 0x69, 0x6e, 0x73, 0x74, 0x61, 0x6c, 0x6c, 0x50, 0x61, 0x74, 0x68, 0x12, 0x18, 0x12, 0x21, 0x0a, 0x0c, 0x69, 0x6e, 0x73, 0x74, 0x61, 0x6c, 0x6c, 0x5f, 0x70, 0x61, 0x74, 0x68,
0x0a, 0x07, 0x65, 0x78, 0x74, 0x72, 0x61, 0x63, 0x74, 0x18, 0x04, 0x20, 0x01, 0x28, 0x08, 0x52, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x69, 0x6e, 0x73, 0x74, 0x61, 0x6c, 0x6c, 0x50,
0x07, 0x65, 0x78, 0x74, 0x72, 0x61, 0x63, 0x74, 0x32, 0x36, 0x0a, 0x03, 0x41, 0x50, 0x49, 0x12, 0x61, 0x74, 0x68, 0x12, 0x18, 0x0a, 0x07, 0x65, 0x78, 0x74, 0x72, 0x61, 0x63, 0x74, 0x18, 0x04,
0x2f, 0x0a, 0x04, 0x49, 0x6e, 0x69, 0x74, 0x12, 0x11, 0x2e, 0x69, 0x6e, 0x69, 0x74, 0x2e, 0x49, 0x20, 0x01, 0x28, 0x08, 0x52, 0x07, 0x65, 0x78, 0x74, 0x72, 0x61, 0x63, 0x74, 0x32, 0x36, 0x0a,
0x6e, 0x69, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x12, 0x2e, 0x69, 0x6e, 0x69, 0x03, 0x41, 0x50, 0x49, 0x12, 0x2f, 0x0a, 0x04, 0x49, 0x6e, 0x69, 0x74, 0x12, 0x11, 0x2e, 0x69,
0x74, 0x2e, 0x49, 0x6e, 0x69, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x30, 0x01, 0x6e, 0x69, 0x74, 0x2e, 0x49, 0x6e, 0x69, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a,
0x42, 0x40, 0x5a, 0x3e, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x65, 0x12, 0x2e, 0x69, 0x6e, 0x69, 0x74, 0x2e, 0x49, 0x6e, 0x69, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f,
0x64, 0x67, 0x65, 0x6c, 0x65, 0x73, 0x73, 0x73, 0x79, 0x73, 0x2f, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x6e, 0x73, 0x65, 0x30, 0x01, 0x42, 0x40, 0x5a, 0x3e, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e,
0x65, 0x6c, 0x6c, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x2f, 0x76, 0x32, 0x2f, 0x62, 0x6f, 0x6f, 0x74, 0x63, 0x6f, 0x6d, 0x2f, 0x65, 0x64, 0x67, 0x65, 0x6c, 0x65, 0x73, 0x73, 0x73, 0x79, 0x73, 0x2f,
0x73, 0x74, 0x72, 0x61, 0x70, 0x70, 0x65, 0x72, 0x2f, 0x69, 0x6e, 0x69, 0x74, 0x70, 0x72, 0x6f, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x65, 0x6c, 0x6c, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x2f, 0x76, 0x32,
0x74, 0x6f, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, 0x2f, 0x62, 0x6f, 0x6f, 0x74, 0x73, 0x74, 0x72, 0x61, 0x70, 0x70, 0x65, 0x72, 0x2f, 0x69, 0x6e,
0x69, 0x74, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
} }
var ( var (

View File

@ -19,7 +19,7 @@ message InitRequest {
string kubernetes_version = 8; string kubernetes_version = 8;
// repeated SSHUserKey ssh_user_keys = 9; removed // repeated SSHUserKey ssh_user_keys = 9; removed
// bytes salt = 10; removed // bytes salt = 10; removed
bytes helm_deployments = 11; // bytes helm_deployments = 11; removed
// repeated uint32 enforced_pcrs = 12; removed // repeated uint32 enforced_pcrs = 12; removed
// bool enforce_idkeydigest = 13; removed // bool enforce_idkeydigest = 13; removed
bool conformance_mode = 14; bool conformance_mode = 14;

View File

@ -215,7 +215,6 @@ func (s *Server) Init(req *initproto.InitRequest, stream initproto.API_InitServe
kubeconfig, err := s.initializer.InitCluster(stream.Context(), kubeconfig, err := s.initializer.InitCluster(stream.Context(),
req.KubernetesVersion, req.KubernetesVersion,
clusterName, clusterName,
req.HelmDeployments,
req.ConformanceMode, req.ConformanceMode,
components.NewComponentsFromInitProto(req.KubernetesComponents), components.NewComponentsFromInitProto(req.KubernetesComponents),
req.ApiserverCertSans, req.ApiserverCertSans,
@ -342,7 +341,6 @@ type ClusterInitializer interface {
ctx context.Context, ctx context.Context,
k8sVersion string, k8sVersion string,
clusterName string, clusterName string,
helmDeployments []byte,
conformanceMode bool, conformanceMode bool,
kubernetesComponents components.Components, kubernetesComponents components.Components,
apiServerCertSANs []string, apiServerCertSANs []string,

View File

@ -407,7 +407,7 @@ type stubClusterInitializer struct {
func (i *stubClusterInitializer) InitCluster( func (i *stubClusterInitializer) InitCluster(
context.Context, string, string, context.Context, string, string,
[]byte, bool, components.Components, []string, *logger.Logger, bool, components.Components, []string, *logger.Logger,
) ([]byte, error) { ) ([]byte, error) {
return i.initClusterKubeconfig, i.initClusterErr return i.initClusterKubeconfig, i.initClusterErr
} }

View File

@ -16,7 +16,6 @@ go_library(
"//internal/cloud/cloudprovider", "//internal/cloud/cloudprovider",
"//internal/cloud/metadata", "//internal/cloud/metadata",
"//internal/constants", "//internal/constants",
"//internal/deploy/helm",
"//internal/kubernetes", "//internal/kubernetes",
"//internal/logger", "//internal/logger",
"//internal/role", "//internal/role",
@ -37,7 +36,6 @@ go_test(
"//bootstrapper/internal/kubernetes/kubewaiter", "//bootstrapper/internal/kubernetes/kubewaiter",
"//internal/cloud/metadata", "//internal/cloud/metadata",
"//internal/constants", "//internal/constants",
"//internal/deploy/helm",
"//internal/kubernetes", "//internal/kubernetes",
"//internal/logger", "//internal/logger",
"//internal/role", "//internal/role",

View File

@ -54,6 +54,7 @@ type Client interface {
ListAllNamespaces(ctx context.Context) (*corev1.NamespaceList, error) ListAllNamespaces(ctx context.Context) (*corev1.NamespaceList, error)
AnnotateNode(ctx context.Context, nodeName, annotationKey, annotationValue string) error AnnotateNode(ctx context.Context, nodeName, annotationKey, annotationValue string) error
EnforceCoreDNSSpread(ctx context.Context) error EnforceCoreDNSSpread(ctx context.Context) error
PatchFirstNodePodCIDR(ctx context.Context, firstNodePodCIDR string) error
} }
type componentsInstaller interface { type componentsInstaller interface {

View File

@ -10,7 +10,6 @@ import (
"context" "context"
"net" "net"
"github.com/edgelesssys/constellation/v2/internal/deploy/helm"
"github.com/edgelesssys/constellation/v2/internal/logger" "github.com/edgelesssys/constellation/v2/internal/logger"
"github.com/edgelesssys/constellation/v2/internal/role" "github.com/edgelesssys/constellation/v2/internal/role"
"github.com/edgelesssys/constellation/v2/internal/versions/components" "github.com/edgelesssys/constellation/v2/internal/versions/components"
@ -24,10 +23,3 @@ type clusterUtil interface {
FixCilium(ctx context.Context) error FixCilium(ctx context.Context) error
StartKubelet() error StartKubelet() error
} }
// helmClient bundles functions related to microservice deployment.
// Only microservices that can be deployed purely via Helm are deployed with this interface.
type helmClient interface {
InstallChart(context.Context, helm.Release) error
InstallChartWithValues(ctx context.Context, release helm.Release, extraValues map[string]any) error
}

View File

@ -9,11 +9,8 @@ package kubernetes
import ( import (
"context" "context"
"encoding/json"
"errors"
"fmt" "fmt"
"net" "net"
"os/exec"
"regexp" "regexp"
"strings" "strings"
"time" "time"
@ -22,7 +19,6 @@ import (
"github.com/edgelesssys/constellation/v2/bootstrapper/internal/kubernetes/kubewaiter" "github.com/edgelesssys/constellation/v2/bootstrapper/internal/kubernetes/kubewaiter"
"github.com/edgelesssys/constellation/v2/internal/cloud/cloudprovider" "github.com/edgelesssys/constellation/v2/internal/cloud/cloudprovider"
"github.com/edgelesssys/constellation/v2/internal/constants" "github.com/edgelesssys/constellation/v2/internal/constants"
"github.com/edgelesssys/constellation/v2/internal/deploy/helm"
"github.com/edgelesssys/constellation/v2/internal/kubernetes" "github.com/edgelesssys/constellation/v2/internal/kubernetes"
"github.com/edgelesssys/constellation/v2/internal/logger" "github.com/edgelesssys/constellation/v2/internal/logger"
"github.com/edgelesssys/constellation/v2/internal/role" "github.com/edgelesssys/constellation/v2/internal/role"
@ -49,7 +45,6 @@ type kubeAPIWaiter interface {
type KubeWrapper struct { type KubeWrapper struct {
cloudProvider string cloudProvider string
clusterUtil clusterUtil clusterUtil clusterUtil
helmClient helmClient
kubeAPIWaiter kubeAPIWaiter kubeAPIWaiter kubeAPIWaiter
configProvider configurationProvider configProvider configurationProvider
client k8sapi.Client client k8sapi.Client
@ -59,12 +54,11 @@ type KubeWrapper struct {
// New creates a new KubeWrapper with real values. // New creates a new KubeWrapper with real values.
func New(cloudProvider string, clusterUtil clusterUtil, configProvider configurationProvider, client k8sapi.Client, func New(cloudProvider string, clusterUtil clusterUtil, configProvider configurationProvider, client k8sapi.Client,
providerMetadata ProviderMetadata, helmClient helmClient, kubeAPIWaiter kubeAPIWaiter, providerMetadata ProviderMetadata, kubeAPIWaiter kubeAPIWaiter,
) *KubeWrapper { ) *KubeWrapper {
return &KubeWrapper{ return &KubeWrapper{
cloudProvider: cloudProvider, cloudProvider: cloudProvider,
clusterUtil: clusterUtil, clusterUtil: clusterUtil,
helmClient: helmClient,
kubeAPIWaiter: kubeAPIWaiter, kubeAPIWaiter: kubeAPIWaiter,
configProvider: configProvider, configProvider: configProvider,
client: client, client: client,
@ -75,15 +69,13 @@ func New(cloudProvider string, clusterUtil clusterUtil, configProvider configura
// InitCluster initializes a new Kubernetes cluster and applies pod network provider. // InitCluster initializes a new Kubernetes cluster and applies pod network provider.
func (k *KubeWrapper) InitCluster( func (k *KubeWrapper) InitCluster(
ctx context.Context, versionString, clusterName string, ctx context.Context, versionString, clusterName string, conformanceMode bool, kubernetesComponents components.Components, apiServerCertSANs []string, log *logger.Logger,
helmReleasesRaw []byte, conformanceMode bool, kubernetesComponents components.Components, apiServerCertSANs []string, log *logger.Logger,
) ([]byte, error) { ) ([]byte, error) {
log.With(zap.String("version", versionString)).Infof("Installing Kubernetes components") log.With(zap.String("version", versionString)).Infof("Installing Kubernetes components")
if err := k.clusterUtil.InstallComponents(ctx, kubernetesComponents); err != nil { if err := k.clusterUtil.InstallComponents(ctx, kubernetesComponents); err != nil {
return nil, err return nil, err
} }
var nodePodCIDR string
var validIPs []net.IP var validIPs []net.IP
// Step 1: retrieve cloud metadata for Kubernetes configuration // Step 1: retrieve cloud metadata for Kubernetes configuration
@ -102,9 +94,6 @@ func (k *KubeWrapper) InitCluster(
nodeIP := instance.VPCIP nodeIP := instance.VPCIP
subnetworkPodCIDR := instance.SecondaryIPRange subnetworkPodCIDR := instance.SecondaryIPRange
if len(instance.AliasIPRanges) > 0 {
nodePodCIDR = instance.AliasIPRanges[0]
}
// this is the endpoint in "kubeadm init --control-plane-endpoint=<IP/DNS>:<port>" // this is the endpoint in "kubeadm init --control-plane-endpoint=<IP/DNS>:<port>"
// TODO(malt3): switch over to DNS name on AWS and Azure // TODO(malt3): switch over to DNS name on AWS and Azure
@ -170,6 +159,17 @@ func (k *KubeWrapper) InitCluster(
return nil, fmt.Errorf("failed to setup k8s version ConfigMap: %w", err) return nil, fmt.Errorf("failed to setup k8s version ConfigMap: %w", err)
} }
if cloudprovider.FromString(k.cloudProvider) == cloudprovider.GCP {
// GCP uses direct routing, so we need to set the pod CIDR of the first control-plane node for Cilium.
var nodePodCIDR string
if len(instance.AliasIPRanges) > 0 {
nodePodCIDR = instance.AliasIPRanges[0]
}
if err := k.client.PatchFirstNodePodCIDR(ctx, nodePodCIDR); err != nil {
return nil, fmt.Errorf("patching first node pod CIDR: %w", err)
}
}
// Annotate Node with the hash of the installed components // Annotate Node with the hash of the installed components
if err := k.client.AnnotateNode(ctx, nodeName, if err := k.client.AnnotateNode(ctx, nodeName,
constants.NodeKubernetesComponentsAnnotationKey, k8sComponentsConfigMap, constants.NodeKubernetesComponentsAnnotationKey, k8sComponentsConfigMap,
@ -177,49 +177,6 @@ func (k *KubeWrapper) InitCluster(
return nil, fmt.Errorf("annotating node with Kubernetes components hash: %w", err) return nil, fmt.Errorf("annotating node with Kubernetes components hash: %w", err)
} }
// Step 3: configure & start kubernetes controllers
log.Infof("Starting Kubernetes controllers and deployments")
setupPodNetworkInput := k8sapi.SetupPodNetworkInput{
CloudProvider: k.cloudProvider,
NodeName: nodeName,
FirstNodePodCIDR: nodePodCIDR,
SubnetworkPodCIDR: subnetworkPodCIDR,
LoadBalancerHost: controlPlaneHost,
LoadBalancerPort: controlPlanePort,
}
var helmReleases helm.Releases
if err := json.Unmarshal(helmReleasesRaw, &helmReleases); err != nil {
return nil, fmt.Errorf("unmarshalling helm releases: %w", err)
}
log.Infof("Installing Cilium")
ciliumVals, err := k.setupCiliumVals(ctx, setupPodNetworkInput)
if err != nil {
return nil, fmt.Errorf("setting up cilium vals: %w", err)
}
if err := k.helmClient.InstallChartWithValues(ctx, helmReleases.Cilium, ciliumVals); err != nil {
return nil, fmt.Errorf("installing cilium pod network: %w", err)
}
log.Infof("Waiting for Cilium to become healthy")
timeToStartWaiting := time.Now()
// TODO(3u13r): Reduce the timeout when we switched the package repository - this is only this high because we once
// saw polling times of ~16 minutes when hitting a slow PoP from Fastly (GitHub's / ghcr.io CDN).
waitCtx, cancel = context.WithTimeout(ctx, 20*time.Minute)
defer cancel()
if err := k.clusterUtil.WaitForCilium(waitCtx, log); err != nil {
return nil, fmt.Errorf("waiting for Cilium to become healthy: %w", err)
}
timeUntilFinishedWaiting := time.Since(timeToStartWaiting)
log.With(zap.Duration("duration", timeUntilFinishedWaiting)).Infof("Cilium became healthy")
log.Infof("Restarting Cilium")
if err := k.clusterUtil.FixCilium(ctx); err != nil {
log.With(zap.Error(err)).Errorf("FixCilium failed")
// Continue and don't throw an error here - things might be okay.
}
log.Infof("Setting up internal-config ConfigMap") log.Infof("Setting up internal-config ConfigMap")
if err := k.setupInternalConfigMap(ctx); err != nil { if err := k.setupInternalConfigMap(ctx); err != nil {
return nil, fmt.Errorf("failed to setup internal ConfigMap: %w", err) return nil, fmt.Errorf("failed to setup internal ConfigMap: %w", err)
@ -378,28 +335,3 @@ func getIPAddr() (string, error) {
return localAddr.IP.String(), nil return localAddr.IP.String(), nil
} }
func (k *KubeWrapper) setupCiliumVals(ctx context.Context, in k8sapi.SetupPodNetworkInput) (map[string]any, error) {
vals := map[string]any{
"k8sServiceHost": in.LoadBalancerHost,
"k8sServicePort": in.LoadBalancerPort,
}
// GCP requires extra configuration for Cilium
if cloudprovider.FromString(k.cloudProvider) == cloudprovider.GCP {
if out, err := exec.CommandContext(
ctx, constants.KubectlPath,
"--kubeconfig", constants.ControlPlaneAdminConfFilename,
"patch", "node", in.NodeName, "-p", "{\"spec\":{\"podCIDR\": \""+in.FirstNodePodCIDR+"\"}}",
).CombinedOutput(); err != nil {
err = errors.New(string(out))
return nil, fmt.Errorf("%s: %w", out, err)
}
vals["ipv4NativeRoutingCIDR"] = in.SubnetworkPodCIDR
vals["strictModeCIDR"] = in.SubnetworkPodCIDR
}
return vals, nil
}

View File

@ -17,7 +17,6 @@ import (
"github.com/edgelesssys/constellation/v2/bootstrapper/internal/kubernetes/kubewaiter" "github.com/edgelesssys/constellation/v2/bootstrapper/internal/kubernetes/kubewaiter"
"github.com/edgelesssys/constellation/v2/internal/cloud/metadata" "github.com/edgelesssys/constellation/v2/internal/cloud/metadata"
"github.com/edgelesssys/constellation/v2/internal/constants" "github.com/edgelesssys/constellation/v2/internal/constants"
"github.com/edgelesssys/constellation/v2/internal/deploy/helm"
"github.com/edgelesssys/constellation/v2/internal/kubernetes" "github.com/edgelesssys/constellation/v2/internal/kubernetes"
"github.com/edgelesssys/constellation/v2/internal/logger" "github.com/edgelesssys/constellation/v2/internal/logger"
"github.com/edgelesssys/constellation/v2/internal/role" "github.com/edgelesssys/constellation/v2/internal/role"
@ -43,7 +42,6 @@ func TestInitCluster(t *testing.T) {
testCases := map[string]struct { testCases := map[string]struct {
clusterUtil stubClusterUtil clusterUtil stubClusterUtil
helmClient stubHelmClient
kubectl stubKubectl kubectl stubKubectl
kubeAPIWaiter stubKubeAPIWaiter kubeAPIWaiter stubKubeAPIWaiter
providerMetadata ProviderMetadata providerMetadata ProviderMetadata
@ -131,14 +129,12 @@ func TestInitCluster(t *testing.T) {
}, },
"kubeadm init fails when deploying cilium": { "kubeadm init fails when deploying cilium": {
clusterUtil: stubClusterUtil{kubeconfig: []byte("someKubeconfig")}, clusterUtil: stubClusterUtil{kubeconfig: []byte("someKubeconfig")},
helmClient: stubHelmClient{ciliumError: assert.AnError},
providerMetadata: &stubProviderMetadata{}, providerMetadata: &stubProviderMetadata{},
wantErr: true, wantErr: true,
k8sVersion: versions.Default, k8sVersion: versions.Default,
}, },
"kubeadm init fails when setting up constellation-services chart": { "kubeadm init fails when setting up constellation-services chart": {
clusterUtil: stubClusterUtil{kubeconfig: []byte("someKubeconfig")}, clusterUtil: stubClusterUtil{kubeconfig: []byte("someKubeconfig")},
helmClient: stubHelmClient{installChartError: assert.AnError},
kubeAPIWaiter: stubKubeAPIWaiter{}, kubeAPIWaiter: stubKubeAPIWaiter{},
providerMetadata: &stubProviderMetadata{}, providerMetadata: &stubProviderMetadata{},
wantErr: true, wantErr: true,
@ -182,7 +178,6 @@ func TestInitCluster(t *testing.T) {
kube := KubeWrapper{ kube := KubeWrapper{
cloudProvider: "aws", // provide a valid cloud provider for cilium installation cloudProvider: "aws", // provide a valid cloud provider for cilium installation
clusterUtil: &tc.clusterUtil, clusterUtil: &tc.clusterUtil,
helmClient: &tc.helmClient,
providerMetadata: tc.providerMetadata, providerMetadata: tc.providerMetadata,
kubeAPIWaiter: &tc.kubeAPIWaiter, kubeAPIWaiter: &tc.kubeAPIWaiter,
configProvider: &stubConfigProvider{initConfig: k8sapi.KubeadmInitYAML{}}, configProvider: &stubConfigProvider{initConfig: k8sapi.KubeadmInitYAML{}},
@ -192,7 +187,7 @@ func TestInitCluster(t *testing.T) {
_, err := kube.InitCluster( _, err := kube.InitCluster(
context.Background(), string(tc.k8sVersion), "kubernetes", context.Background(), string(tc.k8sVersion), "kubernetes",
[]byte("{}"), false, nil, nil, logger.NewTest(t), false, nil, nil, logger.NewTest(t),
) )
if tc.wantErr { if tc.wantErr {
@ -539,6 +534,10 @@ func (s *stubKubectl) AnnotateNode(_ context.Context, _, _, _ string) error {
return s.annotateNodeErr return s.annotateNodeErr
} }
func (s *stubKubectl) PatchFirstNodePodCIDR(_ context.Context, _ string) error {
return nil
}
func (s *stubKubectl) WaitForCRDs(_ context.Context, _ []string) error { func (s *stubKubectl) WaitForCRDs(_ context.Context, _ []string) error {
return s.waitForCRDsErr return s.waitForCRDsErr
} }
@ -551,24 +550,6 @@ func (s *stubKubectl) EnforceCoreDNSSpread(_ context.Context) error {
return s.enforceCoreDNSSpreadErr return s.enforceCoreDNSSpreadErr
} }
type stubHelmClient struct {
ciliumError error
installChartError error
}
func (s *stubHelmClient) InstallChart(ctx context.Context, release helm.Release) error {
return s.InstallChartWithValues(ctx, release, release.Values)
}
func (s *stubHelmClient) InstallChartWithValues(_ context.Context, release helm.Release, _ map[string]any) error {
switch release.ReleaseName {
case "cilium":
return s.ciliumError
default:
return s.installChartError
}
}
type stubKubeAPIWaiter struct { type stubKubeAPIWaiter struct {
waitErr error waitErr error
} }

View File

@ -67,7 +67,6 @@ go_library(
"//internal/config/migration", "//internal/config/migration",
"//internal/constants", "//internal/constants",
"//internal/crypto", "//internal/crypto",
"//internal/deploy/helm",
"//internal/file", "//internal/file",
"//internal/grpc/dialer", "//internal/grpc/dialer",
"//internal/grpc/grpclog", "//internal/grpc/grpclog",
@ -139,6 +138,7 @@ go_test(
"//bootstrapper/initproto", "//bootstrapper/initproto",
"//cli/internal/cloudcmd", "//cli/internal/cloudcmd",
"//cli/internal/clusterid", "//cli/internal/clusterid",
"//cli/internal/helm",
"//cli/internal/iamid", "//cli/internal/iamid",
"//cli/internal/kubernetes", "//cli/internal/kubernetes",
"//cli/internal/terraform", "//cli/internal/terraform",
@ -154,7 +154,6 @@ go_test(
"//internal/config", "//internal/config",
"//internal/constants", "//internal/constants",
"//internal/crypto/testvector", "//internal/crypto/testvector",
"//internal/deploy/helm",
"//internal/file", "//internal/file",
"//internal/grpc/atlscredentials", "//internal/grpc/atlscredentials",
"//internal/grpc/dialer", "//internal/grpc/dialer",

View File

@ -10,7 +10,6 @@ import (
"bytes" "bytes"
"context" "context"
"encoding/hex" "encoding/hex"
"encoding/json"
"errors" "errors"
"fmt" "fmt"
"io" "io"
@ -45,7 +44,6 @@ import (
"github.com/edgelesssys/constellation/v2/internal/config" "github.com/edgelesssys/constellation/v2/internal/config"
"github.com/edgelesssys/constellation/v2/internal/constants" "github.com/edgelesssys/constellation/v2/internal/constants"
"github.com/edgelesssys/constellation/v2/internal/crypto" "github.com/edgelesssys/constellation/v2/internal/crypto"
helmdeploy "github.com/edgelesssys/constellation/v2/internal/deploy/helm"
"github.com/edgelesssys/constellation/v2/internal/file" "github.com/edgelesssys/constellation/v2/internal/file"
"github.com/edgelesssys/constellation/v2/internal/grpc/dialer" "github.com/edgelesssys/constellation/v2/internal/grpc/dialer"
"github.com/edgelesssys/constellation/v2/internal/grpc/grpclog" "github.com/edgelesssys/constellation/v2/internal/grpc/grpclog"
@ -79,7 +77,7 @@ type initCmd struct {
spinner spinnerInterf spinner spinnerInterf
masterSecret uri.MasterSecret masterSecret uri.MasterSecret
fh *file.Handler fh *file.Handler
helmInstaller helm.SuiteInstaller helmInstaller helm.Initializer
} }
// runInitialize runs the initialize command. // runInitialize runs the initialize command.
@ -103,7 +101,7 @@ func runInitialize(cmd *cobra.Command, _ []string) error {
ctx, cancel := context.WithTimeout(cmd.Context(), time.Hour) ctx, cancel := context.WithTimeout(cmd.Context(), time.Hour)
defer cancel() defer cancel()
cmd.SetContext(ctx) cmd.SetContext(ctx)
helmInstaller, err := helm.NewInstallationClient(log) helmInstaller, err := helm.NewInitializer(log)
if err != nil { if err != nil {
return fmt.Errorf("creating Helm installer: %w", err) return fmt.Errorf("creating Helm installer: %w", err)
} }
@ -191,10 +189,6 @@ func (i *initCmd) initialize(cmd *cobra.Command, newDialer func(validator atls.V
if err != nil { if err != nil {
return fmt.Errorf("loading Helm charts: %w", err) return fmt.Errorf("loading Helm charts: %w", err)
} }
helmDeployments, err := json.Marshal(releases)
if err != nil {
return err
}
i.log.Debugf("Loaded Helm deployments") i.log.Debugf("Loaded Helm deployments")
if err != nil { if err != nil {
return fmt.Errorf("loading Helm charts: %w", err) return fmt.Errorf("loading Helm charts: %w", err)
@ -208,7 +202,6 @@ func (i *initCmd) initialize(cmd *cobra.Command, newDialer func(validator atls.V
CloudServiceAccountUri: serviceAccURI, CloudServiceAccountUri: serviceAccURI,
KubernetesVersion: versions.VersionConfigs[k8sVersion].ClusterVersion, KubernetesVersion: versions.VersionConfigs[k8sVersion].ClusterVersion,
KubernetesComponents: versions.VersionConfigs[k8sVersion].KubernetesComponents.ToInitProto(), KubernetesComponents: versions.VersionConfigs[k8sVersion].KubernetesComponents.ToInitProto(),
HelmDeployments: helmDeployments,
ConformanceMode: flags.conformance, ConformanceMode: flags.conformance,
InitSecret: idFile.InitSecret, InitSecret: idFile.InitSecret,
ClusterName: clusterName, ClusterName: clusterName,
@ -441,9 +434,9 @@ func (i *initCmd) evalFlagArgs(cmd *cobra.Command) (initFlags, error) {
if err != nil { if err != nil {
return initFlags{}, fmt.Errorf("parsing skip-helm-wait flag: %w", err) return initFlags{}, fmt.Errorf("parsing skip-helm-wait flag: %w", err)
} }
helmWaitMode := helmdeploy.WaitModeAtomic helmWaitMode := helm.WaitModeAtomic
if skipHelmWait { if skipHelmWait {
helmWaitMode = helmdeploy.WaitModeNone helmWaitMode = helm.WaitModeNone
} }
i.log.Debugf("Helm wait flag is %t", skipHelmWait) i.log.Debugf("Helm wait flag is %t", skipHelmWait)
configPath, err := cmd.Flags().GetString("config") configPath, err := cmd.Flags().GetString("config")
@ -478,7 +471,7 @@ type initFlags struct {
configPath string configPath string
masterSecretPath string masterSecretPath string
conformance bool conformance bool
helmWaitMode helmdeploy.WaitMode helmWaitMode helm.WaitMode
force bool force bool
mergeConfigs bool mergeConfigs bool
} }

View File

@ -21,6 +21,7 @@ import (
"github.com/edgelesssys/constellation/v2/bootstrapper/initproto" "github.com/edgelesssys/constellation/v2/bootstrapper/initproto"
"github.com/edgelesssys/constellation/v2/cli/internal/clusterid" "github.com/edgelesssys/constellation/v2/cli/internal/clusterid"
"github.com/edgelesssys/constellation/v2/cli/internal/helm"
"github.com/edgelesssys/constellation/v2/internal/atls" "github.com/edgelesssys/constellation/v2/internal/atls"
"github.com/edgelesssys/constellation/v2/internal/attestation/measurements" "github.com/edgelesssys/constellation/v2/internal/attestation/measurements"
"github.com/edgelesssys/constellation/v2/internal/attestation/variant" "github.com/edgelesssys/constellation/v2/internal/attestation/variant"
@ -28,7 +29,6 @@ import (
"github.com/edgelesssys/constellation/v2/internal/cloud/gcpshared" "github.com/edgelesssys/constellation/v2/internal/cloud/gcpshared"
"github.com/edgelesssys/constellation/v2/internal/config" "github.com/edgelesssys/constellation/v2/internal/config"
"github.com/edgelesssys/constellation/v2/internal/constants" "github.com/edgelesssys/constellation/v2/internal/constants"
helminstaller "github.com/edgelesssys/constellation/v2/internal/deploy/helm"
"github.com/edgelesssys/constellation/v2/internal/file" "github.com/edgelesssys/constellation/v2/internal/file"
"github.com/edgelesssys/constellation/v2/internal/grpc/atlscredentials" "github.com/edgelesssys/constellation/v2/internal/grpc/atlscredentials"
"github.com/edgelesssys/constellation/v2/internal/grpc/dialer" "github.com/edgelesssys/constellation/v2/internal/grpc/dialer"
@ -672,7 +672,7 @@ type stubHelmInstaller struct{}
func (i *stubHelmInstaller) Install(_ context.Context, _ cloudprovider.Provider, _ uri.MasterSecret, func (i *stubHelmInstaller) Install(_ context.Context, _ cloudprovider.Provider, _ uri.MasterSecret,
_ clusterid.File, _ clusterid.File,
_ string, _ *helminstaller.Releases, _ string, _ *helm.Releases,
) error { ) error {
return nil return nil
} }

View File

@ -74,7 +74,7 @@ func runStatus(cmd *cobra.Command, _ []string) error {
} }
// need helm client to fetch service versions. // need helm client to fetch service versions.
helmClient, err := helm.NewClient(kubectl.New(), constants.AdminConfFilename, constants.HelmNamespace, log) helmClient, err := helm.NewUpgradeClient(kubectl.New(), constants.AdminConfFilename, constants.HelmNamespace, log)
if err != nil { if err != nil {
return fmt.Errorf("setting up helm client: %w", err) return fmt.Errorf("setting up helm client: %w", err)
} }

View File

@ -383,7 +383,7 @@ type currentVersionInfo struct {
} }
func (v *versionCollector) currentVersions(ctx context.Context) (currentVersionInfo, error) { func (v *versionCollector) currentVersions(ctx context.Context) (currentVersionInfo, error) {
helmClient, err := helm.NewClient(kubectl.New(), constants.AdminConfFilename, constants.HelmNamespace, v.log) helmClient, err := helm.NewUpgradeClient(kubectl.New(), constants.AdminConfFilename, constants.HelmNamespace, v.log)
if err != nil { if err != nil {
return currentVersionInfo{}, fmt.Errorf("setting up helm client: %w", err) return currentVersionInfo{}, fmt.Errorf("setting up helm client: %w", err)
} }

View File

@ -5,12 +5,14 @@ go_library(
name = "helm", name = "helm",
srcs = [ srcs = [
"backup.go", "backup.go",
"client.go", "ciliumhelper.go",
"helm.go", "helm.go",
"helminstaller.go", "init.go",
"install.go",
"loader.go", "loader.go",
"release.go",
"serviceversion.go", "serviceversion.go",
"setup.go", "upgrade.go",
"values.go", "values.go",
], ],
embedsrcs = [ embedsrcs = [
@ -427,17 +429,21 @@ go_library(
"//internal/compatibility", "//internal/compatibility",
"//internal/config", "//internal/config",
"//internal/constants", "//internal/constants",
"//internal/deploy/helm",
"//internal/file", "//internal/file",
"//internal/kms/uri", "//internal/kms/uri",
"//internal/retry",
"//internal/semver", "//internal/semver",
"//internal/versions", "//internal/versions",
"@com_github_pkg_errors//:errors", "@com_github_pkg_errors//:errors",
"@com_github_spf13_afero//:afero", "@com_github_spf13_afero//:afero",
"@io_k8s_apiextensions_apiserver//pkg/apis/apiextensions/v1:apiextensions", "@io_k8s_apiextensions_apiserver//pkg/apis/apiextensions/v1:apiextensions",
"@io_k8s_apimachinery//pkg/api/errors", "@io_k8s_apimachinery//pkg/api/errors",
"@io_k8s_apimachinery//pkg/apis/meta/v1:meta",
"@io_k8s_apimachinery//pkg/apis/meta/v1/unstructured", "@io_k8s_apimachinery//pkg/apis/meta/v1/unstructured",
"@io_k8s_apimachinery//pkg/runtime/schema", "@io_k8s_apimachinery//pkg/runtime/schema",
"@io_k8s_apimachinery//pkg/util/wait",
"@io_k8s_client_go//kubernetes",
"@io_k8s_client_go//tools/clientcmd",
"@io_k8s_sigs_yaml//:yaml", "@io_k8s_sigs_yaml//:yaml",
"@sh_helm_helm//pkg/ignore", "@sh_helm_helm//pkg/ignore",
"@sh_helm_helm_v3//pkg/action", "@sh_helm_helm_v3//pkg/action",
@ -453,8 +459,9 @@ go_test(
name = "helm_test", name = "helm_test",
srcs = [ srcs = [
"backup_test.go", "backup_test.go",
"client_test.go", "helm_test.go",
"loader_test.go", "loader_test.go",
"upgrade_test.go",
], ],
data = glob(["testdata/**"]), data = glob(["testdata/**"]),
embed = [":helm"], embed = [":helm"],
@ -465,7 +472,6 @@ go_test(
"//internal/cloud/cloudprovider", "//internal/cloud/cloudprovider",
"//internal/compatibility", "//internal/compatibility",
"//internal/config", "//internal/config",
"//internal/deploy/helm",
"//internal/file", "//internal/file",
"//internal/logger", "//internal/logger",
"//internal/semver", "//internal/semver",

View File

@ -18,7 +18,7 @@ import (
"sigs.k8s.io/yaml" "sigs.k8s.io/yaml"
) )
func (c *Client) backupCRDs(ctx context.Context, upgradeID string) ([]apiextensionsv1.CustomResourceDefinition, error) { func (c *UpgradeClient) backupCRDs(ctx context.Context, upgradeID string) ([]apiextensionsv1.CustomResourceDefinition, error) {
c.log.Debugf("Starting CRD backup") c.log.Debugf("Starting CRD backup")
crds, err := c.kubectl.GetCRDs(ctx) crds, err := c.kubectl.GetCRDs(ctx)
if err != nil { if err != nil {
@ -53,7 +53,7 @@ func (c *Client) backupCRDs(ctx context.Context, upgradeID string) ([]apiextensi
return crds, nil return crds, nil
} }
func (c *Client) backupCRs(ctx context.Context, crds []apiextensionsv1.CustomResourceDefinition, upgradeID string) error { func (c *UpgradeClient) backupCRs(ctx context.Context, crds []apiextensionsv1.CustomResourceDefinition, upgradeID string) error {
c.log.Debugf("Starting CR backup") c.log.Debugf("Starting CR backup")
for _, crd := range crds { for _, crd := range crds {
c.log.Debugf("Creating backup for resource type: %s", crd.Name) c.log.Debugf("Creating backup for resource type: %s", crd.Name)
@ -99,10 +99,10 @@ func (c *Client) backupCRs(ctx context.Context, crds []apiextensionsv1.CustomRes
return nil return nil
} }
func (c *Client) backupFolder(upgradeID string) string { func (c *UpgradeClient) backupFolder(upgradeID string) string {
return filepath.Join(constants.UpgradeDir, upgradeID, "backups") + string(filepath.Separator) return filepath.Join(constants.UpgradeDir, upgradeID, "backups") + string(filepath.Separator)
} }
func (c *Client) crdBackupFolder(upgradeID string) string { func (c *UpgradeClient) crdBackupFolder(upgradeID string) string {
return filepath.Join(c.backupFolder(upgradeID), "crds") + string(filepath.Separator) return filepath.Join(c.backupFolder(upgradeID), "crds") + string(filepath.Separator)
} }

View File

@ -52,7 +52,7 @@ func TestBackupCRDs(t *testing.T) {
crd := apiextensionsv1.CustomResourceDefinition{} crd := apiextensionsv1.CustomResourceDefinition{}
err := yaml.Unmarshal([]byte(tc.crd), &crd) err := yaml.Unmarshal([]byte(tc.crd), &crd)
require.NoError(err) require.NoError(err)
client := Client{ client := UpgradeClient{
config: nil, config: nil,
kubectl: stubCrdClient{crds: []apiextensionsv1.CustomResourceDefinition{crd}, getCRDsError: tc.getCRDsError}, kubectl: stubCrdClient{crds: []apiextensionsv1.CustomResourceDefinition{crd}, getCRDsError: tc.getCRDsError},
fs: file.NewHandler(memFs), fs: file.NewHandler(memFs),
@ -143,7 +143,7 @@ func TestBackupCRs(t *testing.T) {
require := require.New(t) require := require.New(t)
memFs := afero.NewMemMapFs() memFs := afero.NewMemMapFs()
client := Client{ client := UpgradeClient{
config: nil, config: nil,
kubectl: stubCrdClient{crs: []unstructured.Unstructured{tc.resource}, getCRsError: tc.getCRsError}, kubectl: stubCrdClient{crs: []unstructured.Unstructured{tc.resource}, getCRsError: tc.getCRsError},
fs: file.NewHandler(memFs), fs: file.NewHandler(memFs),

View File

@ -0,0 +1,72 @@
/*
Copyright (c) Edgeless Systems GmbH
SPDX-License-Identifier: AGPL-3.0-only
*/
package helm
import (
"context"
"fmt"
"time"
v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/tools/clientcmd"
)
type k8sDsClient struct {
clientset *kubernetes.Clientset
}
func newK8sCiliumHelper(kubeconfigPath string) (*k8sDsClient, error) {
config, err := clientcmd.BuildConfigFromFlags("", kubeconfigPath)
if err != nil {
return nil, err
}
clientset, err := kubernetes.NewForConfig(config)
if err != nil {
return nil, err
}
return &k8sDsClient{clientset: clientset}, nil
}
// WaitForDS waits for a DaemonSet to become ready.
func (h *k8sDsClient) WaitForDS(ctx context.Context, namespace, name string, log debugLog) error {
for {
select {
case <-ctx.Done():
return fmt.Errorf("context expired before DaemonSet %q became ready", name)
default:
ds, err := h.clientset.AppsV1().DaemonSets(namespace).Get(ctx, name, v1.GetOptions{})
if err != nil {
return err
}
if ds.Status.NumberReady == ds.Status.DesiredNumberScheduled {
log.Debugf("DaemonSet %s is ready\n", name)
return nil
}
log.Debugf("Waiting for DaemonSet %s to become ready...\n", name)
time.Sleep(10 * time.Second)
}
}
}
// RestartDS restarts all pods of a DaemonSet by updating its template.
func (h *k8sDsClient) RestartDS(namespace, name string) error {
ds, err := h.clientset.AppsV1().DaemonSets(namespace).Get(context.Background(), name, v1.GetOptions{})
if err != nil {
return err
}
ds.Spec.Template.ObjectMeta.Annotations["restartTimestamp"] = fmt.Sprintf("%d", time.Now().Unix())
_, err = h.clientset.AppsV1().DaemonSets(namespace).Update(context.Background(), ds, v1.UpdateOptions{})
if err != nil {
return err
}
return nil
}

View File

@ -15,3 +15,25 @@ It is used by the CLI to:
- create local backups before running service upgrades - create local backups before running service upgrades
*/ */
package helm package helm
// mergeMaps returns a new map that is the merger of it's inputs.
// Key collisions are resolved by taking the value of the second argument (map b).
// Taken from: https://github.com/helm/helm/blob/dbc6d8e20fe1d58d50e6ed30f09a04a77e4c68db/pkg/cli/values/options.go#L91-L108.
func mergeMaps(a, b map[string]any) map[string]any {
out := make(map[string]any, len(a))
for k, v := range a {
out[k] = v
}
for k, v := range b {
if v, ok := v.(map[string]any); ok {
if bv, ok := out[k]; ok {
if bv, ok := bv.(map[string]any); ok {
out[k] = mergeMaps(bv, v)
continue
}
}
}
out[k] = v
}
return out
}

View File

@ -102,7 +102,7 @@ func TestMergeMaps(t *testing.T) {
for name, tc := range testCases { for name, tc := range testCases {
t.Run(name, func(t *testing.T) { t.Run(name, func(t *testing.T) {
assert := assert.New(t) assert := assert.New(t)
newVals := MergeMaps(tc.vals, tc.extraVals) newVals := mergeMaps(tc.vals, tc.extraVals)
assert.Equal(tc.expected, newVals) assert.Equal(tc.expected, newVals)
}) })
} }

View File

@ -1,100 +0,0 @@
/*
Copyright (c) Edgeless Systems GmbH
SPDX-License-Identifier: AGPL-3.0-only
*/
package helm
import (
"context"
"fmt"
"github.com/edgelesssys/constellation/v2/cli/internal/clusterid"
"github.com/edgelesssys/constellation/v2/internal/cloud/cloudprovider"
"github.com/edgelesssys/constellation/v2/internal/cloud/openstack"
"github.com/edgelesssys/constellation/v2/internal/constants"
helminstaller "github.com/edgelesssys/constellation/v2/internal/deploy/helm"
"github.com/edgelesssys/constellation/v2/internal/kms/uri"
)
// SuiteInstaller installs all Helm charts required for a constellation cluster.
type SuiteInstaller interface {
Install(ctx context.Context, provider cloudprovider.Provider, masterSecret uri.MasterSecret,
idFile clusterid.File,
serviceAccURI string, releases *helminstaller.Releases,
) error
}
type helmInstallationClient struct {
log debugLog
installer helmInstaller
}
// NewInstallationClient creates a new Helm installation client to install all Helm charts required for a constellation cluster.
func NewInstallationClient(log debugLog) (SuiteInstaller, error) {
installer, err := helminstaller.NewInstaller(constants.AdminConfFilename, log)
if err != nil {
return nil, fmt.Errorf("creating Helm installer: %w", err)
}
return &helmInstallationClient{log: log, installer: installer}, nil
}
func (h helmInstallationClient) Install(ctx context.Context, provider cloudprovider.Provider, masterSecret uri.MasterSecret,
idFile clusterid.File,
serviceAccURI string, releases *helminstaller.Releases,
) error {
serviceVals, err := setupMicroserviceVals(ctx, provider, masterSecret.Salt, idFile.UID, serviceAccURI)
if err != nil {
return fmt.Errorf("setting up microservice values: %w", err)
}
if err := h.installer.InstallChartWithValues(ctx, releases.ConstellationServices, serviceVals); err != nil {
return fmt.Errorf("installing microservices: %w", err)
}
h.log.Debugf("Installing cert-manager")
if err := h.installer.InstallChart(ctx, releases.CertManager); err != nil {
return fmt.Errorf("installing cert-manager: %w", err)
}
if releases.CSI != nil {
var csiVals map[string]any
if provider == cloudprovider.OpenStack {
creds, err := openstack.AccountKeyFromURI(serviceAccURI)
if err != nil {
return err
}
cinderIni := creds.CloudINI().CinderCSIConfiguration()
csiVals = map[string]any{
"cinder-config": map[string]any{
"secretData": cinderIni,
},
}
}
h.log.Debugf("Installing CSI deployments")
if err := h.installer.InstallChartWithValues(ctx, *releases.CSI, csiVals); err != nil {
return fmt.Errorf("installing CSI snapshot CRDs: %w", err)
}
}
if releases.AWSLoadBalancerController != nil {
h.log.Debugf("Installing AWS Load Balancer Controller")
if err := h.installer.InstallChart(ctx, *releases.AWSLoadBalancerController); err != nil {
return fmt.Errorf("installing AWS Load Balancer Controller: %w", err)
}
}
h.log.Debugf("Installing constellation operators")
operatorVals := setupOperatorVals(ctx, idFile.UID)
if err := h.installer.InstallChartWithValues(ctx, releases.ConstellationOperators, operatorVals); err != nil {
return fmt.Errorf("installing constellation operators: %w", err)
}
// TODO(elchead): AB#3301 do cilium after version upgrade
return nil
}
type helmInstaller interface {
InstallChart(context.Context, helminstaller.Release) error
InstallChartWithValues(ctx context.Context, release helminstaller.Release, extraValues map[string]any) error
}

258
cli/internal/helm/init.go Normal file
View File

@ -0,0 +1,258 @@
/*
Copyright (c) Edgeless Systems GmbH
SPDX-License-Identifier: AGPL-3.0-only
*/
package helm
import (
"context"
"encoding/base64"
"encoding/json"
"fmt"
"time"
"github.com/edgelesssys/constellation/v2/cli/internal/clusterid"
"github.com/edgelesssys/constellation/v2/cli/internal/terraform"
"github.com/edgelesssys/constellation/v2/internal/cloud/azureshared"
"github.com/edgelesssys/constellation/v2/internal/cloud/cloudprovider"
"github.com/edgelesssys/constellation/v2/internal/cloud/gcpshared"
"github.com/edgelesssys/constellation/v2/internal/cloud/openstack"
"github.com/edgelesssys/constellation/v2/internal/constants"
"github.com/edgelesssys/constellation/v2/internal/kms/uri"
)
// Initializer installs all Helm charts required for a constellation cluster.
type Initializer interface {
Install(ctx context.Context, provider cloudprovider.Provider, masterSecret uri.MasterSecret,
idFile clusterid.File,
serviceAccURI string, releases *Releases,
) error
}
type initializationClient struct {
log debugLog
installer installer
}
// NewInitializer creates a new client to install all Helm charts required for a constellation cluster.
func NewInitializer(log debugLog) (Initializer, error) {
installer, err := NewInstaller(constants.AdminConfFilename, log)
if err != nil {
return nil, fmt.Errorf("creating Helm installer: %w", err)
}
return &initializationClient{log: log, installer: installer}, nil
}
// Install installs all Helm charts required for a constellation cluster.
func (h initializationClient) Install(ctx context.Context, provider cloudprovider.Provider, masterSecret uri.MasterSecret,
idFile clusterid.File,
serviceAccURI string, releases *Releases,
) error {
tfClient, err := terraform.New(ctx, constants.TerraformWorkingDir)
if err != nil {
return fmt.Errorf("creating Terraform client: %w", err)
}
output, err := tfClient.ShowCluster(ctx, provider)
if err != nil {
return fmt.Errorf("getting Terraform output: %w", err)
}
ciliumVals := setupCiliumVals(provider, output)
if err := h.installer.InstallChartWithValues(ctx, releases.Cilium, ciliumVals); err != nil {
return fmt.Errorf("installing Cilium: %w", err)
}
h.log.Debugf("Waiting for Cilium to become ready")
helper, err := newK8sCiliumHelper(constants.AdminConfFilename)
if err != nil {
return fmt.Errorf("creating Kubernetes client: %w", err)
}
timeToStartWaiting := time.Now()
// TODO(3u13r): Reduce the timeout when we switched the package repository - this is only this high because we once
// saw polling times of ~16 minutes when hitting a slow PoP from Fastly (GitHub's / ghcr.io CDN).
if err := helper.WaitForDS(ctx, "kube-system", "cilium", h.log); err != nil {
return fmt.Errorf("waiting for Cilium to become healthy: %w", err)
}
timeUntilFinishedWaiting := time.Since(timeToStartWaiting)
h.log.Debugf("Cilium became healthy after %s", timeUntilFinishedWaiting.String())
h.log.Debugf("Fix Cilium through restart")
if err := helper.RestartDS("kube-system", "cilium"); err != nil {
return fmt.Errorf("restarting Cilium: %w", err)
}
h.log.Debugf("Installing microservices")
serviceVals, err := setupMicroserviceVals(provider, masterSecret.Salt, idFile.UID, serviceAccURI, output)
if err != nil {
return fmt.Errorf("setting up microservice values: %w", err)
}
if err := h.installer.InstallChartWithValues(ctx, releases.ConstellationServices, serviceVals); err != nil {
return fmt.Errorf("installing microservices: %w", err)
}
h.log.Debugf("Installing cert-manager")
if err := h.installer.InstallChart(ctx, releases.CertManager); err != nil {
return fmt.Errorf("installing cert-manager: %w", err)
}
if releases.CSI != nil {
var csiVals map[string]any
if provider == cloudprovider.OpenStack {
creds, err := openstack.AccountKeyFromURI(serviceAccURI)
if err != nil {
return err
}
cinderIni := creds.CloudINI().CinderCSIConfiguration()
csiVals = map[string]any{
"cinder-config": map[string]any{
"secretData": cinderIni,
},
}
}
h.log.Debugf("Installing CSI deployments")
if err := h.installer.InstallChartWithValues(ctx, *releases.CSI, csiVals); err != nil {
return fmt.Errorf("installing CSI snapshot CRDs: %w", err)
}
}
if releases.AWSLoadBalancerController != nil {
h.log.Debugf("Installing AWS Load Balancer Controller")
if err := h.installer.InstallChart(ctx, *releases.AWSLoadBalancerController); err != nil {
return fmt.Errorf("installing AWS Load Balancer Controller: %w", err)
}
}
h.log.Debugf("Installing constellation operators")
operatorVals := setupOperatorVals(ctx, idFile.UID)
if err := h.installer.InstallChartWithValues(ctx, releases.ConstellationOperators, operatorVals); err != nil {
return fmt.Errorf("installing constellation operators: %w", err)
}
return nil
}
// installer is the interface for installing a single Helm chart.
type installer interface {
InstallChart(context.Context, Release) error
InstallChartWithValues(ctx context.Context, release Release, extraValues map[string]any) error
}
// TODO(malt3): switch over to DNS name on AWS and Azure
// soon as every apiserver certificate of every control-plane node
// has the dns endpoint in its SAN list.
func setupCiliumVals(provider cloudprovider.Provider, output terraform.ApplyOutput) map[string]any {
vals := map[string]any{
"k8sServiceHost": output.IP,
"k8sServicePort": constants.KubernetesPort,
}
if provider == cloudprovider.GCP {
vals["ipv4NativeRoutingCIDR"] = output.GCP.IPCidrPod
vals["strictModeCIDR"] = output.GCP.IPCidrPod
}
return vals
}
// setupMicroserviceVals returns the values for the microservice chart.
func setupMicroserviceVals(provider cloudprovider.Provider, measurementSalt []byte, uid, serviceAccURI string, output terraform.ApplyOutput) (map[string]any, error) {
extraVals := map[string]any{
"join-service": map[string]any{
"measurementSalt": base64.StdEncoding.EncodeToString(measurementSalt),
},
"verification-service": map[string]any{
"loadBalancerIP": output.IP,
},
"konnectivity": map[string]any{
"loadBalancerIP": output.IP,
},
}
switch provider {
case cloudprovider.GCP:
serviceAccountKey, err := gcpshared.ServiceAccountKeyFromURI(serviceAccURI)
if err != nil {
return nil, fmt.Errorf("getting service account key: %w", err)
}
rawKey, err := json.Marshal(serviceAccountKey)
if err != nil {
return nil, fmt.Errorf("marshaling service account key: %w", err)
}
if output.GCP == nil {
return nil, fmt.Errorf("no GCP output from Terraform")
}
extraVals["ccm"] = map[string]any{
"GCP": map[string]any{
"projectID": output.GCP.ProjectID,
"uid": uid,
"secretData": string(rawKey),
"subnetworkPodCIDR": output.GCP.IPCidrPod,
},
}
case cloudprovider.Azure:
if output.Azure == nil {
return nil, fmt.Errorf("no Azure output from Terraform")
}
ccmConfig, err := getCCMConfig(*output.Azure, serviceAccURI)
if err != nil {
return nil, fmt.Errorf("getting Azure CCM config: %w", err)
}
extraVals["ccm"] = map[string]any{
"Azure": map[string]any{
"azureConfig": string(ccmConfig),
},
}
}
return extraVals, nil
}
// setupOperatorVals returns the values for the constellation-operator chart.
func setupOperatorVals(_ context.Context, uid string) map[string]any {
return map[string]any{
"constellation-operator": map[string]any{
"constellationUID": uid,
},
}
}
type cloudConfig struct {
Cloud string `json:"cloud,omitempty"`
TenantID string `json:"tenantId,omitempty"`
SubscriptionID string `json:"subscriptionId,omitempty"`
ResourceGroup string `json:"resourceGroup,omitempty"`
Location string `json:"location,omitempty"`
SubnetName string `json:"subnetName,omitempty"`
SecurityGroupName string `json:"securityGroupName,omitempty"`
SecurityGroupResourceGroup string `json:"securityGroupResourceGroup,omitempty"`
LoadBalancerName string `json:"loadBalancerName,omitempty"`
LoadBalancerSku string `json:"loadBalancerSku,omitempty"`
VNetName string `json:"vnetName,omitempty"`
VNetResourceGroup string `json:"vnetResourceGroup,omitempty"`
CloudProviderBackoff bool `json:"cloudProviderBackoff,omitempty"`
UseInstanceMetadata bool `json:"useInstanceMetadata,omitempty"`
VMType string `json:"vmType,omitempty"`
UseManagedIdentityExtension bool `json:"useManagedIdentityExtension,omitempty"`
UserAssignedIdentityID string `json:"userAssignedIdentityID,omitempty"`
}
// GetCCMConfig returns the configuration needed for the Kubernetes Cloud Controller Manager on Azure.
func getCCMConfig(tfOutput terraform.AzureApplyOutput, serviceAccURI string) ([]byte, error) {
creds, err := azureshared.ApplicationCredentialsFromURI(serviceAccURI)
if err != nil {
return nil, fmt.Errorf("getting service account key: %w", err)
}
useManagedIdentityExtension := creds.PreferredAuthMethod == azureshared.AuthMethodUserAssignedIdentity
config := cloudConfig{
Cloud: "AzurePublicCloud",
TenantID: creds.TenantID,
SubscriptionID: tfOutput.SubscriptionID,
ResourceGroup: tfOutput.ResourceGroup,
LoadBalancerSku: "standard",
SecurityGroupName: tfOutput.NetworkSecurityGroupName,
LoadBalancerName: tfOutput.LoadBalancerName,
UseInstanceMetadata: true,
VMType: "vmss",
Location: creds.Location,
UseManagedIdentityExtension: useManagedIdentityExtension,
UserAssignedIdentityID: tfOutput.UserAssignedIdentity,
}
return json.Marshal(config)
}

View File

@ -68,7 +68,7 @@ func (h *Installer) InstallChart(ctx context.Context, release Release) error {
// InstallChartWithValues is the generic install function for helm charts with custom values. // InstallChartWithValues is the generic install function for helm charts with custom values.
func (h *Installer) InstallChartWithValues(ctx context.Context, release Release, extraValues map[string]any) error { func (h *Installer) InstallChartWithValues(ctx context.Context, release Release, extraValues map[string]any) error {
mergedVals := MergeMaps(release.Values, extraValues) mergedVals := mergeMaps(release.Values, extraValues)
h.ReleaseName = release.ReleaseName h.ReleaseName = release.ReleaseName
if err := h.SetWaitMode(release.WaitMode); err != nil { if err := h.SetWaitMode(release.WaitMode); err != nil {
return err return err

View File

@ -27,7 +27,6 @@ import (
"github.com/edgelesssys/constellation/v2/internal/cloud/cloudprovider" "github.com/edgelesssys/constellation/v2/internal/cloud/cloudprovider"
"github.com/edgelesssys/constellation/v2/internal/config" "github.com/edgelesssys/constellation/v2/internal/config"
"github.com/edgelesssys/constellation/v2/internal/constants" "github.com/edgelesssys/constellation/v2/internal/constants"
"github.com/edgelesssys/constellation/v2/internal/deploy/helm"
"github.com/edgelesssys/constellation/v2/internal/semver" "github.com/edgelesssys/constellation/v2/internal/semver"
"github.com/edgelesssys/constellation/v2/internal/versions" "github.com/edgelesssys/constellation/v2/internal/versions"
) )
@ -108,21 +107,8 @@ func NewLoader(csp cloudprovider.Provider, k8sVersion versions.ValidK8sVersion,
} }
} }
// Load the embedded helm charts.
func (i *ChartLoader) Load(config *config.Config, conformanceMode bool, helmWaitMode helm.WaitMode, masterSecret, salt []byte) ([]byte, error) {
releases, err := i.LoadReleases(config, conformanceMode, helmWaitMode, masterSecret, salt)
if err != nil {
return nil, fmt.Errorf("loading releases: %w", err)
}
rel, err := json.Marshal(releases)
if err != nil {
return nil, err
}
return rel, nil
}
// LoadReleases loads the embedded helm charts and returns them as a HelmReleases object. // LoadReleases loads the embedded helm charts and returns them as a HelmReleases object.
func (i *ChartLoader) LoadReleases(config *config.Config, conformanceMode bool, helmWaitMode helm.WaitMode, masterSecret, salt []byte) (*helm.Releases, error) { func (i *ChartLoader) LoadReleases(config *config.Config, conformanceMode bool, helmWaitMode WaitMode, masterSecret, salt []byte) (*Releases, error) {
ciliumRelease, err := i.loadRelease(ciliumInfo, helmWaitMode) ciliumRelease, err := i.loadRelease(ciliumInfo, helmWaitMode)
if err != nil { if err != nil {
return nil, fmt.Errorf("loading cilium: %w", err) return nil, fmt.Errorf("loading cilium: %w", err)
@ -147,7 +133,7 @@ func (i *ChartLoader) LoadReleases(config *config.Config, conformanceMode bool,
return nil, fmt.Errorf("extending constellation-services values: %w", err) return nil, fmt.Errorf("extending constellation-services values: %w", err)
} }
releases := helm.Releases{Cilium: ciliumRelease, CertManager: certManagerRelease, ConstellationOperators: operatorRelease, ConstellationServices: conServicesRelease} releases := Releases{Cilium: ciliumRelease, CertManager: certManagerRelease, ConstellationOperators: operatorRelease, ConstellationServices: conServicesRelease}
if config.HasProvider(cloudprovider.AWS) { if config.HasProvider(cloudprovider.AWS) {
awsRelease, err := i.loadRelease(awsLBControllerInfo, helmWaitMode) awsRelease, err := i.loadRelease(awsLBControllerInfo, helmWaitMode)
if err != nil { if err != nil {
@ -168,10 +154,10 @@ func (i *ChartLoader) LoadReleases(config *config.Config, conformanceMode bool,
// loadRelease loads the embedded chart and values depending on the given info argument. // loadRelease loads the embedded chart and values depending on the given info argument.
// IMPORTANT: .helmignore rules specifying files in subdirectories are not applied (e.g. crds/kustomization.yaml). // IMPORTANT: .helmignore rules specifying files in subdirectories are not applied (e.g. crds/kustomization.yaml).
func (i *ChartLoader) loadRelease(info chartInfo, helmWaitMode helm.WaitMode) (helm.Release, error) { func (i *ChartLoader) loadRelease(info chartInfo, helmWaitMode WaitMode) (Release, error) {
chart, err := loadChartsDir(helmFS, info.path) chart, err := loadChartsDir(helmFS, info.path)
if err != nil { if err != nil {
return helm.Release{}, fmt.Errorf("loading %s chart: %w", info.releaseName, err) return Release{}, fmt.Errorf("loading %s chart: %w", info.releaseName, err)
} }
var values map[string]any var values map[string]any
@ -181,7 +167,7 @@ func (i *ChartLoader) loadRelease(info chartInfo, helmWaitMode helm.WaitMode) (h
var ok bool var ok bool
values, ok = ciliumVals[i.csp.String()] values, ok = ciliumVals[i.csp.String()]
if !ok { if !ok {
return helm.Release{}, fmt.Errorf("cilium values for csp %q not found", i.csp.String()) return Release{}, fmt.Errorf("cilium values for csp %q not found", i.csp.String())
} }
case certManagerInfo.releaseName: case certManagerInfo.releaseName:
values = i.loadCertManagerValues() values = i.loadCertManagerValues()
@ -200,10 +186,10 @@ func (i *ChartLoader) loadRelease(info chartInfo, helmWaitMode helm.WaitMode) (h
chartRaw, err := i.marshalChart(chart) chartRaw, err := i.marshalChart(chart)
if err != nil { if err != nil {
return helm.Release{}, fmt.Errorf("packaging %s chart: %w", info.releaseName, err) return Release{}, fmt.Errorf("packaging %s chart: %w", info.releaseName, err)
} }
return helm.Release{Chart: chartRaw, Values: values, ReleaseName: info.releaseName, WaitMode: helmWaitMode}, nil return Release{Chart: chartRaw, Values: values, ReleaseName: info.releaseName, WaitMode: helmWaitMode}, nil
} }
func (i *ChartLoader) loadAWSLBControllerValues() map[string]any { func (i *ChartLoader) loadAWSLBControllerValues() map[string]any {

View File

@ -8,7 +8,6 @@ package helm
import ( import (
"bytes" "bytes"
"encoding/json"
"fmt" "fmt"
"io/fs" "io/fs"
"os" "os"
@ -29,21 +28,15 @@ import (
"github.com/edgelesssys/constellation/v2/internal/attestation/measurements" "github.com/edgelesssys/constellation/v2/internal/attestation/measurements"
"github.com/edgelesssys/constellation/v2/internal/cloud/cloudprovider" "github.com/edgelesssys/constellation/v2/internal/cloud/cloudprovider"
"github.com/edgelesssys/constellation/v2/internal/config" "github.com/edgelesssys/constellation/v2/internal/config"
"github.com/edgelesssys/constellation/v2/internal/deploy/helm"
) )
// TestLoad checks if the serialized format that Load returns correctly preserves the dependencies of the loaded chart. func TestLoadReleases(t *testing.T) {
func TestLoad(t *testing.T) {
assert := assert.New(t) assert := assert.New(t)
require := require.New(t) require := require.New(t)
config := &config.Config{Provider: config.ProviderConfig{GCP: &config.GCPConfig{}}} config := &config.Config{Provider: config.ProviderConfig{GCP: &config.GCPConfig{}}}
chartLoader := ChartLoader{csp: config.GetProvider()} chartLoader := ChartLoader{csp: config.GetProvider()}
release, err := chartLoader.Load(config, true, helm.WaitModeAtomic, []byte("secret"), []byte("salt")) helmReleases, err := chartLoader.LoadReleases(config, true, WaitModeAtomic, []byte("secret"), []byte("salt"))
require.NoError(err)
var helmReleases helm.Releases
err = json.Unmarshal(release, &helmReleases)
require.NoError(err) require.NoError(err)
reader := bytes.NewReader(helmReleases.ConstellationServices.Chart) reader := bytes.NewReader(helmReleases.ConstellationServices.Chart)
chart, err := loader.LoadArchive(reader) chart, err := loader.LoadArchive(reader)

View File

@ -25,28 +25,6 @@ type Releases struct {
ConstellationServices Release ConstellationServices Release
} }
// MergeMaps returns a new map that is the merger of it's inputs.
// Key collisions are resolved by taking the value of the second argument (map b).
// Taken from: https://github.com/helm/helm/blob/dbc6d8e20fe1d58d50e6ed30f09a04a77e4c68db/pkg/cli/values/options.go#L91-L108.
func MergeMaps(a, b map[string]any) map[string]any {
out := make(map[string]any, len(a))
for k, v := range a {
out[k] = v
}
for k, v := range b {
if v, ok := v.(map[string]any); ok {
if bv, ok := out[k]; ok {
if bv, ok := bv.(map[string]any); ok {
out[k] = MergeMaps(bv, v)
continue
}
}
}
out[k] = v
}
return out
}
// WaitMode specifies the wait mode for a helm release. // WaitMode specifies the wait mode for a helm release.
type WaitMode string type WaitMode string

View File

@ -1,134 +0,0 @@
/*
Copyright (c) Edgeless Systems GmbH
SPDX-License-Identifier: AGPL-3.0-only
*/
package helm
import (
"context"
"encoding/base64"
"encoding/json"
"fmt"
"github.com/edgelesssys/constellation/v2/cli/internal/terraform"
"github.com/edgelesssys/constellation/v2/internal/cloud/azureshared"
"github.com/edgelesssys/constellation/v2/internal/cloud/cloudprovider"
"github.com/edgelesssys/constellation/v2/internal/cloud/gcpshared"
"github.com/edgelesssys/constellation/v2/internal/constants"
)
// setupMicroserviceVals returns the values for the microservice chart.
func setupMicroserviceVals(ctx context.Context, provider cloudprovider.Provider, measurementSalt []byte, uid, serviceAccURI string) (map[string]any, error) {
tfClient, err := terraform.New(ctx, constants.TerraformWorkingDir)
if err != nil {
return nil, fmt.Errorf("creating Terraform client: %w", err)
}
output, err := tfClient.ShowCluster(ctx, provider)
if err != nil {
return nil, fmt.Errorf("getting Terraform output: %w", err)
}
extraVals := map[string]any{
"join-service": map[string]any{
"measurementSalt": base64.StdEncoding.EncodeToString(measurementSalt),
},
"verification-service": map[string]any{
"loadBalancerIP": output.IP,
},
"konnectivity": map[string]any{
"loadBalancerIP": output.IP,
},
}
switch provider {
case cloudprovider.GCP:
serviceAccountKey, err := gcpshared.ServiceAccountKeyFromURI(serviceAccURI)
if err != nil {
return nil, fmt.Errorf("getting service account key: %w", err)
}
rawKey, err := json.Marshal(serviceAccountKey)
if err != nil {
return nil, fmt.Errorf("marshaling service account key: %w", err)
}
if output.GCP == nil {
return nil, fmt.Errorf("no GCP output from Terraform")
}
extraVals["ccm"] = map[string]any{
"GCP": map[string]any{
"projectID": output.GCP.ProjectID,
"uid": uid,
"secretData": string(rawKey),
"subnetworkPodCIDR": output.GCP.IPCidrPod,
},
}
case cloudprovider.Azure:
if output.Azure == nil {
return nil, fmt.Errorf("no Azure output from Terraform")
}
ccmConfig, err := getCCMConfig(*output.Azure, serviceAccURI)
if err != nil {
return nil, fmt.Errorf("getting Azure CCM config: %w", err)
}
extraVals["ccm"] = map[string]any{
"Azure": map[string]any{
"azureConfig": string(ccmConfig),
},
}
}
return extraVals, nil
}
// setupOperatorVals returns the values for the constellation-operator chart.
func setupOperatorVals(_ context.Context, uid string) map[string]any {
return map[string]any{
"constellation-operator": map[string]any{
"constellationUID": uid,
},
}
}
type cloudConfig struct {
Cloud string `json:"cloud,omitempty"`
TenantID string `json:"tenantId,omitempty"`
SubscriptionID string `json:"subscriptionId,omitempty"`
ResourceGroup string `json:"resourceGroup,omitempty"`
Location string `json:"location,omitempty"`
SubnetName string `json:"subnetName,omitempty"`
SecurityGroupName string `json:"securityGroupName,omitempty"`
SecurityGroupResourceGroup string `json:"securityGroupResourceGroup,omitempty"`
LoadBalancerName string `json:"loadBalancerName,omitempty"`
LoadBalancerSku string `json:"loadBalancerSku,omitempty"`
VNetName string `json:"vnetName,omitempty"`
VNetResourceGroup string `json:"vnetResourceGroup,omitempty"`
CloudProviderBackoff bool `json:"cloudProviderBackoff,omitempty"`
UseInstanceMetadata bool `json:"useInstanceMetadata,omitempty"`
VMType string `json:"vmType,omitempty"`
UseManagedIdentityExtension bool `json:"useManagedIdentityExtension,omitempty"`
UserAssignedIdentityID string `json:"userAssignedIdentityID,omitempty"`
}
// GetCCMConfig returns the configuration needed for the Kubernetes Cloud Controller Manager on Azure.
func getCCMConfig(tfOutput terraform.AzureApplyOutput, serviceAccURI string) ([]byte, error) {
creds, err := azureshared.ApplicationCredentialsFromURI(serviceAccURI)
if err != nil {
return nil, fmt.Errorf("getting service account key: %w", err)
}
useManagedIdentityExtension := creds.PreferredAuthMethod == azureshared.AuthMethodUserAssignedIdentity
config := cloudConfig{
Cloud: "AzurePublicCloud",
TenantID: creds.TenantID,
SubscriptionID: tfOutput.SubscriptionID,
ResourceGroup: tfOutput.ResourceGroup,
LoadBalancerSku: "standard",
SecurityGroupName: tfOutput.NetworkSecurityGroupName,
LoadBalancerName: tfOutput.LoadBalancerName,
UseInstanceMetadata: true,
VMType: "vmss",
Location: creds.Location,
UseManagedIdentityExtension: useManagedIdentityExtension,
UserAssignedIdentityID: tfOutput.UserAssignedIdentity,
}
return json.Marshal(config)
}

View File

@ -13,20 +13,20 @@ import (
"strings" "strings"
"time" "time"
"helm.sh/helm/v3/pkg/action"
"helm.sh/helm/v3/pkg/chart"
"helm.sh/helm/v3/pkg/cli"
"helm.sh/helm/v3/pkg/release"
"github.com/edgelesssys/constellation/v2/cli/internal/clusterid" "github.com/edgelesssys/constellation/v2/cli/internal/clusterid"
"github.com/edgelesssys/constellation/v2/internal/cloud/cloudprovider" "github.com/edgelesssys/constellation/v2/internal/cloud/cloudprovider"
"github.com/edgelesssys/constellation/v2/internal/compatibility" "github.com/edgelesssys/constellation/v2/internal/compatibility"
"github.com/edgelesssys/constellation/v2/internal/config" "github.com/edgelesssys/constellation/v2/internal/config"
"github.com/edgelesssys/constellation/v2/internal/constants" "github.com/edgelesssys/constellation/v2/internal/constants"
"github.com/edgelesssys/constellation/v2/internal/deploy/helm"
"github.com/edgelesssys/constellation/v2/internal/file" "github.com/edgelesssys/constellation/v2/internal/file"
"github.com/edgelesssys/constellation/v2/internal/semver" "github.com/edgelesssys/constellation/v2/internal/semver"
"github.com/edgelesssys/constellation/v2/internal/versions" "github.com/edgelesssys/constellation/v2/internal/versions"
"github.com/spf13/afero" "github.com/spf13/afero"
"helm.sh/helm/v3/pkg/action"
"helm.sh/helm/v3/pkg/chart"
"helm.sh/helm/v3/pkg/cli"
"helm.sh/helm/v3/pkg/release"
apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/runtime/schema"
@ -44,8 +44,8 @@ var ErrConfirmationMissing = errors.New("action requires user confirmation")
var errReleaseNotFound = errors.New("release not found") var errReleaseNotFound = errors.New("release not found")
// Client handles interaction with helm and the cluster. // UpgradeClient handles interaction with helm and the cluster.
type Client struct { type UpgradeClient struct {
config *action.Configuration config *action.Configuration
kubectl crdClient kubectl crdClient
fs file.Handler fs file.Handler
@ -53,8 +53,8 @@ type Client struct {
log debugLog log debugLog
} }
// NewClient returns a new initializes client for the namespace Client. // NewUpgradeClient returns a new initializes upgrade client for the given namespace.
func NewClient(client crdClient, kubeConfigPath, helmNamespace string, log debugLog) (*Client, error) { func NewUpgradeClient(client crdClient, kubeConfigPath, helmNamespace string, log debugLog) (*UpgradeClient, error) {
settings := cli.New() settings := cli.New()
settings.KubeConfig = kubeConfigPath // constants.AdminConfFilename settings.KubeConfig = kubeConfigPath // constants.AdminConfFilename
@ -74,10 +74,10 @@ func NewClient(client crdClient, kubeConfigPath, helmNamespace string, log debug
return nil, fmt.Errorf("initializing kubectl: %w", err) return nil, fmt.Errorf("initializing kubectl: %w", err)
} }
return &Client{kubectl: client, fs: fileHandler, actions: actions{config: actionConfig}, log: log}, nil return &UpgradeClient{kubectl: client, fs: fileHandler, actions: actions{config: actionConfig}, log: log}, nil
} }
func (c *Client) shouldUpgrade(releaseName string, newVersion semver.Semver, force bool) error { func (c *UpgradeClient) shouldUpgrade(releaseName string, newVersion semver.Semver, force bool) error {
currentVersion, err := c.currentVersion(releaseName) currentVersion, err := c.currentVersion(releaseName)
if err != nil { if err != nil {
return fmt.Errorf("getting version for %s: %w", releaseName, err) return fmt.Errorf("getting version for %s: %w", releaseName, err)
@ -107,7 +107,7 @@ func (c *Client) shouldUpgrade(releaseName string, newVersion semver.Semver, for
// Upgrade runs a helm-upgrade on all deployments that are managed via Helm. // Upgrade runs a helm-upgrade on all deployments that are managed via Helm.
// If the CLI receives an interrupt signal it will cancel the context. // If the CLI receives an interrupt signal it will cancel the context.
// Canceling the context will prompt helm to abort and roll back the ongoing upgrade. // Canceling the context will prompt helm to abort and roll back the ongoing upgrade.
func (c *Client) Upgrade(ctx context.Context, config *config.Config, idFile clusterid.File, timeout time.Duration, allowDestructive, force bool, upgradeID string) error { func (c *UpgradeClient) Upgrade(ctx context.Context, config *config.Config, idFile clusterid.File, timeout time.Duration, allowDestructive, force bool, upgradeID string) error {
upgradeErrs := []error{} upgradeErrs := []error{}
upgradeReleases := []*chart.Chart{} upgradeReleases := []*chart.Chart{}
newReleases := []*chart.Chart{} newReleases := []*chart.Chart{}
@ -206,7 +206,7 @@ func getManagedCharts(config *config.Config) []chartInfo {
} }
// Versions queries the cluster for running versions and returns a map of releaseName -> version. // Versions queries the cluster for running versions and returns a map of releaseName -> version.
func (c *Client) Versions() (ServiceVersions, error) { func (c *UpgradeClient) Versions() (ServiceVersions, error) {
ciliumVersion, err := c.currentVersion(ciliumInfo.releaseName) ciliumVersion, err := c.currentVersion(ciliumInfo.releaseName)
if err != nil { if err != nil {
return ServiceVersions{}, fmt.Errorf("getting %s version: %w", ciliumInfo.releaseName, err) return ServiceVersions{}, fmt.Errorf("getting %s version: %w", ciliumInfo.releaseName, err)
@ -246,7 +246,7 @@ func (c *Client) Versions() (ServiceVersions, error) {
} }
// currentVersion returns the version of the currently installed helm release. // currentVersion returns the version of the currently installed helm release.
func (c *Client) currentVersion(release string) (semver.Semver, error) { func (c *UpgradeClient) currentVersion(release string) (semver.Semver, error) {
rel, err := c.actions.listAction(release) rel, err := c.actions.listAction(release)
if err != nil { if err != nil {
return semver.Semver{}, err return semver.Semver{}, err
@ -266,7 +266,7 @@ func (c *Client) currentVersion(release string) (semver.Semver, error) {
return semver.New(rel[0].Chart.Metadata.Version) return semver.New(rel[0].Chart.Metadata.Version)
} }
func (c *Client) csiVersions() (map[string]semver.Semver, error) { func (c *UpgradeClient) csiVersions() (map[string]semver.Semver, error) {
packedChartRelease, err := c.actions.listAction(csiInfo.releaseName) packedChartRelease, err := c.actions.listAction(csiInfo.releaseName)
if err != nil { if err != nil {
return nil, fmt.Errorf("listing %s: %w", csiInfo.releaseName, err) return nil, fmt.Errorf("listing %s: %w", csiInfo.releaseName, err)
@ -299,7 +299,7 @@ func (c *Client) csiVersions() (map[string]semver.Semver, error) {
} }
// installNewRelease installs a previously not installed release on the cluster. // installNewRelease installs a previously not installed release on the cluster.
func (c *Client) installNewRelease( func (c *UpgradeClient) installNewRelease(
ctx context.Context, timeout time.Duration, conf *config.Config, idFile clusterid.File, chart *chart.Chart, ctx context.Context, timeout time.Duration, conf *config.Config, idFile clusterid.File, chart *chart.Chart,
) error { ) error {
releaseName, values, err := c.loadUpgradeValues(ctx, conf, idFile, chart) releaseName, values, err := c.loadUpgradeValues(ctx, conf, idFile, chart)
@ -310,7 +310,7 @@ func (c *Client) installNewRelease(
} }
// upgradeRelease upgrades a release running on the cluster. // upgradeRelease upgrades a release running on the cluster.
func (c *Client) upgradeRelease( func (c *UpgradeClient) upgradeRelease(
ctx context.Context, timeout time.Duration, conf *config.Config, idFile clusterid.File, chart *chart.Chart, ctx context.Context, timeout time.Duration, conf *config.Config, idFile clusterid.File, chart *chart.Chart,
) error { ) error {
releaseName, values, err := c.loadUpgradeValues(ctx, conf, idFile, chart) releaseName, values, err := c.loadUpgradeValues(ctx, conf, idFile, chart)
@ -327,7 +327,7 @@ func (c *Client) upgradeRelease(
} }
// loadUpgradeValues loads values for a chart required for running an upgrade. // loadUpgradeValues loads values for a chart required for running an upgrade.
func (c *Client) loadUpgradeValues(ctx context.Context, conf *config.Config, idFile clusterid.File, chart *chart.Chart, func (c *UpgradeClient) loadUpgradeValues(ctx context.Context, conf *config.Config, idFile clusterid.File, chart *chart.Chart,
) (string, map[string]any, error) { ) (string, map[string]any, error) {
// We need to load all values that can be statically loaded before merging them with the cluster // We need to load all values that can be statically loaded before merging them with the cluster
// values. Otherwise the templates are not rendered correctly. // values. Otherwise the templates are not rendered correctly.
@ -385,7 +385,7 @@ func (c *Client) loadUpgradeValues(ctx context.Context, conf *config.Config, idF
// applyMigrations checks the from version and applies the necessary migrations. // applyMigrations checks the from version and applies the necessary migrations.
// The function assumes the caller has verified that our version drift restriction is not violated, // The function assumes the caller has verified that our version drift restriction is not violated,
// Currently, this is done during config validation. // Currently, this is done during config validation.
func (c *Client) applyMigrations(ctx context.Context, releaseName string, values map[string]any, conf *config.Config) error { func (c *UpgradeClient) applyMigrations(ctx context.Context, releaseName string, values map[string]any, conf *config.Config) error {
current, err := c.currentVersion(releaseName) current, err := c.currentVersion(releaseName)
if err != nil { if err != nil {
return fmt.Errorf("getting %s version: %w", releaseName, err) return fmt.Errorf("getting %s version: %w", releaseName, err)
@ -413,7 +413,7 @@ func migrateFrom2_8(_ context.Context, _ map[string]any, _ *config.Config, _ crd
// and merging the fetched values with the locally found values. // and merging the fetched values with the locally found values.
// This is done to ensure that new values (from upgrades of the local files) end up in the cluster. // This is done to ensure that new values (from upgrades of the local files) end up in the cluster.
// reuse-values does not ensure this. // reuse-values does not ensure this.
func (c *Client) mergeClusterValues(localValues map[string]any, releaseName string) (map[string]any, error) { func (c *UpgradeClient) mergeClusterValues(localValues map[string]any, releaseName string) (map[string]any, error) {
// Ensure installCRDs is set for cert-manager chart. // Ensure installCRDs is set for cert-manager chart.
if releaseName == certManagerInfo.releaseName { if releaseName == certManagerInfo.releaseName {
localValues["installCRDs"] = true localValues["installCRDs"] = true
@ -423,11 +423,11 @@ func (c *Client) mergeClusterValues(localValues map[string]any, releaseName stri
return nil, fmt.Errorf("getting values for %s: %w", releaseName, err) return nil, fmt.Errorf("getting values for %s: %w", releaseName, err)
} }
return helm.MergeMaps(clusterValues, localValues), nil return mergeMaps(clusterValues, localValues), nil
} }
// GetValues queries the cluster for the values of the given release. // GetValues queries the cluster for the values of the given release.
func (c *Client) GetValues(release string) (map[string]any, error) { func (c *UpgradeClient) GetValues(release string) (map[string]any, error) {
client := action.NewGetValues(c.config) client := action.NewGetValues(c.config)
// Version corresponds to the releases revision. Specifying a Version <= 0 yields the latest release. // Version corresponds to the releases revision. Specifying a Version <= 0 yields the latest release.
client.Version = 0 client.Version = 0
@ -441,7 +441,7 @@ func (c *Client) GetValues(release string) (map[string]any, error) {
// updateCRDs walks through the dependencies of the given chart and applies // updateCRDs walks through the dependencies of the given chart and applies
// the files in the dependencie's 'crds' folder. // the files in the dependencie's 'crds' folder.
// This function is NOT recursive! // This function is NOT recursive!
func (c *Client) updateCRDs(ctx context.Context, chart *chart.Chart) error { func (c *UpgradeClient) updateCRDs(ctx context.Context, chart *chart.Chart) error {
for _, dep := range chart.Dependencies() { for _, dep := range chart.Dependencies() {
for _, crdFile := range dep.Files { for _, crdFile := range dep.Files {
if strings.HasPrefix(crdFile.Name, "crds/") { if strings.HasPrefix(crdFile.Name, "crds/") {
@ -456,11 +456,6 @@ func (c *Client) updateCRDs(ctx context.Context, chart *chart.Chart) error {
return nil return nil
} }
type debugLog interface {
Debugf(format string, args ...any)
Sync()
}
type crdClient interface { type crdClient interface {
Initialize(kubeconfig []byte) error Initialize(kubeconfig []byte) error
ApplyCRD(ctx context.Context, rawCRD []byte) error ApplyCRD(ctx context.Context, rawCRD []byte) error

View File

@ -46,7 +46,7 @@ func TestShouldUpgrade(t *testing.T) {
assert := assert.New(t) assert := assert.New(t)
require := require.New(t) require := require.New(t)
client := Client{kubectl: nil, actions: &stubActionWrapper{version: tc.version}, log: logger.NewTest(t)} client := UpgradeClient{kubectl: nil, actions: &stubActionWrapper{version: tc.version}, log: logger.NewTest(t)}
chart, err := loadChartsDir(helmFS, certManagerInfo.path) chart, err := loadChartsDir(helmFS, certManagerInfo.path)
require.NoError(err) require.NoError(err)
@ -77,7 +77,7 @@ func TestUpgradeRelease(t *testing.T) {
assert := assert.New(t) assert := assert.New(t)
require := require.New(t) require := require.New(t)
client := Client{kubectl: nil, actions: &stubActionWrapper{version: tc.version}, log: logger.NewTest(t)} client := UpgradeClient{kubectl: nil, actions: &stubActionWrapper{version: tc.version}, log: logger.NewTest(t)}
chart, err := loadChartsDir(helmFS, certManagerInfo.path) chart, err := loadChartsDir(helmFS, certManagerInfo.path)
require.NoError(err) require.NoError(err)

View File

@ -136,7 +136,7 @@ func NewUpgrader(ctx context.Context, outWriter io.Writer, fileHandler file.Hand
} }
u.dynamicInterface = &NodeVersionClient{client: unstructuredClient} u.dynamicInterface = &NodeVersionClient{client: unstructuredClient}
helmClient, err := helm.NewClient(kubectl.New(), constants.AdminConfFilename, constants.HelmNamespace, log) helmClient, err := helm.NewUpgradeClient(kubectl.New(), constants.AdminConfFilename, constants.HelmNamespace, log)
if err != nil { if err != nil {
return nil, fmt.Errorf("setting up helm client: %w", err) return nil, fmt.Errorf("setting up helm client: %w", err)
} }

View File

@ -1,28 +0,0 @@
load("@io_bazel_rules_go//go:def.bzl", "go_library")
load("//bazel/go:go_test.bzl", "go_test")
go_library(
name = "helm",
srcs = [
"helm.go",
"install.go",
],
importpath = "github.com/edgelesssys/constellation/v2/internal/deploy/helm",
visibility = ["//:__subpackages__"],
deps = [
"//internal/constants",
"//internal/retry",
"@io_k8s_apimachinery//pkg/util/wait",
"@sh_helm_helm_v3//pkg/action",
"@sh_helm_helm_v3//pkg/chart",
"@sh_helm_helm_v3//pkg/chart/loader",
"@sh_helm_helm_v3//pkg/cli",
],
)
go_test(
name = "helm_test",
srcs = ["helm_test.go"],
embed = [":helm"],
deps = ["@com_github_stretchr_testify//assert"],
)

View File

@ -12,9 +12,11 @@ go_library(
"@io_k8s_apiextensions_apiserver//pkg/client/clientset/clientset/typed/apiextensions/v1:apiextensions", "@io_k8s_apiextensions_apiserver//pkg/client/clientset/clientset/typed/apiextensions/v1:apiextensions",
"@io_k8s_apimachinery//pkg/apis/meta/v1:meta", "@io_k8s_apimachinery//pkg/apis/meta/v1:meta",
"@io_k8s_apimachinery//pkg/apis/meta/v1/unstructured", "@io_k8s_apimachinery//pkg/apis/meta/v1/unstructured",
"@io_k8s_apimachinery//pkg/labels",
"@io_k8s_apimachinery//pkg/runtime", "@io_k8s_apimachinery//pkg/runtime",
"@io_k8s_apimachinery//pkg/runtime/schema", "@io_k8s_apimachinery//pkg/runtime/schema",
"@io_k8s_apimachinery//pkg/runtime/serializer", "@io_k8s_apimachinery//pkg/runtime/serializer",
"@io_k8s_apimachinery//pkg/types",
"@io_k8s_client_go//dynamic", "@io_k8s_client_go//dynamic",
"@io_k8s_client_go//kubernetes", "@io_k8s_client_go//kubernetes",
"@io_k8s_client_go//scale/scheme", "@io_k8s_client_go//scale/scheme",

View File

@ -19,9 +19,11 @@ import (
apiextensionsclientv1 "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset/typed/apiextensions/v1" apiextensionsclientv1 "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset/typed/apiextensions/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/labels"
"k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apimachinery/pkg/runtime/serializer" "k8s.io/apimachinery/pkg/runtime/serializer"
"k8s.io/apimachinery/pkg/types"
"k8s.io/client-go/dynamic" "k8s.io/client-go/dynamic"
"k8s.io/client-go/kubernetes" "k8s.io/client-go/kubernetes"
"k8s.io/client-go/scale/scheme" "k8s.io/client-go/scale/scheme"
@ -130,6 +132,22 @@ func (k *Kubectl) AnnotateNode(ctx context.Context, nodeName, annotationKey, ann
}) })
} }
// PatchFirstNodePodCIDR patches the firstNodePodCIDR of the first control-plane node for Cilium.
func (k *Kubectl) PatchFirstNodePodCIDR(ctx context.Context, firstNodePodCIDR string) error {
selector := labels.Set{"node-role.kubernetes.io/control-plane": ""}.AsSelector()
controlPlaneList, err := k.CoreV1().Nodes().List(ctx, metav1.ListOptions{LabelSelector: selector.String()})
if err != nil {
return err
}
if len(controlPlaneList.Items) != 1 {
return fmt.Errorf("expected 1 control-plane node, got %d", len(controlPlaneList.Items))
}
nodeName := controlPlaneList.Items[0].Name
// Update the node's spec
_, err = k.CoreV1().Nodes().Patch(context.Background(), nodeName, types.MergePatchType, []byte(fmt.Sprintf(`{"spec":{"podCIDR":"%s"}}`, firstNodePodCIDR)), metav1.PatchOptions{})
return err
}
// ListAllNamespaces returns all namespaces in the cluster. // ListAllNamespaces returns all namespaces in the cluster.
func (k *Kubectl) ListAllNamespaces(ctx context.Context) (*corev1.NamespaceList, error) { func (k *Kubectl) ListAllNamespaces(ctx context.Context) (*corev1.NamespaceList, error) {
return k.CoreV1().Namespaces().List(ctx, metav1.ListOptions{}) return k.CoreV1().Namespaces().List(ctx, metav1.ListOptions{})