Git-Mirrors/constellation
1
0
Fork 0
mirror of https://github.com/edgelesssys/constellation.git synced 2024-10-01 01:36:09 -04:00
Code Issues Packages Projects Releases Wiki Activity

cli: install cilium in cli instead of bootstrapper (#2146)

Browse Source 
* add wait and restartDS

* cilium working (tested on azure + gcp)

* clean helm code from bootstrapper

* fixup! clean helm code from bootstrapper

* fixup! clean helm code from bootstrapper

* fixup! clean helm code from bootstrapper

* add patchnode for gcp

* fix gcp

* patch node inside bootstrapper

* apply renaming of client

* fixup! apply renaming of client

* otto feedback
This commit is contained in:
Adrian Stobbe 2023-08-02 15:49:40 +02:00 committed by GitHub
parent da1376cd90
commit 13eea1ca31
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
36 changed files with 519 additions and 575 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
bootstrapper/cmd/bootstrapper/BUILD.bazel
View File

@ -35,7 +35,6 @@ go_library(
"//internal/cloud/openstack",
"//internal/cloud/qemu",
"//internal/constants",
"//internal/deploy/helm",
"//internal/file",
"//internal/grpc/dialer",
"//internal/kubernetes/kubectl",

16
bootstrapper/cmd/bootstrapper/main.go
View File

@ -32,7 +32,6 @@ import (
openstackcloud "github.com/edgelesssys/constellation/v2/internal/cloud/openstack"
qemucloud "github.com/edgelesssys/constellation/v2/internal/cloud/qemu"
"github.com/edgelesssys/constellation/v2/internal/constants"
"github.com/edgelesssys/constellation/v2/internal/deploy/helm"
"github.com/edgelesssys/constellation/v2/internal/file"
"github.com/edgelesssys/constellation/v2/internal/kubernetes/kubectl"
"github.com/edgelesssys/constellation/v2/internal/logger"
@ -67,11 +66,6 @@ func main() {
var openDevice vtpm.TPMOpenFunc
var fs afero.Fs
helmClient, err := helm.NewInstaller(constants.ControlPlaneAdminConfFilename, log)
if err != nil {
log.With(zap.Error(err)).Fatalf("Helm client could not be initialized")
}
attestVariant, err := variant.FromString(os.Getenv(constants.AttestationVariant))
if err != nil {
log.With(zap.Error(err)).Fatalf("Failed to parse attestation variant")
@ -96,7 +90,7 @@ func main() {
clusterInitJoiner = kubernetes.New(
"aws", k8sapi.NewKubernetesUtil(), &k8sapi.KubdeadmConfiguration{}, kubectl.New(),
metadata, helmClient, &kubewaiter.CloudKubeAPIWaiter{},
metadata, &kubewaiter.CloudKubeAPIWaiter{},
)
openDevice = vtpm.OpenVTPM
fs = afero.NewOsFs()
@ -116,7 +110,7 @@ func main() {
metadataAPI = metadata
clusterInitJoiner = kubernetes.New(
"gcp", k8sapi.NewKubernetesUtil(), &k8sapi.KubdeadmConfiguration{}, kubectl.New(),
metadata, helmClient, &kubewaiter.CloudKubeAPIWaiter{},
metadata, &kubewaiter.CloudKubeAPIWaiter{},
)
openDevice = vtpm.OpenVTPM
fs = afero.NewOsFs()
@ -134,7 +128,7 @@ func main() {
metadataAPI = metadata
clusterInitJoiner = kubernetes.New(
"azure", k8sapi.NewKubernetesUtil(), &k8sapi.KubdeadmConfiguration{}, kubectl.New(),
metadata, helmClient, &kubewaiter.CloudKubeAPIWaiter{},
metadata, &kubewaiter.CloudKubeAPIWaiter{},
)
openDevice = vtpm.OpenVTPM
@ -145,7 +139,7 @@ func main() {
metadata := qemucloud.New()
clusterInitJoiner = kubernetes.New(
"qemu", k8sapi.NewKubernetesUtil(), &k8sapi.KubdeadmConfiguration{}, kubectl.New(),
metadata, helmClient, &kubewaiter.CloudKubeAPIWaiter{},
metadata, &kubewaiter.CloudKubeAPIWaiter{},
)
metadataAPI = metadata
@ -168,7 +162,7 @@ func main() {
}
clusterInitJoiner = kubernetes.New(
"openstack", k8sapi.NewKubernetesUtil(), &k8sapi.KubdeadmConfiguration{}, kubectl.New(),
metadata, helmClient, &kubewaiter.CloudKubeAPIWaiter{},
metadata, &kubewaiter.CloudKubeAPIWaiter{},
)
metadataAPI = metadata
openDevice = vtpm.OpenVTPM

2
bootstrapper/cmd/bootstrapper/test.go
View File

@ -22,7 +22,7 @@ type clusterFake struct{}
// InitCluster fakes bootstrapping a new cluster with the current node being the master, returning the arguments required to join the cluster.
func (c *clusterFake) InitCluster(
context.Context, string, string,
[]byte, bool, components.Components, []string, *logger.Logger,
bool, components.Components, []string, *logger.Logger,
) ([]byte, error) {
return []byte{}, nil
}

121
bootstrapper/initproto/init.pb.go
View File

@ -33,7 +33,6 @@ type InitRequest struct {
StorageUri string `protobuf:"bytes,4,opt,name=storage_uri,json=storageUri,proto3" json:"storage_uri,omitempty"`
CloudServiceAccountUri string `protobuf:"bytes,7,opt,name=cloud_service_account_uri,json=cloudServiceAccountUri,proto3" json:"cloud_service_account_uri,omitempty"`
KubernetesVersion string `protobuf:"bytes,8,opt,name=kubernetes_version,json=kubernetesVersion,proto3" json:"kubernetes_version,omitempty"`
HelmDeployments []byte `protobuf:"bytes,11,opt,name=helm_deployments,json=helmDeployments,proto3" json:"helm_deployments,omitempty"`
ConformanceMode bool `protobuf:"varint,14,opt,name=conformance_mode,json=conformanceMode,proto3" json:"conformance_mode,omitempty"`
KubernetesComponents []*KubernetesComponent `protobuf:"bytes,15,rep,name=kubernetes_components,json=kubernetesComponents,proto3" json:"kubernetes_components,omitempty"`
InitSecret []byte `protobuf:"bytes,16,opt,name=init_secret,json=initSecret,proto3" json:"init_secret,omitempty"`
@ -101,13 +100,6 @@ func (x *InitRequest) GetKubernetesVersion() string {
return ""
}
func (x *InitRequest) GetHelmDeployments() []byte {
if x != nil {
return x.HelmDeployments
}
return nil
}
func (x *InitRequest) GetConformanceMode() bool {
if x != nil {
return x.ConformanceMode
@ -471,7 +463,7 @@ var File_bootstrapper_initproto_init_proto protoreflect.FileDescriptor
var file_bootstrapper_initproto_init_proto_rawDesc = []byte{
0x0a, 0x21, 0x62, 0x6f, 0x6f, 0x74, 0x73, 0x74, 0x72, 0x61, 0x70, 0x70, 0x65, 0x72, 0x2f, 0x69,
0x6e, 0x69, 0x74, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x69, 0x6e, 0x69, 0x74, 0x2e, 0x70, 0x72,
0x6f, 0x74, 0x6f, 0x12, 0x04, 0x69, 0x6e, 0x69, 0x74, 0x22, 0xcb, 0x03, 0x0a, 0x0b, 0x49, 0x6e,
0x6f, 0x74, 0x6f, 0x12, 0x04, 0x69, 0x6e, 0x69, 0x74, 0x22, 0xa0, 0x03, 0x0a, 0x0b, 0x49, 0x6e,
0x69, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x17, 0x0a, 0x07, 0x6b, 0x6d, 0x73,
0x5f, 0x75, 0x72, 0x69, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x6b, 0x6d, 0x73, 0x55,
0x72, 0x69, 0x12, 0x1f, 0x0a, 0x0b, 0x73, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x5f, 0x75, 0x72,
@ -483,64 +475,61 @@ var file_bootstrapper_initproto_init_proto_rawDesc = []byte{
0x0a, 0x12, 0x6b, 0x75, 0x62, 0x65, 0x72, 0x6e, 0x65, 0x74, 0x65, 0x73, 0x5f, 0x76, 0x65, 0x72,
0x73, 0x69, 0x6f, 0x6e, 0x18, 0x08, 0x20, 0x01, 0x28, 0x09, 0x52, 0x11, 0x6b, 0x75, 0x62, 0x65,
0x72, 0x6e, 0x65, 0x74, 0x65, 0x73, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x29, 0x0a,
0x10, 0x68, 0x65, 0x6c, 0x6d, 0x5f, 0x64, 0x65, 0x70, 0x6c, 0x6f, 0x79, 0x6d, 0x65, 0x6e, 0x74,
0x73, 0x18, 0x0b, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0f, 0x68, 0x65, 0x6c, 0x6d, 0x44, 0x65, 0x70,
0x6c, 0x6f, 0x79, 0x6d, 0x65, 0x6e, 0x74, 0x73, 0x12, 0x29, 0x0a, 0x10, 0x63, 0x6f, 0x6e, 0x66,
0x6f, 0x72, 0x6d, 0x61, 0x6e, 0x63, 0x65, 0x5f, 0x6d, 0x6f, 0x64, 0x65, 0x18, 0x0e, 0x20, 0x01,
0x28, 0x08, 0x52, 0x0f, 0x63, 0x6f, 0x6e, 0x66, 0x6f, 0x72, 0x6d, 0x61, 0x6e, 0x63, 0x65, 0x4d,
0x6f, 0x64, 0x65, 0x12, 0x4e, 0x0a, 0x15, 0x6b, 0x75, 0x62, 0x65, 0x72, 0x6e, 0x65, 0x74, 0x65,
0x73, 0x5f, 0x63, 0x6f, 0x6d, 0x70, 0x6f, 0x6e, 0x65, 0x6e, 0x74, 0x73, 0x18, 0x0f, 0x20, 0x03,
0x28, 0x0b, 0x32, 0x19, 0x2e, 0x69, 0x6e, 0x69, 0x74, 0x2e, 0x4b, 0x75, 0x62, 0x65, 0x72, 0x6e,
0x65, 0x74, 0x65, 0x73, 0x43, 0x6f, 0x6d, 0x70, 0x6f, 0x6e, 0x65, 0x6e, 0x74, 0x52, 0x14, 0x6b,
0x10, 0x63, 0x6f, 0x6e, 0x66, 0x6f, 0x72, 0x6d, 0x61, 0x6e, 0x63, 0x65, 0x5f, 0x6d, 0x6f, 0x64,
0x65, 0x18, 0x0e, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0f, 0x63, 0x6f, 0x6e, 0x66, 0x6f, 0x72, 0x6d,
0x61, 0x6e, 0x63, 0x65, 0x4d, 0x6f, 0x64, 0x65, 0x12, 0x4e, 0x0a, 0x15, 0x6b, 0x75, 0x62, 0x65,
0x72, 0x6e, 0x65, 0x74, 0x65, 0x73, 0x5f, 0x63, 0x6f, 0x6d, 0x70, 0x6f, 0x6e, 0x65, 0x6e, 0x74,
0x73, 0x18, 0x0f, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x69, 0x6e, 0x69, 0x74, 0x2e, 0x4b,
0x75, 0x62, 0x65, 0x72, 0x6e, 0x65, 0x74, 0x65, 0x73, 0x43, 0x6f, 0x6d, 0x70, 0x6f, 0x6e, 0x65,
0x6e, 0x74, 0x73, 0x12, 0x1f, 0x0a, 0x0b, 0x69, 0x6e, 0x69, 0x74, 0x5f, 0x73, 0x65, 0x63, 0x72,
0x65, 0x74, 0x18, 0x10, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0a, 0x69, 0x6e, 0x69, 0x74, 0x53, 0x65,
0x63, 0x72, 0x65, 0x74, 0x12, 0x21, 0x0a, 0x0c, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x5f,
0x6e, 0x61, 0x6d, 0x65, 0x18, 0x11, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x63, 0x6c, 0x75, 0x73,
0x74, 0x65, 0x72, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x2e, 0x0a, 0x13, 0x61, 0x70, 0x69, 0x73, 0x65,
0x72, 0x76, 0x65, 0x72, 0x5f, 0x63, 0x65, 0x72, 0x74, 0x5f, 0x73, 0x61, 0x6e, 0x73, 0x18, 0x12,
0x20, 0x03, 0x28, 0x09, 0x52, 0x11, 0x61, 0x70, 0x69, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x43,
0x65, 0x72, 0x74, 0x53, 0x61, 0x6e, 0x73, 0x22, 0xc1, 0x01, 0x0a, 0x0c, 0x49, 0x6e, 0x69, 0x74,
0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x3e, 0x0a, 0x0c, 0x69, 0x6e, 0x69, 0x74,
0x5f, 0x73, 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x19,
0x2e, 0x69, 0x6e, 0x69, 0x74, 0x2e, 0x49, 0x6e, 0x69, 0x74, 0x53, 0x75, 0x63, 0x63, 0x65, 0x73,
0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x48, 0x00, 0x52, 0x0b, 0x69, 0x6e, 0x69,
0x74, 0x53, 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, 0x12, 0x3e, 0x0a, 0x0c, 0x69, 0x6e, 0x69, 0x74,
0x5f, 0x66, 0x61, 0x69, 0x6c, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x19,
0x2e, 0x69, 0x6e, 0x69, 0x74, 0x2e, 0x49, 0x6e, 0x69, 0x74, 0x46, 0x61, 0x69, 0x6c, 0x75, 0x72,
0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x48, 0x00, 0x52, 0x0b, 0x69, 0x6e, 0x69,
0x74, 0x46, 0x61, 0x69, 0x6c, 0x75, 0x72, 0x65, 0x12, 0x29, 0x0a, 0x03, 0x6c, 0x6f, 0x67, 0x18,
0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x15, 0x2e, 0x69, 0x6e, 0x69, 0x74, 0x2e, 0x4c, 0x6f, 0x67,
0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x54, 0x79, 0x70, 0x65, 0x48, 0x00, 0x52, 0x03,
0x6c, 0x6f, 0x67, 0x42, 0x06, 0x0a, 0x04, 0x6b, 0x69, 0x6e, 0x64, 0x22, 0x6f, 0x0a, 0x13, 0x49,
0x6e, 0x69, 0x74, 0x53, 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e,
0x73, 0x65, 0x12, 0x1e, 0x0a, 0x0a, 0x6b, 0x75, 0x62, 0x65, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67,
0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0a, 0x6b, 0x75, 0x62, 0x65, 0x63, 0x6f, 0x6e, 0x66,
0x69, 0x67, 0x12, 0x19, 0x0a, 0x08, 0x6f, 0x77, 0x6e, 0x65, 0x72, 0x5f, 0x69, 0x64, 0x18, 0x02,
0x20, 0x01, 0x28, 0x0c, 0x52, 0x07, 0x6f, 0x77, 0x6e, 0x65, 0x72, 0x49, 0x64, 0x12, 0x1d, 0x0a,
0x0a, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x69, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28,
0x0c, 0x52, 0x09, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x49, 0x64, 0x22, 0x2b, 0x0a, 0x13,
0x49, 0x6e, 0x69, 0x74, 0x46, 0x61, 0x69, 0x6c, 0x75, 0x72, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f,
0x6e, 0x73, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x18, 0x01, 0x20, 0x01,
0x28, 0x09, 0x52, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x22, 0x23, 0x0a, 0x0f, 0x4c, 0x6f, 0x67,
0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x54, 0x79, 0x70, 0x65, 0x12, 0x10, 0x0a, 0x03,
0x6c, 0x6f, 0x67, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x03, 0x6c, 0x6f, 0x67, 0x22, 0x78,
0x0a, 0x13, 0x4b, 0x75, 0x62, 0x65, 0x72, 0x6e, 0x65, 0x74, 0x65, 0x73, 0x43, 0x6f, 0x6d, 0x70,
0x6f, 0x6e, 0x65, 0x6e, 0x74, 0x12, 0x10, 0x0a, 0x03, 0x75, 0x72, 0x6c, 0x18, 0x01, 0x20, 0x01,
0x28, 0x09, 0x52, 0x03, 0x75, 0x72, 0x6c, 0x12, 0x12, 0x0a, 0x04, 0x68, 0x61, 0x73, 0x68, 0x18,
0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x68, 0x61, 0x73, 0x68, 0x12, 0x21, 0x0a, 0x0c, 0x69,
0x6e, 0x73, 0x74, 0x61, 0x6c, 0x6c, 0x5f, 0x70, 0x61, 0x74, 0x68, 0x18, 0x03, 0x20, 0x01, 0x28,
0x09, 0x52, 0x0b, 0x69, 0x6e, 0x73, 0x74, 0x61, 0x6c, 0x6c, 0x50, 0x61, 0x74, 0x68, 0x12, 0x18,
0x0a, 0x07, 0x65, 0x78, 0x74, 0x72, 0x61, 0x63, 0x74, 0x18, 0x04, 0x20, 0x01, 0x28, 0x08, 0x52,
0x07, 0x65, 0x78, 0x74, 0x72, 0x61, 0x63, 0x74, 0x32, 0x36, 0x0a, 0x03, 0x41, 0x50, 0x49, 0x12,
0x2f, 0x0a, 0x04, 0x49, 0x6e, 0x69, 0x74, 0x12, 0x11, 0x2e, 0x69, 0x6e, 0x69, 0x74, 0x2e, 0x49,
0x6e, 0x69, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x12, 0x2e, 0x69, 0x6e, 0x69,
0x74, 0x2e, 0x49, 0x6e, 0x69, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x30, 0x01,
0x42, 0x40, 0x5a, 0x3e, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x65,
0x64, 0x67, 0x65, 0x6c, 0x65, 0x73, 0x73, 0x73, 0x79, 0x73, 0x2f, 0x63, 0x6f, 0x6e, 0x73, 0x74,
0x65, 0x6c, 0x6c, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x2f, 0x76, 0x32, 0x2f, 0x62, 0x6f, 0x6f, 0x74,
0x73, 0x74, 0x72, 0x61, 0x70, 0x70, 0x65, 0x72, 0x2f, 0x69, 0x6e, 0x69, 0x74, 0x70, 0x72, 0x6f,
0x74, 0x6f, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
0x6e, 0x74, 0x52, 0x14, 0x6b, 0x75, 0x62, 0x65, 0x72, 0x6e, 0x65, 0x74, 0x65, 0x73, 0x43, 0x6f,
0x6d, 0x70, 0x6f, 0x6e, 0x65, 0x6e, 0x74, 0x73, 0x12, 0x1f, 0x0a, 0x0b, 0x69, 0x6e, 0x69, 0x74,
0x5f, 0x73, 0x65, 0x63, 0x72, 0x65, 0x74, 0x18, 0x10, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0a, 0x69,
0x6e, 0x69, 0x74, 0x53, 0x65, 0x63, 0x72, 0x65, 0x74, 0x12, 0x21, 0x0a, 0x0c, 0x63, 0x6c, 0x75,
0x73, 0x74, 0x65, 0x72, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x11, 0x20, 0x01, 0x28, 0x09, 0x52,
0x0b, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x2e, 0x0a, 0x13,
0x61, 0x70, 0x69, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x5f, 0x63, 0x65, 0x72, 0x74, 0x5f, 0x73,
0x61, 0x6e, 0x73, 0x18, 0x12, 0x20, 0x03, 0x28, 0x09, 0x52, 0x11, 0x61, 0x70, 0x69, 0x73, 0x65,
0x72, 0x76, 0x65, 0x72, 0x43, 0x65, 0x72, 0x74, 0x53, 0x61, 0x6e, 0x73, 0x22, 0xc1, 0x01, 0x0a,
0x0c, 0x49, 0x6e, 0x69, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x3e, 0x0a,
0x0c, 0x69, 0x6e, 0x69, 0x74, 0x5f, 0x73, 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, 0x18, 0x01, 0x20,
0x01, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x69, 0x6e, 0x69, 0x74, 0x2e, 0x49, 0x6e, 0x69, 0x74, 0x53,
0x75, 0x63, 0x63, 0x65, 0x73, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x48, 0x00,
0x52, 0x0b, 0x69, 0x6e, 0x69, 0x74, 0x53, 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, 0x12, 0x3e, 0x0a,
0x0c, 0x69, 0x6e, 0x69, 0x74, 0x5f, 0x66, 0x61, 0x69, 0x6c, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20,
0x01, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x69, 0x6e, 0x69, 0x74, 0x2e, 0x49, 0x6e, 0x69, 0x74, 0x46,
0x61, 0x69, 0x6c, 0x75, 0x72, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x48, 0x00,
0x52, 0x0b, 0x69, 0x6e, 0x69, 0x74, 0x46, 0x61, 0x69, 0x6c, 0x75, 0x72, 0x65, 0x12, 0x29, 0x0a,
0x03, 0x6c, 0x6f, 0x67, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x15, 0x2e, 0x69, 0x6e, 0x69,
0x74, 0x2e, 0x4c, 0x6f, 0x67, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x54, 0x79, 0x70,
0x65, 0x48, 0x00, 0x52, 0x03, 0x6c, 0x6f, 0x67, 0x42, 0x06, 0x0a, 0x04, 0x6b, 0x69, 0x6e, 0x64,
0x22, 0x6f, 0x0a, 0x13, 0x49, 0x6e, 0x69, 0x74, 0x53, 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, 0x52,
0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x1e, 0x0a, 0x0a, 0x6b, 0x75, 0x62, 0x65, 0x63,
0x6f, 0x6e, 0x66, 0x69, 0x67, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0a, 0x6b, 0x75, 0x62,
0x65, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x19, 0x0a, 0x08, 0x6f, 0x77, 0x6e, 0x65, 0x72,
0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x07, 0x6f, 0x77, 0x6e, 0x65, 0x72,
0x49, 0x64, 0x12, 0x1d, 0x0a, 0x0a, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x69, 0x64,
0x18, 0x03, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x09, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x49,
0x64, 0x22, 0x2b, 0x0a, 0x13, 0x49, 0x6e, 0x69, 0x74, 0x46, 0x61, 0x69, 0x6c, 0x75, 0x72, 0x65,
0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x65, 0x72, 0x72, 0x6f,
0x72, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x22, 0x23,
0x0a, 0x0f, 0x4c, 0x6f, 0x67, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x54, 0x79, 0x70,
0x65, 0x12, 0x10, 0x0a, 0x03, 0x6c, 0x6f, 0x67, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x03,
0x6c, 0x6f, 0x67, 0x22, 0x78, 0x0a, 0x13, 0x4b, 0x75, 0x62, 0x65, 0x72, 0x6e, 0x65, 0x74, 0x65,
0x73, 0x43, 0x6f, 0x6d, 0x70, 0x6f, 0x6e, 0x65, 0x6e, 0x74, 0x12, 0x10, 0x0a, 0x03, 0x75, 0x72,
0x6c, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x75, 0x72, 0x6c, 0x12, 0x12, 0x0a, 0x04,
0x68, 0x61, 0x73, 0x68, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x68, 0x61, 0x73, 0x68,
0x12, 0x21, 0x0a, 0x0c, 0x69, 0x6e, 0x73, 0x74, 0x61, 0x6c, 0x6c, 0x5f, 0x70, 0x61, 0x74, 0x68,
0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x69, 0x6e, 0x73, 0x74, 0x61, 0x6c, 0x6c, 0x50,
0x61, 0x74, 0x68, 0x12, 0x18, 0x0a, 0x07, 0x65, 0x78, 0x74, 0x72, 0x61, 0x63, 0x74, 0x18, 0x04,
0x20, 0x01, 0x28, 0x08, 0x52, 0x07, 0x65, 0x78, 0x74, 0x72, 0x61, 0x63, 0x74, 0x32, 0x36, 0x0a,
0x03, 0x41, 0x50, 0x49, 0x12, 0x2f, 0x0a, 0x04, 0x49, 0x6e, 0x69, 0x74, 0x12, 0x11, 0x2e, 0x69,
0x6e, 0x69, 0x74, 0x2e, 0x49, 0x6e, 0x69, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a,
0x12, 0x2e, 0x69, 0x6e, 0x69, 0x74, 0x2e, 0x49, 0x6e, 0x69, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f,
0x6e, 0x73, 0x65, 0x30, 0x01, 0x42, 0x40, 0x5a, 0x3e, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e,
0x63, 0x6f, 0x6d, 0x2f, 0x65, 0x64, 0x67, 0x65, 0x6c, 0x65, 0x73, 0x73, 0x73, 0x79, 0x73, 0x2f,
0x63, 0x6f, 0x6e, 0x73, 0x74, 0x65, 0x6c, 0x6c, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x2f, 0x76, 0x32,
0x2f, 0x62, 0x6f, 0x6f, 0x74, 0x73, 0x74, 0x72, 0x61, 0x70, 0x70, 0x65, 0x72, 0x2f, 0x69, 0x6e,
0x69, 0x74, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
}
var (

2
bootstrapper/initproto/init.proto
View File

@ -19,7 +19,7 @@ message InitRequest {
string kubernetes_version = 8;
// repeated SSHUserKey ssh_user_keys = 9; removed
// bytes salt = 10; removed
bytes helm_deployments = 11;
// bytes helm_deployments = 11; removed
// repeated uint32 enforced_pcrs = 12; removed
// bool enforce_idkeydigest = 13; removed
bool conformance_mode = 14;

2
bootstrapper/internal/initserver/initserver.go
View File

@ -215,7 +215,6 @@ func (s *Server) Init(req *initproto.InitRequest, stream initproto.API_InitServe
kubeconfig, err := s.initializer.InitCluster(stream.Context(),
req.KubernetesVersion,
clusterName,
req.HelmDeployments,
req.ConformanceMode,
components.NewComponentsFromInitProto(req.KubernetesComponents),
req.ApiserverCertSans,
@ -342,7 +341,6 @@ type ClusterInitializer interface {
ctx context.Context,
k8sVersion string,
clusterName string,
helmDeployments []byte,
conformanceMode bool,
kubernetesComponents components.Components,
apiServerCertSANs []string,

2
bootstrapper/internal/initserver/initserver_test.go
View File

@ -407,7 +407,7 @@ type stubClusterInitializer struct {
func (i *stubClusterInitializer) InitCluster(
context.Context, string, string,
[]byte, bool, components.Components, []string, *logger.Logger,
bool, components.Components, []string, *logger.Logger,
) ([]byte, error) {
return i.initClusterKubeconfig, i.initClusterErr
}

2
bootstrapper/internal/kubernetes/BUILD.bazel
View File

@ -16,7 +16,6 @@ go_library(
"//internal/cloud/cloudprovider",
"//internal/cloud/metadata",
"//internal/constants",
"//internal/deploy/helm",
"//internal/kubernetes",
"//internal/logger",
"//internal/role",
@ -37,7 +36,6 @@ go_test(
"//bootstrapper/internal/kubernetes/kubewaiter",
"//internal/cloud/metadata",
"//internal/constants",
"//internal/deploy/helm",
"//internal/kubernetes",
"//internal/logger",
"//internal/role",

1
bootstrapper/internal/kubernetes/k8sapi/k8sutil.go
View File

@ -54,6 +54,7 @@ type Client interface {
ListAllNamespaces(ctx context.Context) (*corev1.NamespaceList, error)
AnnotateNode(ctx context.Context, nodeName, annotationKey, annotationValue string) error
EnforceCoreDNSSpread(ctx context.Context) error
PatchFirstNodePodCIDR(ctx context.Context, firstNodePodCIDR string) error
}
type componentsInstaller interface {

8
bootstrapper/internal/kubernetes/k8sutil.go
View File

@ -10,7 +10,6 @@ import (
"context"
"net"
"github.com/edgelesssys/constellation/v2/internal/deploy/helm"
"github.com/edgelesssys/constellation/v2/internal/logger"
"github.com/edgelesssys/constellation/v2/internal/role"
"github.com/edgelesssys/constellation/v2/internal/versions/components"
@ -24,10 +23,3 @@ type clusterUtil interface {
FixCilium(ctx context.Context) error
StartKubelet() error
}
// helmClient bundles functions related to microservice deployment.
// Only microservices that can be deployed purely via Helm are deployed with this interface.
type helmClient interface {
InstallChart(context.Context, helm.Release) error
InstallChartWithValues(ctx context.Context, release helm.Release, extraValues map[string]any) error
}

94
bootstrapper/internal/kubernetes/kubernetes.go
View File

@ -9,11 +9,8 @@ package kubernetes
import (
"context"
"encoding/json"
"errors"
"fmt"
"net"
"os/exec"
"regexp"
"strings"
"time"
@ -22,7 +19,6 @@ import (
"github.com/edgelesssys/constellation/v2/bootstrapper/internal/kubernetes/kubewaiter"
"github.com/edgelesssys/constellation/v2/internal/cloud/cloudprovider"
"github.com/edgelesssys/constellation/v2/internal/constants"
"github.com/edgelesssys/constellation/v2/internal/deploy/helm"
"github.com/edgelesssys/constellation/v2/internal/kubernetes"
"github.com/edgelesssys/constellation/v2/internal/logger"
"github.com/edgelesssys/constellation/v2/internal/role"
@ -49,7 +45,6 @@ type kubeAPIWaiter interface {
type KubeWrapper struct {
cloudProvider string
clusterUtil clusterUtil
helmClient helmClient
kubeAPIWaiter kubeAPIWaiter
configProvider configurationProvider
client k8sapi.Client
@ -59,12 +54,11 @@ type KubeWrapper struct {
// New creates a new KubeWrapper with real values.
func New(cloudProvider string, clusterUtil clusterUtil, configProvider configurationProvider, client k8sapi.Client,
providerMetadata ProviderMetadata, helmClient helmClient, kubeAPIWaiter kubeAPIWaiter,
providerMetadata ProviderMetadata, kubeAPIWaiter kubeAPIWaiter,
) *KubeWrapper {
return &KubeWrapper{
cloudProvider: cloudProvider,
clusterUtil: clusterUtil,
helmClient: helmClient,
kubeAPIWaiter: kubeAPIWaiter,
configProvider: configProvider,
client: client,
@ -75,15 +69,13 @@ func New(cloudProvider string, clusterUtil clusterUtil, configProvider configura
// InitCluster initializes a new Kubernetes cluster and applies pod network provider.
func (k *KubeWrapper) InitCluster(
ctx context.Context, versionString, clusterName string,
helmReleasesRaw []byte, conformanceMode bool, kubernetesComponents components.Components, apiServerCertSANs []string, log *logger.Logger,
ctx context.Context, versionString, clusterName string, conformanceMode bool, kubernetesComponents components.Components, apiServerCertSANs []string, log *logger.Logger,
) ([]byte, error) {
log.With(zap.String("version", versionString)).Infof("Installing Kubernetes components")
if err := k.clusterUtil.InstallComponents(ctx, kubernetesComponents); err != nil {
return nil, err
}
var nodePodCIDR string
var validIPs []net.IP
// Step 1: retrieve cloud metadata for Kubernetes configuration
@ -102,9 +94,6 @@ func (k *KubeWrapper) InitCluster(
nodeIP := instance.VPCIP
subnetworkPodCIDR := instance.SecondaryIPRange
if len(instance.AliasIPRanges) > 0 {
nodePodCIDR = instance.AliasIPRanges[0]
}
// this is the endpoint in "kubeadm init --control-plane-endpoint=<IP/DNS>:<port>"
// TODO(malt3): switch over to DNS name on AWS and Azure
@ -170,6 +159,17 @@ func (k *KubeWrapper) InitCluster(
return nil, fmt.Errorf("failed to setup k8s version ConfigMap: %w", err)
}
if cloudprovider.FromString(k.cloudProvider) == cloudprovider.GCP {
// GCP uses direct routing, so we need to set the pod CIDR of the first control-plane node for Cilium.
var nodePodCIDR string
if len(instance.AliasIPRanges) > 0 {
nodePodCIDR = instance.AliasIPRanges[0]
}
if err := k.client.PatchFirstNodePodCIDR(ctx, nodePodCIDR); err != nil {
return nil, fmt.Errorf("patching first node pod CIDR: %w", err)
}
}
// Annotate Node with the hash of the installed components
if err := k.client.AnnotateNode(ctx, nodeName,
constants.NodeKubernetesComponentsAnnotationKey, k8sComponentsConfigMap,
@ -177,49 +177,6 @@ func (k *KubeWrapper) InitCluster(
return nil, fmt.Errorf("annotating node with Kubernetes components hash: %w", err)
}
// Step 3: configure & start kubernetes controllers
log.Infof("Starting Kubernetes controllers and deployments")
setupPodNetworkInput := k8sapi.SetupPodNetworkInput{
CloudProvider: k.cloudProvider,
NodeName: nodeName,
FirstNodePodCIDR: nodePodCIDR,
SubnetworkPodCIDR: subnetworkPodCIDR,
LoadBalancerHost: controlPlaneHost,
LoadBalancerPort: controlPlanePort,
}
var helmReleases helm.Releases
if err := json.Unmarshal(helmReleasesRaw, &helmReleases); err != nil {
return nil, fmt.Errorf("unmarshalling helm releases: %w", err)
}
log.Infof("Installing Cilium")
ciliumVals, err := k.setupCiliumVals(ctx, setupPodNetworkInput)
if err != nil {
return nil, fmt.Errorf("setting up cilium vals: %w", err)
}
if err := k.helmClient.InstallChartWithValues(ctx, helmReleases.Cilium, ciliumVals); err != nil {
return nil, fmt.Errorf("installing cilium pod network: %w", err)
}
log.Infof("Waiting for Cilium to become healthy")
timeToStartWaiting := time.Now()
// TODO(3u13r): Reduce the timeout when we switched the package repository - this is only this high because we once
// saw polling times of ~16 minutes when hitting a slow PoP from Fastly (GitHub's / ghcr.io CDN).
waitCtx, cancel = context.WithTimeout(ctx, 20*time.Minute)
defer cancel()
if err := k.clusterUtil.WaitForCilium(waitCtx, log); err != nil {
return nil, fmt.Errorf("waiting for Cilium to become healthy: %w", err)
}
timeUntilFinishedWaiting := time.Since(timeToStartWaiting)
log.With(zap.Duration("duration", timeUntilFinishedWaiting)).Infof("Cilium became healthy")
log.Infof("Restarting Cilium")
if err := k.clusterUtil.FixCilium(ctx); err != nil {
log.With(zap.Error(err)).Errorf("FixCilium failed")
// Continue and don't throw an error here - things might be okay.
}
log.Infof("Setting up internal-config ConfigMap")
if err := k.setupInternalConfigMap(ctx); err != nil {
return nil, fmt.Errorf("failed to setup internal ConfigMap: %w", err)
@ -378,28 +335,3 @@ func getIPAddr() (string, error) {
return localAddr.IP.String(), nil
}
func (k *KubeWrapper) setupCiliumVals(ctx context.Context, in k8sapi.SetupPodNetworkInput) (map[string]any, error) {
vals := map[string]any{
"k8sServiceHost": in.LoadBalancerHost,
"k8sServicePort": in.LoadBalancerPort,
}
// GCP requires extra configuration for Cilium
if cloudprovider.FromString(k.cloudProvider) == cloudprovider.GCP {
if out, err := exec.CommandContext(
ctx, constants.KubectlPath,
"--kubeconfig", constants.ControlPlaneAdminConfFilename,
"patch", "node", in.NodeName, "-p", "{\"spec\":{\"podCIDR\": \""+in.FirstNodePodCIDR+"\"}}",
).CombinedOutput(); err != nil {
err = errors.New(string(out))
return nil, fmt.Errorf("%s: %w", out, err)
}
vals["ipv4NativeRoutingCIDR"] = in.SubnetworkPodCIDR
vals["strictModeCIDR"] = in.SubnetworkPodCIDR
}
return vals, nil
}

29
bootstrapper/internal/kubernetes/kubernetes_test.go
View File

@ -17,7 +17,6 @@ import (
"github.com/edgelesssys/constellation/v2/bootstrapper/internal/kubernetes/kubewaiter"
"github.com/edgelesssys/constellation/v2/internal/cloud/metadata"
"github.com/edgelesssys/constellation/v2/internal/constants"
"github.com/edgelesssys/constellation/v2/internal/deploy/helm"
"github.com/edgelesssys/constellation/v2/internal/kubernetes"
"github.com/edgelesssys/constellation/v2/internal/logger"
"github.com/edgelesssys/constellation/v2/internal/role"
@ -43,7 +42,6 @@ func TestInitCluster(t *testing.T) {
testCases := map[string]struct {
clusterUtil stubClusterUtil
helmClient stubHelmClient
kubectl stubKubectl
kubeAPIWaiter stubKubeAPIWaiter
providerMetadata ProviderMetadata
@ -131,14 +129,12 @@ func TestInitCluster(t *testing.T) {
},
"kubeadm init fails when deploying cilium": {
clusterUtil: stubClusterUtil{kubeconfig: []byte("someKubeconfig")},
helmClient: stubHelmClient{ciliumError: assert.AnError},
providerMetadata: &stubProviderMetadata{},
wantErr: true,
k8sVersion: versions.Default,
},
"kubeadm init fails when setting up constellation-services chart": {
clusterUtil: stubClusterUtil{kubeconfig: []byte("someKubeconfig")},
helmClient: stubHelmClient{installChartError: assert.AnError},
kubeAPIWaiter: stubKubeAPIWaiter{},
providerMetadata: &stubProviderMetadata{},
wantErr: true,
@ -182,7 +178,6 @@ func TestInitCluster(t *testing.T) {
kube := KubeWrapper{
cloudProvider: "aws", // provide a valid cloud provider for cilium installation
clusterUtil: &tc.clusterUtil,
helmClient: &tc.helmClient,
providerMetadata: tc.providerMetadata,
kubeAPIWaiter: &tc.kubeAPIWaiter,
configProvider: &stubConfigProvider{initConfig: k8sapi.KubeadmInitYAML{}},
@ -192,7 +187,7 @@ func TestInitCluster(t *testing.T) {
_, err := kube.InitCluster(
context.Background(), string(tc.k8sVersion), "kubernetes",
[]byte("{}"), false, nil, nil, logger.NewTest(t),
false, nil, nil, logger.NewTest(t),
)
if tc.wantErr {
@ -539,6 +534,10 @@ func (s *stubKubectl) AnnotateNode(_ context.Context, _, _, _ string) error {
return s.annotateNodeErr
}
func (s *stubKubectl) PatchFirstNodePodCIDR(_ context.Context, _ string) error {
return nil
}
func (s *stubKubectl) WaitForCRDs(_ context.Context, _ []string) error {
return s.waitForCRDsErr
}
@ -551,24 +550,6 @@ func (s *stubKubectl) EnforceCoreDNSSpread(_ context.Context) error {
return s.enforceCoreDNSSpreadErr
}
type stubHelmClient struct {
ciliumError error
installChartError error
}
func (s *stubHelmClient) InstallChart(ctx context.Context, release helm.Release) error {
return s.InstallChartWithValues(ctx, release, release.Values)
}
func (s *stubHelmClient) InstallChartWithValues(_ context.Context, release helm.Release, _ map[string]any) error {
switch release.ReleaseName {
case "cilium":
return s.ciliumError
default:
return s.installChartError
}
}
type stubKubeAPIWaiter struct {
waitErr error
}

3
cli/internal/cmd/BUILD.bazel
View File

@ -67,7 +67,6 @@ go_library(
"//internal/config/migration",
"//internal/constants",
"//internal/crypto",
"//internal/deploy/helm",
"//internal/file",
"//internal/grpc/dialer",
"//internal/grpc/grpclog",
@ -139,6 +138,7 @@ go_test(
"//bootstrapper/initproto",
"//cli/internal/cloudcmd",
"//cli/internal/clusterid",
"//cli/internal/helm",
"//cli/internal/iamid",
"//cli/internal/kubernetes",
"//cli/internal/terraform",
@ -154,7 +154,6 @@ go_test(
"//internal/config",
"//internal/constants",
"//internal/crypto/testvector",
"//internal/deploy/helm",
"//internal/file",
"//internal/grpc/atlscredentials",
"//internal/grpc/dialer",

17
cli/internal/cmd/init.go
View File

@ -10,7 +10,6 @@ import (
"bytes"
"context"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"io"
@ -45,7 +44,6 @@ import (
"github.com/edgelesssys/constellation/v2/internal/config"
"github.com/edgelesssys/constellation/v2/internal/constants"
"github.com/edgelesssys/constellation/v2/internal/crypto"
helmdeploy "github.com/edgelesssys/constellation/v2/internal/deploy/helm"
"github.com/edgelesssys/constellation/v2/internal/file"
"github.com/edgelesssys/constellation/v2/internal/grpc/dialer"
"github.com/edgelesssys/constellation/v2/internal/grpc/grpclog"
@ -79,7 +77,7 @@ type initCmd struct {
spinner spinnerInterf
masterSecret uri.MasterSecret
fh *file.Handler
helmInstaller helm.SuiteInstaller
helmInstaller helm.Initializer
}
// runInitialize runs the initialize command.
@ -103,7 +101,7 @@ func runInitialize(cmd *cobra.Command, _ []string) error {
ctx, cancel := context.WithTimeout(cmd.Context(), time.Hour)
defer cancel()
cmd.SetContext(ctx)
helmInstaller, err := helm.NewInstallationClient(log)
helmInstaller, err := helm.NewInitializer(log)
if err != nil {
return fmt.Errorf("creating Helm installer: %w", err)
}
@ -191,10 +189,6 @@ func (i *initCmd) initialize(cmd *cobra.Command, newDialer func(validator atls.V
if err != nil {
return fmt.Errorf("loading Helm charts: %w", err)
}
helmDeployments, err := json.Marshal(releases)
if err != nil {
return err
}
i.log.Debugf("Loaded Helm deployments")
if err != nil {
return fmt.Errorf("loading Helm charts: %w", err)
@ -208,7 +202,6 @@ func (i *initCmd) initialize(cmd *cobra.Command, newDialer func(validator atls.V
CloudServiceAccountUri: serviceAccURI,
KubernetesVersion: versions.VersionConfigs[k8sVersion].ClusterVersion,
KubernetesComponents: versions.VersionConfigs[k8sVersion].KubernetesComponents.ToInitProto(),
HelmDeployments: helmDeployments,
ConformanceMode: flags.conformance,
InitSecret: idFile.InitSecret,
ClusterName: clusterName,
@ -441,9 +434,9 @@ func (i *initCmd) evalFlagArgs(cmd *cobra.Command) (initFlags, error) {
if err != nil {
return initFlags{}, fmt.Errorf("parsing skip-helm-wait flag: %w", err)
}
helmWaitMode := helmdeploy.WaitModeAtomic
helmWaitMode := helm.WaitModeAtomic
if skipHelmWait {
helmWaitMode = helmdeploy.WaitModeNone
helmWaitMode = helm.WaitModeNone
}
i.log.Debugf("Helm wait flag is %t", skipHelmWait)
configPath, err := cmd.Flags().GetString("config")
@ -478,7 +471,7 @@ type initFlags struct {
configPath string
masterSecretPath string
conformance bool
helmWaitMode helmdeploy.WaitMode
helmWaitMode helm.WaitMode
force bool
mergeConfigs bool
}

4
cli/internal/cmd/init_test.go
View File

@ -21,6 +21,7 @@ import (
"github.com/edgelesssys/constellation/v2/bootstrapper/initproto"
"github.com/edgelesssys/constellation/v2/cli/internal/clusterid"
"github.com/edgelesssys/constellation/v2/cli/internal/helm"
"github.com/edgelesssys/constellation/v2/internal/atls"
"github.com/edgelesssys/constellation/v2/internal/attestation/measurements"
"github.com/edgelesssys/constellation/v2/internal/attestation/variant"
@ -28,7 +29,6 @@ import (
"github.com/edgelesssys/constellation/v2/internal/cloud/gcpshared"
"github.com/edgelesssys/constellation/v2/internal/config"
"github.com/edgelesssys/constellation/v2/internal/constants"
helminstaller "github.com/edgelesssys/constellation/v2/internal/deploy/helm"
"github.com/edgelesssys/constellation/v2/internal/file"
"github.com/edgelesssys/constellation/v2/internal/grpc/atlscredentials"
"github.com/edgelesssys/constellation/v2/internal/grpc/dialer"
@ -672,7 +672,7 @@ type stubHelmInstaller struct{}
func (i *stubHelmInstaller) Install(_ context.Context, _ cloudprovider.Provider, _ uri.MasterSecret,
_ clusterid.File,
_ string, _ *helminstaller.Releases,
_ string, _ *helm.Releases,
) error {
return nil
}

2
cli/internal/cmd/status.go
View File

@ -74,7 +74,7 @@ func runStatus(cmd *cobra.Command, _ []string) error {
}
// need helm client to fetch service versions.
helmClient, err := helm.NewClient(kubectl.New(), constants.AdminConfFilename, constants.HelmNamespace, log)
helmClient, err := helm.NewUpgradeClient(kubectl.New(), constants.AdminConfFilename, constants.HelmNamespace, log)
if err != nil {
return fmt.Errorf("setting up helm client: %w", err)
}

2
cli/internal/cmd/upgradecheck.go
View File

@ -383,7 +383,7 @@ type currentVersionInfo struct {
}
func (v *versionCollector) currentVersions(ctx context.Context) (currentVersionInfo, error) {
helmClient, err := helm.NewClient(kubectl.New(), constants.AdminConfFilename, constants.HelmNamespace, v.log)
helmClient, err := helm.NewUpgradeClient(kubectl.New(), constants.AdminConfFilename, constants.HelmNamespace, v.log)
if err != nil {
return currentVersionInfo{}, fmt.Errorf("setting up helm client: %w", err)
}

18
cli/internal/helm/BUILD.bazel
View File

@ -5,12 +5,14 @@ go_library(
name = "helm",
srcs = [
"backup.go",
"client.go",
"ciliumhelper.go",
"helm.go",
"helminstaller.go",
"init.go",
"install.go",
"loader.go",
"release.go",
"serviceversion.go",
"setup.go",
"upgrade.go",
"values.go",
],
embedsrcs = [
@ -427,17 +429,21 @@ go_library(
"//internal/compatibility",
"//internal/config",
"//internal/constants",
"//internal/deploy/helm",
"//internal/file",
"//internal/kms/uri",
"//internal/retry",
"//internal/semver",
"//internal/versions",
"@com_github_pkg_errors//:errors",
"@com_github_spf13_afero//:afero",
"@io_k8s_apiextensions_apiserver//pkg/apis/apiextensions/v1:apiextensions",
"@io_k8s_apimachinery//pkg/api/errors",
"@io_k8s_apimachinery//pkg/apis/meta/v1:meta",
"@io_k8s_apimachinery//pkg/apis/meta/v1/unstructured",
"@io_k8s_apimachinery//pkg/runtime/schema",
"@io_k8s_apimachinery//pkg/util/wait",
"@io_k8s_client_go//kubernetes",
"@io_k8s_client_go//tools/clientcmd",
"@io_k8s_sigs_yaml//:yaml",
"@sh_helm_helm//pkg/ignore",
"@sh_helm_helm_v3//pkg/action",
@ -453,8 +459,9 @@ go_test(
name = "helm_test",
srcs = [
"backup_test.go",
"client_test.go",
"helm_test.go",
"loader_test.go",
"upgrade_test.go",
],
data = glob(["testdata/**"]),
embed = [":helm"],
@ -465,7 +472,6 @@ go_test(
"//internal/cloud/cloudprovider",
"//internal/compatibility",
"//internal/config",
"//internal/deploy/helm",
"//internal/file",
"//internal/logger",
"//internal/semver",

8
cli/internal/helm/backup.go
View File

@ -18,7 +18,7 @@ import (
"sigs.k8s.io/yaml"
)
func (c *Client) backupCRDs(ctx context.Context, upgradeID string) ([]apiextensionsv1.CustomResourceDefinition, error) {
func (c *UpgradeClient) backupCRDs(ctx context.Context, upgradeID string) ([]apiextensionsv1.CustomResourceDefinition, error) {
c.log.Debugf("Starting CRD backup")
crds, err := c.kubectl.GetCRDs(ctx)
if err != nil {
@ -53,7 +53,7 @@ func (c *Client) backupCRDs(ctx context.Context, upgradeID string) ([]apiextensi
return crds, nil
}
func (c *Client) backupCRs(ctx context.Context, crds []apiextensionsv1.CustomResourceDefinition, upgradeID string) error {
func (c *UpgradeClient) backupCRs(ctx context.Context, crds []apiextensionsv1.CustomResourceDefinition, upgradeID string) error {
c.log.Debugf("Starting CR backup")
for _, crd := range crds {
c.log.Debugf("Creating backup for resource type: %s", crd.Name)
@ -99,10 +99,10 @@ func (c *Client) backupCRs(ctx context.Context, crds []apiextensionsv1.CustomRes
return nil
}
func (c *Client) backupFolder(upgradeID string) string {
func (c *UpgradeClient) backupFolder(upgradeID string) string {
return filepath.Join(constants.UpgradeDir, upgradeID, "backups") + string(filepath.Separator)
}
func (c *Client) crdBackupFolder(upgradeID string) string {
func (c *UpgradeClient) crdBackupFolder(upgradeID string) string {
return filepath.Join(c.backupFolder(upgradeID), "crds") + string(filepath.Separator)
}

4
cli/internal/helm/backup_test.go
View File

@ -52,7 +52,7 @@ func TestBackupCRDs(t *testing.T) {
crd := apiextensionsv1.CustomResourceDefinition{}
err := yaml.Unmarshal([]byte(tc.crd), &crd)
require.NoError(err)
client := Client{
client := UpgradeClient{
config: nil,
kubectl: stubCrdClient{crds: []apiextensionsv1.CustomResourceDefinition{crd}, getCRDsError: tc.getCRDsError},
fs: file.NewHandler(memFs),
@ -143,7 +143,7 @@ func TestBackupCRs(t *testing.T) {
require := require.New(t)
memFs := afero.NewMemMapFs()
client := Client{
client := UpgradeClient{
config: nil,
kubectl: stubCrdClient{crs: []unstructured.Unstructured{tc.resource}, getCRsError: tc.getCRsError},
fs: file.NewHandler(memFs),

72
cli/internal/helm/ciliumhelper.go Normal file
View File

@ -0,0 +1,72 @@
/*
Copyright (c) Edgeless Systems GmbH
SPDX-License-Identifier: AGPL-3.0-only
*/
package helm
import (
"context"
"fmt"
"time"
v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/tools/clientcmd"
)
type k8sDsClient struct {
clientset *kubernetes.Clientset
}
func newK8sCiliumHelper(kubeconfigPath string) (*k8sDsClient, error) {
config, err := clientcmd.BuildConfigFromFlags("", kubeconfigPath)
if err != nil {
return nil, err
}
clientset, err := kubernetes.NewForConfig(config)
if err != nil {
return nil, err
}
return &k8sDsClient{clientset: clientset}, nil
}
// WaitForDS waits for a DaemonSet to become ready.
func (h *k8sDsClient) WaitForDS(ctx context.Context, namespace, name string, log debugLog) error {
for {
select {
case <-ctx.Done():
return fmt.Errorf("context expired before DaemonSet %q became ready", name)
default:
ds, err := h.clientset.AppsV1().DaemonSets(namespace).Get(ctx, name, v1.GetOptions{})
if err != nil {
return err
}
if ds.Status.NumberReady == ds.Status.DesiredNumberScheduled {
log.Debugf("DaemonSet %s is ready\n", name)
return nil
}
log.Debugf("Waiting for DaemonSet %s to become ready...\n", name)
time.Sleep(10 * time.Second)
}
}
}
// RestartDS restarts all pods of a DaemonSet by updating its template.
func (h *k8sDsClient) RestartDS(namespace, name string) error {
ds, err := h.clientset.AppsV1().DaemonSets(namespace).Get(context.Background(), name, v1.GetOptions{})
if err != nil {
return err
}
ds.Spec.Template.ObjectMeta.Annotations["restartTimestamp"] = fmt.Sprintf("%d", time.Now().Unix())
_, err = h.clientset.AppsV1().DaemonSets(namespace).Update(context.Background(), ds, v1.UpdateOptions{})
if err != nil {
return err
}
return nil
}

22
cli/internal/helm/helm.go
View File

@ -15,3 +15,25 @@ It is used by the CLI to:
- create local backups before running service upgrades
*/
package helm
// mergeMaps returns a new map that is the merger of it's inputs.
// Key collisions are resolved by taking the value of the second argument (map b).
// Taken from: https://github.com/helm/helm/blob/dbc6d8e20fe1d58d50e6ed30f09a04a77e4c68db/pkg/cli/values/options.go#L91-L108.
func mergeMaps(a, b map[string]any) map[string]any {
out := make(map[string]any, len(a))
for k, v := range a {
out[k] = v
}
for k, v := range b {
if v, ok := v.(map[string]any); ok {
if bv, ok := out[k]; ok {
if bv, ok := bv.(map[string]any); ok {
out[k] = mergeMaps(bv, v)
continue
}
}
}
out[k] = v
}
return out
}

2
internal/deploy/helm/helm_test.go → cli/internal/helm/helm_test.go
View File

@ -102,7 +102,7 @@ func TestMergeMaps(t *testing.T) {
for name, tc := range testCases {
t.Run(name, func(t *testing.T) {
assert := assert.New(t)
newVals := MergeMaps(tc.vals, tc.extraVals)
newVals := mergeMaps(tc.vals, tc.extraVals)
assert.Equal(tc.expected, newVals)
})
}

100
cli/internal/helm/helminstaller.go
View File

@ -1,100 +0,0 @@
/*
Copyright (c) Edgeless Systems GmbH
SPDX-License-Identifier: AGPL-3.0-only
*/
package helm
import (
"context"
"fmt"
"github.com/edgelesssys/constellation/v2/cli/internal/clusterid"
"github.com/edgelesssys/constellation/v2/internal/cloud/cloudprovider"
"github.com/edgelesssys/constellation/v2/internal/cloud/openstack"
"github.com/edgelesssys/constellation/v2/internal/constants"
helminstaller "github.com/edgelesssys/constellation/v2/internal/deploy/helm"
"github.com/edgelesssys/constellation/v2/internal/kms/uri"
)
// SuiteInstaller installs all Helm charts required for a constellation cluster.
type SuiteInstaller interface {
Install(ctx context.Context, provider cloudprovider.Provider, masterSecret uri.MasterSecret,
idFile clusterid.File,
serviceAccURI string, releases *helminstaller.Releases,
) error
}
type helmInstallationClient struct {
log debugLog
installer helmInstaller
}
// NewInstallationClient creates a new Helm installation client to install all Helm charts required for a constellation cluster.
func NewInstallationClient(log debugLog) (SuiteInstaller, error) {
installer, err := helminstaller.NewInstaller(constants.AdminConfFilename, log)
if err != nil {
return nil, fmt.Errorf("creating Helm installer: %w", err)
}
return &helmInstallationClient{log: log, installer: installer}, nil
}
func (h helmInstallationClient) Install(ctx context.Context, provider cloudprovider.Provider, masterSecret uri.MasterSecret,
idFile clusterid.File,
serviceAccURI string, releases *helminstaller.Releases,
) error {
serviceVals, err := setupMicroserviceVals(ctx, provider, masterSecret.Salt, idFile.UID, serviceAccURI)
if err != nil {
return fmt.Errorf("setting up microservice values: %w", err)
}
if err := h.installer.InstallChartWithValues(ctx, releases.ConstellationServices, serviceVals); err != nil {
return fmt.Errorf("installing microservices: %w", err)
}
h.log.Debugf("Installing cert-manager")
if err := h.installer.InstallChart(ctx, releases.CertManager); err != nil {
return fmt.Errorf("installing cert-manager: %w", err)
}
if releases.CSI != nil {
var csiVals map[string]any
if provider == cloudprovider.OpenStack {
creds, err := openstack.AccountKeyFromURI(serviceAccURI)
if err != nil {
return err
}
cinderIni := creds.CloudINI().CinderCSIConfiguration()
csiVals = map[string]any{
"cinder-config": map[string]any{
"secretData": cinderIni,
},
}
}
h.log.Debugf("Installing CSI deployments")
if err := h.installer.InstallChartWithValues(ctx, *releases.CSI, csiVals); err != nil {
return fmt.Errorf("installing CSI snapshot CRDs: %w", err)
}
}
if releases.AWSLoadBalancerController != nil {
h.log.Debugf("Installing AWS Load Balancer Controller")
if err := h.installer.InstallChart(ctx, *releases.AWSLoadBalancerController); err != nil {
return fmt.Errorf("installing AWS Load Balancer Controller: %w", err)
}
}
h.log.Debugf("Installing constellation operators")
operatorVals := setupOperatorVals(ctx, idFile.UID)
if err := h.installer.InstallChartWithValues(ctx, releases.ConstellationOperators, operatorVals); err != nil {
return fmt.Errorf("installing constellation operators: %w", err)
}
// TODO(elchead): AB#3301 do cilium after version upgrade
return nil
}
type helmInstaller interface {
InstallChart(context.Context, helminstaller.Release) error
InstallChartWithValues(ctx context.Context, release helminstaller.Release, extraValues map[string]any) error
}

258
cli/internal/helm/init.go Normal file
View File

@ -0,0 +1,258 @@
/*
Copyright (c) Edgeless Systems GmbH
SPDX-License-Identifier: AGPL-3.0-only
*/
package helm
import (
"context"
"encoding/base64"
"encoding/json"
"fmt"
"time"
"github.com/edgelesssys/constellation/v2/cli/internal/clusterid"
"github.com/edgelesssys/constellation/v2/cli/internal/terraform"
"github.com/edgelesssys/constellation/v2/internal/cloud/azureshared"
"github.com/edgelesssys/constellation/v2/internal/cloud/cloudprovider"
"github.com/edgelesssys/constellation/v2/internal/cloud/gcpshared"
"github.com/edgelesssys/constellation/v2/internal/cloud/openstack"
"github.com/edgelesssys/constellation/v2/internal/constants"
"github.com/edgelesssys/constellation/v2/internal/kms/uri"
)
// Initializer installs all Helm charts required for a constellation cluster.
type Initializer interface {
Install(ctx context.Context, provider cloudprovider.Provider, masterSecret uri.MasterSecret,
idFile clusterid.File,
serviceAccURI string, releases *Releases,
) error
}
type initializationClient struct {
log debugLog
installer installer
}
// NewInitializer creates a new client to install all Helm charts required for a constellation cluster.
func NewInitializer(log debugLog) (Initializer, error) {
installer, err := NewInstaller(constants.AdminConfFilename, log)
if err != nil {
return nil, fmt.Errorf("creating Helm installer: %w", err)
}
return &initializationClient{log: log, installer: installer}, nil
}
// Install installs all Helm charts required for a constellation cluster.
func (h initializationClient) Install(ctx context.Context, provider cloudprovider.Provider, masterSecret uri.MasterSecret,
idFile clusterid.File,
serviceAccURI string, releases *Releases,
) error {
tfClient, err := terraform.New(ctx, constants.TerraformWorkingDir)
if err != nil {
return fmt.Errorf("creating Terraform client: %w", err)
}
output, err := tfClient.ShowCluster(ctx, provider)
if err != nil {
return fmt.Errorf("getting Terraform output: %w", err)
}
ciliumVals := setupCiliumVals(provider, output)
if err := h.installer.InstallChartWithValues(ctx, releases.Cilium, ciliumVals); err != nil {
return fmt.Errorf("installing Cilium: %w", err)
}
h.log.Debugf("Waiting for Cilium to become ready")
helper, err := newK8sCiliumHelper(constants.AdminConfFilename)
if err != nil {
return fmt.Errorf("creating Kubernetes client: %w", err)
}
timeToStartWaiting := time.Now()
// TODO(3u13r): Reduce the timeout when we switched the package repository - this is only this high because we once
// saw polling times of ~16 minutes when hitting a slow PoP from Fastly (GitHub's / ghcr.io CDN).
if err := helper.WaitForDS(ctx, "kube-system", "cilium", h.log); err != nil {
return fmt.Errorf("waiting for Cilium to become healthy: %w", err)
}
timeUntilFinishedWaiting := time.Since(timeToStartWaiting)
h.log.Debugf("Cilium became healthy after %s", timeUntilFinishedWaiting.String())
h.log.Debugf("Fix Cilium through restart")
if err := helper.RestartDS("kube-system", "cilium"); err != nil {
return fmt.Errorf("restarting Cilium: %w", err)
}
h.log.Debugf("Installing microservices")
serviceVals, err := setupMicroserviceVals(provider, masterSecret.Salt, idFile.UID, serviceAccURI, output)
if err != nil {
return fmt.Errorf("setting up microservice values: %w", err)
}
if err := h.installer.InstallChartWithValues(ctx, releases.ConstellationServices, serviceVals); err != nil {
return fmt.Errorf("installing microservices: %w", err)
}
h.log.Debugf("Installing cert-manager")
if err := h.installer.InstallChart(ctx, releases.CertManager); err != nil {
return fmt.Errorf("installing cert-manager: %w", err)
}
if releases.CSI != nil {
var csiVals map[string]any
if provider == cloudprovider.OpenStack {
creds, err := openstack.AccountKeyFromURI(serviceAccURI)
if err != nil {
return err
}
cinderIni := creds.CloudINI().CinderCSIConfiguration()
csiVals = map[string]any{
"cinder-config": map[string]any{
"secretData": cinderIni,
},
}
}
h.log.Debugf("Installing CSI deployments")
if err := h.installer.InstallChartWithValues(ctx, *releases.CSI, csiVals); err != nil {
return fmt.Errorf("installing CSI snapshot CRDs: %w", err)
}
}
if releases.AWSLoadBalancerController != nil {
h.log.Debugf("Installing AWS Load Balancer Controller")
if err := h.installer.InstallChart(ctx, *releases.AWSLoadBalancerController); err != nil {
return fmt.Errorf("installing AWS Load Balancer Controller: %w", err)
}
}
h.log.Debugf("Installing constellation operators")
operatorVals := setupOperatorVals(ctx, idFile.UID)
if err := h.installer.InstallChartWithValues(ctx, releases.ConstellationOperators, operatorVals); err != nil {
return fmt.Errorf("installing constellation operators: %w", err)
}
return nil
}
// installer is the interface for installing a single Helm chart.
type installer interface {
InstallChart(context.Context, Release) error
InstallChartWithValues(ctx context.Context, release Release, extraValues map[string]any) error
}
// TODO(malt3): switch over to DNS name on AWS and Azure
// soon as every apiserver certificate of every control-plane node
// has the dns endpoint in its SAN list.
func setupCiliumVals(provider cloudprovider.Provider, output terraform.ApplyOutput) map[string]any {
vals := map[string]any{
"k8sServiceHost": output.IP,
"k8sServicePort": constants.KubernetesPort,
}
if provider == cloudprovider.GCP {
vals["ipv4NativeRoutingCIDR"] = output.GCP.IPCidrPod
vals["strictModeCIDR"] = output.GCP.IPCidrPod
}
return vals
}
// setupMicroserviceVals returns the values for the microservice chart.
func setupMicroserviceVals(provider cloudprovider.Provider, measurementSalt []byte, uid, serviceAccURI string, output terraform.ApplyOutput) (map[string]any, error) {
extraVals := map[string]any{
"join-service": map[string]any{
"measurementSalt": base64.StdEncoding.EncodeToString(measurementSalt),
},
"verification-service": map[string]any{
"loadBalancerIP": output.IP,
},
"konnectivity": map[string]any{
"loadBalancerIP": output.IP,
},
}
switch provider {
case cloudprovider.GCP:
serviceAccountKey, err := gcpshared.ServiceAccountKeyFromURI(serviceAccURI)
if err != nil {
return nil, fmt.Errorf("getting service account key: %w", err)
}
rawKey, err := json.Marshal(serviceAccountKey)
if err != nil {
return nil, fmt.Errorf("marshaling service account key: %w", err)
}
if output.GCP == nil {
return nil, fmt.Errorf("no GCP output from Terraform")
}
extraVals["ccm"] = map[string]any{
"GCP": map[string]any{
"projectID": output.GCP.ProjectID,
"uid": uid,
"secretData": string(rawKey),
"subnetworkPodCIDR": output.GCP.IPCidrPod,
},
}
case cloudprovider.Azure:
if output.Azure == nil {
return nil, fmt.Errorf("no Azure output from Terraform")
}
ccmConfig, err := getCCMConfig(*output.Azure, serviceAccURI)
if err != nil {
return nil, fmt.Errorf("getting Azure CCM config: %w", err)
}
extraVals["ccm"] = map[string]any{
"Azure": map[string]any{
"azureConfig": string(ccmConfig),
},
}
}
return extraVals, nil
}
// setupOperatorVals returns the values for the constellation-operator chart.
func setupOperatorVals(_ context.Context, uid string) map[string]any {
return map[string]any{
"constellation-operator": map[string]any{
"constellationUID": uid,
},
}
}
type cloudConfig struct {
Cloud string `json:"cloud,omitempty"`
TenantID string `json:"tenantId,omitempty"`
SubscriptionID string `json:"subscriptionId,omitempty"`
ResourceGroup string `json:"resourceGroup,omitempty"`
Location string `json:"location,omitempty"`
SubnetName string `json:"subnetName,omitempty"`
SecurityGroupName string `json:"securityGroupName,omitempty"`
SecurityGroupResourceGroup string `json:"securityGroupResourceGroup,omitempty"`
LoadBalancerName string `json:"loadBalancerName,omitempty"`
LoadBalancerSku string `json:"loadBalancerSku,omitempty"`
VNetName string `json:"vnetName,omitempty"`
VNetResourceGroup string `json:"vnetResourceGroup,omitempty"`
CloudProviderBackoff bool `json:"cloudProviderBackoff,omitempty"`
UseInstanceMetadata bool `json:"useInstanceMetadata,omitempty"`
VMType string `json:"vmType,omitempty"`
UseManagedIdentityExtension bool `json:"useManagedIdentityExtension,omitempty"`
UserAssignedIdentityID string `json:"userAssignedIdentityID,omitempty"`
}
// GetCCMConfig returns the configuration needed for the Kubernetes Cloud Controller Manager on Azure.
func getCCMConfig(tfOutput terraform.AzureApplyOutput, serviceAccURI string) ([]byte, error) {
creds, err := azureshared.ApplicationCredentialsFromURI(serviceAccURI)
if err != nil {
return nil, fmt.Errorf("getting service account key: %w", err)
}
useManagedIdentityExtension := creds.PreferredAuthMethod == azureshared.AuthMethodUserAssignedIdentity
config := cloudConfig{
Cloud: "AzurePublicCloud",
TenantID: creds.TenantID,
SubscriptionID: tfOutput.SubscriptionID,
ResourceGroup: tfOutput.ResourceGroup,
LoadBalancerSku: "standard",
SecurityGroupName: tfOutput.NetworkSecurityGroupName,
LoadBalancerName: tfOutput.LoadBalancerName,
UseInstanceMetadata: true,
VMType: "vmss",
Location: creds.Location,
UseManagedIdentityExtension: useManagedIdentityExtension,
UserAssignedIdentityID: tfOutput.UserAssignedIdentity,
}
return json.Marshal(config)
}

2
internal/deploy/helm/install.go → cli/internal/helm/install.go
View File

@ -68,7 +68,7 @@ func (h *Installer) InstallChart(ctx context.Context, release Release) error {
// InstallChartWithValues is the generic install function for helm charts with custom values.
func (h *Installer) InstallChartWithValues(ctx context.Context, release Release, extraValues map[string]any) error {
mergedVals := MergeMaps(release.Values, extraValues)
mergedVals := mergeMaps(release.Values, extraValues)
h.ReleaseName = release.ReleaseName
if err := h.SetWaitMode(release.WaitMode); err != nil {
return err

28
cli/internal/helm/loader.go
View File

@ -27,7 +27,6 @@ import (
"github.com/edgelesssys/constellation/v2/internal/cloud/cloudprovider"
"github.com/edgelesssys/constellation/v2/internal/config"
"github.com/edgelesssys/constellation/v2/internal/constants"
"github.com/edgelesssys/constellation/v2/internal/deploy/helm"
"github.com/edgelesssys/constellation/v2/internal/semver"
"github.com/edgelesssys/constellation/v2/internal/versions"
)
@ -108,21 +107,8 @@ func NewLoader(csp cloudprovider.Provider, k8sVersion versions.ValidK8sVersion,
}
}
// Load the embedded helm charts.
func (i *ChartLoader) Load(config *config.Config, conformanceMode bool, helmWaitMode helm.WaitMode, masterSecret, salt []byte) ([]byte, error) {
releases, err := i.LoadReleases(config, conformanceMode, helmWaitMode, masterSecret, salt)
if err != nil {
return nil, fmt.Errorf("loading releases: %w", err)
}
rel, err := json.Marshal(releases)
if err != nil {
return nil, err
}
return rel, nil
}
// LoadReleases loads the embedded helm charts and returns them as a HelmReleases object.
func (i *ChartLoader) LoadReleases(config *config.Config, conformanceMode bool, helmWaitMode helm.WaitMode, masterSecret, salt []byte) (*helm.Releases, error) {
func (i *ChartLoader) LoadReleases(config *config.Config, conformanceMode bool, helmWaitMode WaitMode, masterSecret, salt []byte) (*Releases, error) {
ciliumRelease, err := i.loadRelease(ciliumInfo, helmWaitMode)
if err != nil {
return nil, fmt.Errorf("loading cilium: %w", err)
@ -147,7 +133,7 @@ func (i *ChartLoader) LoadReleases(config *config.Config, conformanceMode bool,
return nil, fmt.Errorf("extending constellation-services values: %w", err)
}
releases := helm.Releases{Cilium: ciliumRelease, CertManager: certManagerRelease, ConstellationOperators: operatorRelease, ConstellationServices: conServicesRelease}
releases := Releases{Cilium: ciliumRelease, CertManager: certManagerRelease, ConstellationOperators: operatorRelease, ConstellationServices: conServicesRelease}
if config.HasProvider(cloudprovider.AWS) {
awsRelease, err := i.loadRelease(awsLBControllerInfo, helmWaitMode)
if err != nil {
@ -168,10 +154,10 @@ func (i *ChartLoader) LoadReleases(config *config.Config, conformanceMode bool,
// loadRelease loads the embedded chart and values depending on the given info argument.
// IMPORTANT: .helmignore rules specifying files in subdirectories are not applied (e.g. crds/kustomization.yaml).
func (i *ChartLoader) loadRelease(info chartInfo, helmWaitMode helm.WaitMode) (helm.Release, error) {
func (i *ChartLoader) loadRelease(info chartInfo, helmWaitMode WaitMode) (Release, error) {
chart, err := loadChartsDir(helmFS, info.path)
if err != nil {
return helm.Release{}, fmt.Errorf("loading %s chart: %w", info.releaseName, err)
return Release{}, fmt.Errorf("loading %s chart: %w", info.releaseName, err)
}
var values map[string]any
@ -181,7 +167,7 @@ func (i *ChartLoader) loadRelease(info chartInfo, helmWaitMode helm.WaitMode) (h
var ok bool
values, ok = ciliumVals[i.csp.String()]
if !ok {
return helm.Release{}, fmt.Errorf("cilium values for csp %q not found", i.csp.String())
return Release{}, fmt.Errorf("cilium values for csp %q not found", i.csp.String())
}
case certManagerInfo.releaseName:
values = i.loadCertManagerValues()
@ -200,10 +186,10 @@ func (i *ChartLoader) loadRelease(info chartInfo, helmWaitMode helm.WaitMode) (h
chartRaw, err := i.marshalChart(chart)
if err != nil {
return helm.Release{}, fmt.Errorf("packaging %s chart: %w", info.releaseName, err)
return Release{}, fmt.Errorf("packaging %s chart: %w", info.releaseName, err)
}
return helm.Release{Chart: chartRaw, Values: values, ReleaseName: info.releaseName, WaitMode: helmWaitMode}, nil
return Release{Chart: chartRaw, Values: values, ReleaseName: info.releaseName, WaitMode: helmWaitMode}, nil
}
func (i *ChartLoader) loadAWSLBControllerValues() map[string]any {

11
cli/internal/helm/loader_test.go
View File

@ -8,7 +8,6 @@ package helm
import (
"bytes"
"encoding/json"
"fmt"
"io/fs"
"os"
@ -29,21 +28,15 @@ import (
"github.com/edgelesssys/constellation/v2/internal/attestation/measurements"
"github.com/edgelesssys/constellation/v2/internal/cloud/cloudprovider"
"github.com/edgelesssys/constellation/v2/internal/config"
"github.com/edgelesssys/constellation/v2/internal/deploy/helm"
)
// TestLoad checks if the serialized format that Load returns correctly preserves the dependencies of the loaded chart.
func TestLoad(t *testing.T) {
func TestLoadReleases(t *testing.T) {
assert := assert.New(t)
require := require.New(t)
config := &config.Config{Provider: config.ProviderConfig{GCP: &config.GCPConfig{}}}
chartLoader := ChartLoader{csp: config.GetProvider()}
release, err := chartLoader.Load(config, true, helm.WaitModeAtomic, []byte("secret"), []byte("salt"))
require.NoError(err)
var helmReleases helm.Releases
err = json.Unmarshal(release, &helmReleases)
helmReleases, err := chartLoader.LoadReleases(config, true, WaitModeAtomic, []byte("secret"), []byte("salt"))
require.NoError(err)
reader := bytes.NewReader(helmReleases.ConstellationServices.Chart)
chart, err := loader.LoadArchive(reader)

22
internal/deploy/helm/helm.go → cli/internal/helm/release.go
View File

@ -25,28 +25,6 @@ type Releases struct {
ConstellationServices Release
}
// MergeMaps returns a new map that is the merger of it's inputs.
// Key collisions are resolved by taking the value of the second argument (map b).
// Taken from: https://github.com/helm/helm/blob/dbc6d8e20fe1d58d50e6ed30f09a04a77e4c68db/pkg/cli/values/options.go#L91-L108.
func MergeMaps(a, b map[string]any) map[string]any {
out := make(map[string]any, len(a))
for k, v := range a {
out[k] = v
}
for k, v := range b {
if v, ok := v.(map[string]any); ok {
if bv, ok := out[k]; ok {
if bv, ok := bv.(map[string]any); ok {
out[k] = MergeMaps(bv, v)
continue
}
}
}
out[k] = v
}
return out
}
// WaitMode specifies the wait mode for a helm release.
type WaitMode string

134
cli/internal/helm/setup.go
View File

@ -1,134 +0,0 @@
/*
Copyright (c) Edgeless Systems GmbH
SPDX-License-Identifier: AGPL-3.0-only
*/
package helm
import (
"context"
"encoding/base64"
"encoding/json"
"fmt"
"github.com/edgelesssys/constellation/v2/cli/internal/terraform"
"github.com/edgelesssys/constellation/v2/internal/cloud/azureshared"
"github.com/edgelesssys/constellation/v2/internal/cloud/cloudprovider"
"github.com/edgelesssys/constellation/v2/internal/cloud/gcpshared"
"github.com/edgelesssys/constellation/v2/internal/constants"
)
// setupMicroserviceVals returns the values for the microservice chart.
func setupMicroserviceVals(ctx context.Context, provider cloudprovider.Provider, measurementSalt []byte, uid, serviceAccURI string) (map[string]any, error) {
tfClient, err := terraform.New(ctx, constants.TerraformWorkingDir)
if err != nil {
return nil, fmt.Errorf("creating Terraform client: %w", err)
}
output, err := tfClient.ShowCluster(ctx, provider)
if err != nil {
return nil, fmt.Errorf("getting Terraform output: %w", err)
}
extraVals := map[string]any{
"join-service": map[string]any{
"measurementSalt": base64.StdEncoding.EncodeToString(measurementSalt),
},
"verification-service": map[string]any{
"loadBalancerIP": output.IP,
},
"konnectivity": map[string]any{
"loadBalancerIP": output.IP,
},
}
switch provider {
case cloudprovider.GCP:
serviceAccountKey, err := gcpshared.ServiceAccountKeyFromURI(serviceAccURI)
if err != nil {
return nil, fmt.Errorf("getting service account key: %w", err)
}
rawKey, err := json.Marshal(serviceAccountKey)
if err != nil {
return nil, fmt.Errorf("marshaling service account key: %w", err)
}
if output.GCP == nil {
return nil, fmt.Errorf("no GCP output from Terraform")
}
extraVals["ccm"] = map[string]any{
"GCP": map[string]any{
"projectID": output.GCP.ProjectID,
"uid": uid,
"secretData": string(rawKey),
"subnetworkPodCIDR": output.GCP.IPCidrPod,
},
}
case cloudprovider.Azure:
if output.Azure == nil {
return nil, fmt.Errorf("no Azure output from Terraform")
}
ccmConfig, err := getCCMConfig(*output.Azure, serviceAccURI)
if err != nil {
return nil, fmt.Errorf("getting Azure CCM config: %w", err)
}
extraVals["ccm"] = map[string]any{
"Azure": map[string]any{
"azureConfig": string(ccmConfig),
},
}
}
return extraVals, nil
}
// setupOperatorVals returns the values for the constellation-operator chart.
func setupOperatorVals(_ context.Context, uid string) map[string]any {
return map[string]any{
"constellation-operator": map[string]any{
"constellationUID": uid,
},
}
}
type cloudConfig struct {
Cloud string `json:"cloud,omitempty"`
TenantID string `json:"tenantId,omitempty"`
SubscriptionID string `json:"subscriptionId,omitempty"`
ResourceGroup string `json:"resourceGroup,omitempty"`
Location string `json:"location,omitempty"`
SubnetName string `json:"subnetName,omitempty"`
SecurityGroupName string `json:"securityGroupName,omitempty"`
SecurityGroupResourceGroup string `json:"securityGroupResourceGroup,omitempty"`
LoadBalancerName string `json:"loadBalancerName,omitempty"`
LoadBalancerSku string `json:"loadBalancerSku,omitempty"`
VNetName string `json:"vnetName,omitempty"`
VNetResourceGroup string `json:"vnetResourceGroup,omitempty"`
CloudProviderBackoff bool `json:"cloudProviderBackoff,omitempty"`
UseInstanceMetadata bool `json:"useInstanceMetadata,omitempty"`
VMType string `json:"vmType,omitempty"`
UseManagedIdentityExtension bool `json:"useManagedIdentityExtension,omitempty"`
UserAssignedIdentityID string `json:"userAssignedIdentityID,omitempty"`
}
// GetCCMConfig returns the configuration needed for the Kubernetes Cloud Controller Manager on Azure.
func getCCMConfig(tfOutput terraform.AzureApplyOutput, serviceAccURI string) ([]byte, error) {
creds, err := azureshared.ApplicationCredentialsFromURI(serviceAccURI)
if err != nil {
return nil, fmt.Errorf("getting service account key: %w", err)
}
useManagedIdentityExtension := creds.PreferredAuthMethod == azureshared.AuthMethodUserAssignedIdentity
config := cloudConfig{
Cloud: "AzurePublicCloud",
TenantID: creds.TenantID,
SubscriptionID: tfOutput.SubscriptionID,
ResourceGroup: tfOutput.ResourceGroup,
LoadBalancerSku: "standard",
SecurityGroupName: tfOutput.NetworkSecurityGroupName,
LoadBalancerName: tfOutput.LoadBalancerName,
UseInstanceMetadata: true,
VMType: "vmss",
Location: creds.Location,
UseManagedIdentityExtension: useManagedIdentityExtension,
UserAssignedIdentityID: tfOutput.UserAssignedIdentity,
}
return json.Marshal(config)
}

51
cli/internal/helm/client.go → cli/internal/helm/upgrade.go
View File

@ -13,20 +13,20 @@ import (
"strings"
"time"
"helm.sh/helm/v3/pkg/action"
"helm.sh/helm/v3/pkg/chart"
"helm.sh/helm/v3/pkg/cli"
"helm.sh/helm/v3/pkg/release"
"github.com/edgelesssys/constellation/v2/cli/internal/clusterid"
"github.com/edgelesssys/constellation/v2/internal/cloud/cloudprovider"
"github.com/edgelesssys/constellation/v2/internal/compatibility"
"github.com/edgelesssys/constellation/v2/internal/config"
"github.com/edgelesssys/constellation/v2/internal/constants"
"github.com/edgelesssys/constellation/v2/internal/deploy/helm"
"github.com/edgelesssys/constellation/v2/internal/file"
"github.com/edgelesssys/constellation/v2/internal/semver"
"github.com/edgelesssys/constellation/v2/internal/versions"
"github.com/spf13/afero"
"helm.sh/helm/v3/pkg/action"
"helm.sh/helm/v3/pkg/chart"
"helm.sh/helm/v3/pkg/cli"
"helm.sh/helm/v3/pkg/release"
apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/runtime/schema"
@ -44,8 +44,8 @@ var ErrConfirmationMissing = errors.New("action requires user confirmation")
var errReleaseNotFound = errors.New("release not found")
// Client handles interaction with helm and the cluster.
type Client struct {
// UpgradeClient handles interaction with helm and the cluster.
type UpgradeClient struct {
config *action.Configuration
kubectl crdClient
fs file.Handler
@ -53,8 +53,8 @@ type Client struct {
log debugLog
}
// NewClient returns a new initializes client for the namespace Client.
func NewClient(client crdClient, kubeConfigPath, helmNamespace string, log debugLog) (*Client, error) {
// NewUpgradeClient returns a new initializes upgrade client for the given namespace.
func NewUpgradeClient(client crdClient, kubeConfigPath, helmNamespace string, log debugLog) (*UpgradeClient, error) {
settings := cli.New()
settings.KubeConfig = kubeConfigPath // constants.AdminConfFilename
@ -74,10 +74,10 @@ func NewClient(client crdClient, kubeConfigPath, helmNamespace string, log debug
return nil, fmt.Errorf("initializing kubectl: %w", err)
}
return &Client{kubectl: client, fs: fileHandler, actions: actions{config: actionConfig}, log: log}, nil
return &UpgradeClient{kubectl: client, fs: fileHandler, actions: actions{config: actionConfig}, log: log}, nil
}
func (c *Client) shouldUpgrade(releaseName string, newVersion semver.Semver, force bool) error {
func (c *UpgradeClient) shouldUpgrade(releaseName string, newVersion semver.Semver, force bool) error {
currentVersion, err := c.currentVersion(releaseName)
if err != nil {
return fmt.Errorf("getting version for %s: %w", releaseName, err)
@ -107,7 +107,7 @@ func (c *Client) shouldUpgrade(releaseName string, newVersion semver.Semver, for
// Upgrade runs a helm-upgrade on all deployments that are managed via Helm.
// If the CLI receives an interrupt signal it will cancel the context.
// Canceling the context will prompt helm to abort and roll back the ongoing upgrade.
func (c *Client) Upgrade(ctx context.Context, config *config.Config, idFile clusterid.File, timeout time.Duration, allowDestructive, force bool, upgradeID string) error {
func (c *UpgradeClient) Upgrade(ctx context.Context, config *config.Config, idFile clusterid.File, timeout time.Duration, allowDestructive, force bool, upgradeID string) error {
upgradeErrs := []error{}
upgradeReleases := []*chart.Chart{}
newReleases := []*chart.Chart{}
@ -206,7 +206,7 @@ func getManagedCharts(config *config.Config) []chartInfo {
}
// Versions queries the cluster for running versions and returns a map of releaseName -> version.
func (c *Client) Versions() (ServiceVersions, error) {
func (c *UpgradeClient) Versions() (ServiceVersions, error) {
ciliumVersion, err := c.currentVersion(ciliumInfo.releaseName)
if err != nil {
return ServiceVersions{}, fmt.Errorf("getting %s version: %w", ciliumInfo.releaseName, err)
@ -246,7 +246,7 @@ func (c *Client) Versions() (ServiceVersions, error) {
}
// currentVersion returns the version of the currently installed helm release.
func (c *Client) currentVersion(release string) (semver.Semver, error) {
func (c *UpgradeClient) currentVersion(release string) (semver.Semver, error) {
rel, err := c.actions.listAction(release)
if err != nil {
return semver.Semver{}, err
@ -266,7 +266,7 @@ func (c *Client) currentVersion(release string) (semver.Semver, error) {
return semver.New(rel[0].Chart.Metadata.Version)
}
func (c *Client) csiVersions() (map[string]semver.Semver, error) {
func (c *UpgradeClient) csiVersions() (map[string]semver.Semver, error) {
packedChartRelease, err := c.actions.listAction(csiInfo.releaseName)
if err != nil {
return nil, fmt.Errorf("listing %s: %w", csiInfo.releaseName, err)
@ -299,7 +299,7 @@ func (c *Client) csiVersions() (map[string]semver.Semver, error) {
}
// installNewRelease installs a previously not installed release on the cluster.
func (c *Client) installNewRelease(
func (c *UpgradeClient) installNewRelease(
ctx context.Context, timeout time.Duration, conf *config.Config, idFile clusterid.File, chart *chart.Chart,
) error {
releaseName, values, err := c.loadUpgradeValues(ctx, conf, idFile, chart)
@ -310,7 +310,7 @@ func (c *Client) installNewRelease(
}
// upgradeRelease upgrades a release running on the cluster.
func (c *Client) upgradeRelease(
func (c *UpgradeClient) upgradeRelease(
ctx context.Context, timeout time.Duration, conf *config.Config, idFile clusterid.File, chart *chart.Chart,
) error {
releaseName, values, err := c.loadUpgradeValues(ctx, conf, idFile, chart)
@ -327,7 +327,7 @@ func (c *Client) upgradeRelease(
}
// loadUpgradeValues loads values for a chart required for running an upgrade.
func (c *Client) loadUpgradeValues(ctx context.Context, conf *config.Config, idFile clusterid.File, chart *chart.Chart,
func (c *UpgradeClient) loadUpgradeValues(ctx context.Context, conf *config.Config, idFile clusterid.File, chart *chart.Chart,
) (string, map[string]any, error) {
// We need to load all values that can be statically loaded before merging them with the cluster
// values. Otherwise the templates are not rendered correctly.
@ -385,7 +385,7 @@ func (c *Client) loadUpgradeValues(ctx context.Context, conf *config.Config, idF
// applyMigrations checks the from version and applies the necessary migrations.
// The function assumes the caller has verified that our version drift restriction is not violated,
// Currently, this is done during config validation.
func (c *Client) applyMigrations(ctx context.Context, releaseName string, values map[string]any, conf *config.Config) error {
func (c *UpgradeClient) applyMigrations(ctx context.Context, releaseName string, values map[string]any, conf *config.Config) error {
current, err := c.currentVersion(releaseName)
if err != nil {
return fmt.Errorf("getting %s version: %w", releaseName, err)
@ -413,7 +413,7 @@ func migrateFrom2_8(_ context.Context, _ map[string]any, _ *config.Config, _ crd
// and merging the fetched values with the locally found values.
// This is done to ensure that new values (from upgrades of the local files) end up in the cluster.
// reuse-values does not ensure this.
func (c *Client) mergeClusterValues(localValues map[string]any, releaseName string) (map[string]any, error) {
func (c *UpgradeClient) mergeClusterValues(localValues map[string]any, releaseName string) (map[string]any, error) {
// Ensure installCRDs is set for cert-manager chart.
if releaseName == certManagerInfo.releaseName {
localValues["installCRDs"] = true
@ -423,11 +423,11 @@ func (c *Client) mergeClusterValues(localValues map[string]any, releaseName stri
return nil, fmt.Errorf("getting values for %s: %w", releaseName, err)
}
return helm.MergeMaps(clusterValues, localValues), nil
return mergeMaps(clusterValues, localValues), nil
}
// GetValues queries the cluster for the values of the given release.
func (c *Client) GetValues(release string) (map[string]any, error) {
func (c *UpgradeClient) GetValues(release string) (map[string]any, error) {
client := action.NewGetValues(c.config)
// Version corresponds to the releases revision. Specifying a Version <= 0 yields the latest release.
client.Version = 0
@ -441,7 +441,7 @@ func (c *Client) GetValues(release string) (map[string]any, error) {
// updateCRDs walks through the dependencies of the given chart and applies
// the files in the dependencie's 'crds' folder.
// This function is NOT recursive!
func (c *Client) updateCRDs(ctx context.Context, chart *chart.Chart) error {
func (c *UpgradeClient) updateCRDs(ctx context.Context, chart *chart.Chart) error {
for _, dep := range chart.Dependencies() {
for _, crdFile := range dep.Files {
if strings.HasPrefix(crdFile.Name, "crds/") {
@ -456,11 +456,6 @@ func (c *Client) updateCRDs(ctx context.Context, chart *chart.Chart) error {
return nil
}
type debugLog interface {
Debugf(format string, args ...any)
Sync()
}
type crdClient interface {
Initialize(kubeconfig []byte) error
ApplyCRD(ctx context.Context, rawCRD []byte) error

4
cli/internal/helm/client_test.go → cli/internal/helm/upgrade_test.go
View File

@ -46,7 +46,7 @@ func TestShouldUpgrade(t *testing.T) {
assert := assert.New(t)
require := require.New(t)
client := Client{kubectl: nil, actions: &stubActionWrapper{version: tc.version}, log: logger.NewTest(t)}
client := UpgradeClient{kubectl: nil, actions: &stubActionWrapper{version: tc.version}, log: logger.NewTest(t)}
chart, err := loadChartsDir(helmFS, certManagerInfo.path)
require.NoError(err)
@ -77,7 +77,7 @@ func TestUpgradeRelease(t *testing.T) {
assert := assert.New(t)
require := require.New(t)
client := Client{kubectl: nil, actions: &stubActionWrapper{version: tc.version}, log: logger.NewTest(t)}
client := UpgradeClient{kubectl: nil, actions: &stubActionWrapper{version: tc.version}, log: logger.NewTest(t)}
chart, err := loadChartsDir(helmFS, certManagerInfo.path)
require.NoError(err)

2
cli/internal/kubernetes/upgrade.go
View File

@ -136,7 +136,7 @@ func NewUpgrader(ctx context.Context, outWriter io.Writer, fileHandler file.Hand
}
u.dynamicInterface = &NodeVersionClient{client: unstructuredClient}
helmClient, err := helm.NewClient(kubectl.New(), constants.AdminConfFilename, constants.HelmNamespace, log)
helmClient, err := helm.NewUpgradeClient(kubectl.New(), constants.AdminConfFilename, constants.HelmNamespace, log)
if err != nil {
return nil, fmt.Errorf("setting up helm client: %w", err)
}

28
internal/deploy/helm/BUILD.bazel
View File

@ -1,28 +0,0 @@
load("@io_bazel_rules_go//go:def.bzl", "go_library")
load("//bazel/go:go_test.bzl", "go_test")
go_library(
name = "helm",
srcs = [
"helm.go",
"install.go",
],
importpath = "github.com/edgelesssys/constellation/v2/internal/deploy/helm",
visibility = ["//:__subpackages__"],
deps = [
"//internal/constants",
"//internal/retry",
"@io_k8s_apimachinery//pkg/util/wait",
"@sh_helm_helm_v3//pkg/action",
"@sh_helm_helm_v3//pkg/chart",
"@sh_helm_helm_v3//pkg/chart/loader",
"@sh_helm_helm_v3//pkg/cli",
],
)
go_test(
name = "helm_test",
srcs = ["helm_test.go"],
embed = [":helm"],
deps = ["@com_github_stretchr_testify//assert"],
)

2
internal/kubernetes/kubectl/BUILD.bazel
View File

@ -12,9 +12,11 @@ go_library(
"@io_k8s_apiextensions_apiserver//pkg/client/clientset/clientset/typed/apiextensions/v1:apiextensions",
"@io_k8s_apimachinery//pkg/apis/meta/v1:meta",
"@io_k8s_apimachinery//pkg/apis/meta/v1/unstructured",
"@io_k8s_apimachinery//pkg/labels",
"@io_k8s_apimachinery//pkg/runtime",
"@io_k8s_apimachinery//pkg/runtime/schema",
"@io_k8s_apimachinery//pkg/runtime/serializer",
"@io_k8s_apimachinery//pkg/types",
"@io_k8s_client_go//dynamic",
"@io_k8s_client_go//kubernetes",
"@io_k8s_client_go//scale/scheme",

18
internal/kubernetes/kubectl/kubectl.go
View File

@ -19,9 +19,11 @@ import (
apiextensionsclientv1 "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset/typed/apiextensions/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/labels"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apimachinery/pkg/runtime/serializer"
"k8s.io/apimachinery/pkg/types"
"k8s.io/client-go/dynamic"
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/scale/scheme"
@ -130,6 +132,22 @@ func (k *Kubectl) AnnotateNode(ctx context.Context, nodeName, annotationKey, ann
})
}
// PatchFirstNodePodCIDR patches the firstNodePodCIDR of the first control-plane node for Cilium.
func (k *Kubectl) PatchFirstNodePodCIDR(ctx context.Context, firstNodePodCIDR string) error {
selector := labels.Set{"node-role.kubernetes.io/control-plane": ""}.AsSelector()
controlPlaneList, err := k.CoreV1().Nodes().List(ctx, metav1.ListOptions{LabelSelector: selector.String()})
if err != nil {
return err
}
if len(controlPlaneList.Items) != 1 {
return fmt.Errorf("expected 1 control-plane node, got %d", len(controlPlaneList.Items))
}
nodeName := controlPlaneList.Items[0].Name
// Update the node's spec
_, err = k.CoreV1().Nodes().Patch(context.Background(), nodeName, types.MergePatchType, []byte(fmt.Sprintf(`{"spec":{"podCIDR":"%s"}}`, firstNodePodCIDR)), metav1.PatchOptions{})
return err
}
// ListAllNamespaces returns all namespaces in the cluster.
func (k *Kubectl) ListAllNamespaces(ctx context.Context) (*corev1.NamespaceList, error) {
return k.CoreV1().Namespaces().List(ctx, metav1.ListOptions{})