diff --git a/bootstrapper/cmd/bootstrapper/BUILD.bazel b/bootstrapper/cmd/bootstrapper/BUILD.bazel index 4bde699c4..52eab989d 100644 --- a/bootstrapper/cmd/bootstrapper/BUILD.bazel +++ b/bootstrapper/cmd/bootstrapper/BUILD.bazel @@ -35,7 +35,6 @@ go_library( "//internal/cloud/openstack", "//internal/cloud/qemu", "//internal/constants", - "//internal/deploy/helm", "//internal/file", "//internal/grpc/dialer", "//internal/kubernetes/kubectl", diff --git a/bootstrapper/cmd/bootstrapper/main.go b/bootstrapper/cmd/bootstrapper/main.go index 9c41c9176..950ef0d75 100644 --- a/bootstrapper/cmd/bootstrapper/main.go +++ b/bootstrapper/cmd/bootstrapper/main.go @@ -32,7 +32,6 @@ import ( openstackcloud "github.com/edgelesssys/constellation/v2/internal/cloud/openstack" qemucloud "github.com/edgelesssys/constellation/v2/internal/cloud/qemu" "github.com/edgelesssys/constellation/v2/internal/constants" - "github.com/edgelesssys/constellation/v2/internal/deploy/helm" "github.com/edgelesssys/constellation/v2/internal/file" "github.com/edgelesssys/constellation/v2/internal/kubernetes/kubectl" "github.com/edgelesssys/constellation/v2/internal/logger" @@ -67,11 +66,6 @@ func main() { var openDevice vtpm.TPMOpenFunc var fs afero.Fs - helmClient, err := helm.NewInstaller(constants.ControlPlaneAdminConfFilename, log) - if err != nil { - log.With(zap.Error(err)).Fatalf("Helm client could not be initialized") - } - attestVariant, err := variant.FromString(os.Getenv(constants.AttestationVariant)) if err != nil { log.With(zap.Error(err)).Fatalf("Failed to parse attestation variant") @@ -96,7 +90,7 @@ func main() { clusterInitJoiner = kubernetes.New( "aws", k8sapi.NewKubernetesUtil(), &k8sapi.KubdeadmConfiguration{}, kubectl.New(), - metadata, helmClient, &kubewaiter.CloudKubeAPIWaiter{}, + metadata, &kubewaiter.CloudKubeAPIWaiter{}, ) openDevice = vtpm.OpenVTPM fs = afero.NewOsFs() @@ -116,7 +110,7 @@ func main() { metadataAPI = metadata clusterInitJoiner = kubernetes.New( "gcp", k8sapi.NewKubernetesUtil(), &k8sapi.KubdeadmConfiguration{}, kubectl.New(), - metadata, helmClient, &kubewaiter.CloudKubeAPIWaiter{}, + metadata, &kubewaiter.CloudKubeAPIWaiter{}, ) openDevice = vtpm.OpenVTPM fs = afero.NewOsFs() @@ -134,7 +128,7 @@ func main() { metadataAPI = metadata clusterInitJoiner = kubernetes.New( "azure", k8sapi.NewKubernetesUtil(), &k8sapi.KubdeadmConfiguration{}, kubectl.New(), - metadata, helmClient, &kubewaiter.CloudKubeAPIWaiter{}, + metadata, &kubewaiter.CloudKubeAPIWaiter{}, ) openDevice = vtpm.OpenVTPM @@ -145,7 +139,7 @@ func main() { metadata := qemucloud.New() clusterInitJoiner = kubernetes.New( "qemu", k8sapi.NewKubernetesUtil(), &k8sapi.KubdeadmConfiguration{}, kubectl.New(), - metadata, helmClient, &kubewaiter.CloudKubeAPIWaiter{}, + metadata, &kubewaiter.CloudKubeAPIWaiter{}, ) metadataAPI = metadata @@ -168,7 +162,7 @@ func main() { } clusterInitJoiner = kubernetes.New( "openstack", k8sapi.NewKubernetesUtil(), &k8sapi.KubdeadmConfiguration{}, kubectl.New(), - metadata, helmClient, &kubewaiter.CloudKubeAPIWaiter{}, + metadata, &kubewaiter.CloudKubeAPIWaiter{}, ) metadataAPI = metadata openDevice = vtpm.OpenVTPM diff --git a/bootstrapper/cmd/bootstrapper/test.go b/bootstrapper/cmd/bootstrapper/test.go index ee7d7dea0..bdf03bbd3 100644 --- a/bootstrapper/cmd/bootstrapper/test.go +++ b/bootstrapper/cmd/bootstrapper/test.go @@ -22,7 +22,7 @@ type clusterFake struct{} // InitCluster fakes bootstrapping a new cluster with the current node being the master, returning the arguments required to join the cluster. func (c *clusterFake) InitCluster( context.Context, string, string, - []byte, bool, components.Components, []string, *logger.Logger, + bool, components.Components, []string, *logger.Logger, ) ([]byte, error) { return []byte{}, nil } diff --git a/bootstrapper/initproto/init.pb.go b/bootstrapper/initproto/init.pb.go index 31695ca5d..80f094aac 100644 --- a/bootstrapper/initproto/init.pb.go +++ b/bootstrapper/initproto/init.pb.go @@ -33,7 +33,6 @@ type InitRequest struct { StorageUri string `protobuf:"bytes,4,opt,name=storage_uri,json=storageUri,proto3" json:"storage_uri,omitempty"` CloudServiceAccountUri string `protobuf:"bytes,7,opt,name=cloud_service_account_uri,json=cloudServiceAccountUri,proto3" json:"cloud_service_account_uri,omitempty"` KubernetesVersion string `protobuf:"bytes,8,opt,name=kubernetes_version,json=kubernetesVersion,proto3" json:"kubernetes_version,omitempty"` - HelmDeployments []byte `protobuf:"bytes,11,opt,name=helm_deployments,json=helmDeployments,proto3" json:"helm_deployments,omitempty"` ConformanceMode bool `protobuf:"varint,14,opt,name=conformance_mode,json=conformanceMode,proto3" json:"conformance_mode,omitempty"` KubernetesComponents []*KubernetesComponent `protobuf:"bytes,15,rep,name=kubernetes_components,json=kubernetesComponents,proto3" json:"kubernetes_components,omitempty"` InitSecret []byte `protobuf:"bytes,16,opt,name=init_secret,json=initSecret,proto3" json:"init_secret,omitempty"` @@ -101,13 +100,6 @@ func (x *InitRequest) GetKubernetesVersion() string { return "" } -func (x *InitRequest) GetHelmDeployments() []byte { - if x != nil { - return x.HelmDeployments - } - return nil -} - func (x *InitRequest) GetConformanceMode() bool { if x != nil { return x.ConformanceMode @@ -471,7 +463,7 @@ var File_bootstrapper_initproto_init_proto protoreflect.FileDescriptor var file_bootstrapper_initproto_init_proto_rawDesc = []byte{ 0x0a, 0x21, 0x62, 0x6f, 0x6f, 0x74, 0x73, 0x74, 0x72, 0x61, 0x70, 0x70, 0x65, 0x72, 0x2f, 0x69, 0x6e, 0x69, 0x74, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x69, 0x6e, 0x69, 0x74, 0x2e, 0x70, 0x72, - 0x6f, 0x74, 0x6f, 0x12, 0x04, 0x69, 0x6e, 0x69, 0x74, 0x22, 0xcb, 0x03, 0x0a, 0x0b, 0x49, 0x6e, + 0x6f, 0x74, 0x6f, 0x12, 0x04, 0x69, 0x6e, 0x69, 0x74, 0x22, 0xa0, 0x03, 0x0a, 0x0b, 0x49, 0x6e, 0x69, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x17, 0x0a, 0x07, 0x6b, 0x6d, 0x73, 0x5f, 0x75, 0x72, 0x69, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x6b, 0x6d, 0x73, 0x55, 0x72, 0x69, 0x12, 0x1f, 0x0a, 0x0b, 0x73, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x5f, 0x75, 0x72, @@ -483,64 +475,61 @@ var file_bootstrapper_initproto_init_proto_rawDesc = []byte{ 0x0a, 0x12, 0x6b, 0x75, 0x62, 0x65, 0x72, 0x6e, 0x65, 0x74, 0x65, 0x73, 0x5f, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x08, 0x20, 0x01, 0x28, 0x09, 0x52, 0x11, 0x6b, 0x75, 0x62, 0x65, 0x72, 0x6e, 0x65, 0x74, 0x65, 0x73, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x29, 0x0a, - 0x10, 0x68, 0x65, 0x6c, 0x6d, 0x5f, 0x64, 0x65, 0x70, 0x6c, 0x6f, 0x79, 0x6d, 0x65, 0x6e, 0x74, - 0x73, 0x18, 0x0b, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0f, 0x68, 0x65, 0x6c, 0x6d, 0x44, 0x65, 0x70, - 0x6c, 0x6f, 0x79, 0x6d, 0x65, 0x6e, 0x74, 0x73, 0x12, 0x29, 0x0a, 0x10, 0x63, 0x6f, 0x6e, 0x66, - 0x6f, 0x72, 0x6d, 0x61, 0x6e, 0x63, 0x65, 0x5f, 0x6d, 0x6f, 0x64, 0x65, 0x18, 0x0e, 0x20, 0x01, - 0x28, 0x08, 0x52, 0x0f, 0x63, 0x6f, 0x6e, 0x66, 0x6f, 0x72, 0x6d, 0x61, 0x6e, 0x63, 0x65, 0x4d, - 0x6f, 0x64, 0x65, 0x12, 0x4e, 0x0a, 0x15, 0x6b, 0x75, 0x62, 0x65, 0x72, 0x6e, 0x65, 0x74, 0x65, - 0x73, 0x5f, 0x63, 0x6f, 0x6d, 0x70, 0x6f, 0x6e, 0x65, 0x6e, 0x74, 0x73, 0x18, 0x0f, 0x20, 0x03, - 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x69, 0x6e, 0x69, 0x74, 0x2e, 0x4b, 0x75, 0x62, 0x65, 0x72, 0x6e, - 0x65, 0x74, 0x65, 0x73, 0x43, 0x6f, 0x6d, 0x70, 0x6f, 0x6e, 0x65, 0x6e, 0x74, 0x52, 0x14, 0x6b, + 0x10, 0x63, 0x6f, 0x6e, 0x66, 0x6f, 0x72, 0x6d, 0x61, 0x6e, 0x63, 0x65, 0x5f, 0x6d, 0x6f, 0x64, + 0x65, 0x18, 0x0e, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0f, 0x63, 0x6f, 0x6e, 0x66, 0x6f, 0x72, 0x6d, + 0x61, 0x6e, 0x63, 0x65, 0x4d, 0x6f, 0x64, 0x65, 0x12, 0x4e, 0x0a, 0x15, 0x6b, 0x75, 0x62, 0x65, + 0x72, 0x6e, 0x65, 0x74, 0x65, 0x73, 0x5f, 0x63, 0x6f, 0x6d, 0x70, 0x6f, 0x6e, 0x65, 0x6e, 0x74, + 0x73, 0x18, 0x0f, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x69, 0x6e, 0x69, 0x74, 0x2e, 0x4b, 0x75, 0x62, 0x65, 0x72, 0x6e, 0x65, 0x74, 0x65, 0x73, 0x43, 0x6f, 0x6d, 0x70, 0x6f, 0x6e, 0x65, - 0x6e, 0x74, 0x73, 0x12, 0x1f, 0x0a, 0x0b, 0x69, 0x6e, 0x69, 0x74, 0x5f, 0x73, 0x65, 0x63, 0x72, - 0x65, 0x74, 0x18, 0x10, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0a, 0x69, 0x6e, 0x69, 0x74, 0x53, 0x65, - 0x63, 0x72, 0x65, 0x74, 0x12, 0x21, 0x0a, 0x0c, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x5f, - 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x11, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x63, 0x6c, 0x75, 0x73, - 0x74, 0x65, 0x72, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x2e, 0x0a, 0x13, 0x61, 0x70, 0x69, 0x73, 0x65, - 0x72, 0x76, 0x65, 0x72, 0x5f, 0x63, 0x65, 0x72, 0x74, 0x5f, 0x73, 0x61, 0x6e, 0x73, 0x18, 0x12, - 0x20, 0x03, 0x28, 0x09, 0x52, 0x11, 0x61, 0x70, 0x69, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x43, - 0x65, 0x72, 0x74, 0x53, 0x61, 0x6e, 0x73, 0x22, 0xc1, 0x01, 0x0a, 0x0c, 0x49, 0x6e, 0x69, 0x74, - 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x3e, 0x0a, 0x0c, 0x69, 0x6e, 0x69, 0x74, - 0x5f, 0x73, 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x19, - 0x2e, 0x69, 0x6e, 0x69, 0x74, 0x2e, 0x49, 0x6e, 0x69, 0x74, 0x53, 0x75, 0x63, 0x63, 0x65, 0x73, - 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x48, 0x00, 0x52, 0x0b, 0x69, 0x6e, 0x69, - 0x74, 0x53, 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, 0x12, 0x3e, 0x0a, 0x0c, 0x69, 0x6e, 0x69, 0x74, - 0x5f, 0x66, 0x61, 0x69, 0x6c, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x19, - 0x2e, 0x69, 0x6e, 0x69, 0x74, 0x2e, 0x49, 0x6e, 0x69, 0x74, 0x46, 0x61, 0x69, 0x6c, 0x75, 0x72, - 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x48, 0x00, 0x52, 0x0b, 0x69, 0x6e, 0x69, - 0x74, 0x46, 0x61, 0x69, 0x6c, 0x75, 0x72, 0x65, 0x12, 0x29, 0x0a, 0x03, 0x6c, 0x6f, 0x67, 0x18, - 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x15, 0x2e, 0x69, 0x6e, 0x69, 0x74, 0x2e, 0x4c, 0x6f, 0x67, - 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x54, 0x79, 0x70, 0x65, 0x48, 0x00, 0x52, 0x03, - 0x6c, 0x6f, 0x67, 0x42, 0x06, 0x0a, 0x04, 0x6b, 0x69, 0x6e, 0x64, 0x22, 0x6f, 0x0a, 0x13, 0x49, - 0x6e, 0x69, 0x74, 0x53, 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, - 0x73, 0x65, 0x12, 0x1e, 0x0a, 0x0a, 0x6b, 0x75, 0x62, 0x65, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, - 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0a, 0x6b, 0x75, 0x62, 0x65, 0x63, 0x6f, 0x6e, 0x66, - 0x69, 0x67, 0x12, 0x19, 0x0a, 0x08, 0x6f, 0x77, 0x6e, 0x65, 0x72, 0x5f, 0x69, 0x64, 0x18, 0x02, - 0x20, 0x01, 0x28, 0x0c, 0x52, 0x07, 0x6f, 0x77, 0x6e, 0x65, 0x72, 0x49, 0x64, 0x12, 0x1d, 0x0a, - 0x0a, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x69, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, - 0x0c, 0x52, 0x09, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x49, 0x64, 0x22, 0x2b, 0x0a, 0x13, - 0x49, 0x6e, 0x69, 0x74, 0x46, 0x61, 0x69, 0x6c, 0x75, 0x72, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, - 0x6e, 0x73, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x18, 0x01, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x22, 0x23, 0x0a, 0x0f, 0x4c, 0x6f, 0x67, - 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x54, 0x79, 0x70, 0x65, 0x12, 0x10, 0x0a, 0x03, - 0x6c, 0x6f, 0x67, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x03, 0x6c, 0x6f, 0x67, 0x22, 0x78, - 0x0a, 0x13, 0x4b, 0x75, 0x62, 0x65, 0x72, 0x6e, 0x65, 0x74, 0x65, 0x73, 0x43, 0x6f, 0x6d, 0x70, - 0x6f, 0x6e, 0x65, 0x6e, 0x74, 0x12, 0x10, 0x0a, 0x03, 0x75, 0x72, 0x6c, 0x18, 0x01, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x03, 0x75, 0x72, 0x6c, 0x12, 0x12, 0x0a, 0x04, 0x68, 0x61, 0x73, 0x68, 0x18, - 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x68, 0x61, 0x73, 0x68, 0x12, 0x21, 0x0a, 0x0c, 0x69, - 0x6e, 0x73, 0x74, 0x61, 0x6c, 0x6c, 0x5f, 0x70, 0x61, 0x74, 0x68, 0x18, 0x03, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x0b, 0x69, 0x6e, 0x73, 0x74, 0x61, 0x6c, 0x6c, 0x50, 0x61, 0x74, 0x68, 0x12, 0x18, - 0x0a, 0x07, 0x65, 0x78, 0x74, 0x72, 0x61, 0x63, 0x74, 0x18, 0x04, 0x20, 0x01, 0x28, 0x08, 0x52, - 0x07, 0x65, 0x78, 0x74, 0x72, 0x61, 0x63, 0x74, 0x32, 0x36, 0x0a, 0x03, 0x41, 0x50, 0x49, 0x12, - 0x2f, 0x0a, 0x04, 0x49, 0x6e, 0x69, 0x74, 0x12, 0x11, 0x2e, 0x69, 0x6e, 0x69, 0x74, 0x2e, 0x49, - 0x6e, 0x69, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x12, 0x2e, 0x69, 0x6e, 0x69, - 0x74, 0x2e, 0x49, 0x6e, 0x69, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x30, 0x01, - 0x42, 0x40, 0x5a, 0x3e, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x65, - 0x64, 0x67, 0x65, 0x6c, 0x65, 0x73, 0x73, 0x73, 0x79, 0x73, 0x2f, 0x63, 0x6f, 0x6e, 0x73, 0x74, - 0x65, 0x6c, 0x6c, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x2f, 0x76, 0x32, 0x2f, 0x62, 0x6f, 0x6f, 0x74, - 0x73, 0x74, 0x72, 0x61, 0x70, 0x70, 0x65, 0x72, 0x2f, 0x69, 0x6e, 0x69, 0x74, 0x70, 0x72, 0x6f, - 0x74, 0x6f, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x6e, 0x74, 0x52, 0x14, 0x6b, 0x75, 0x62, 0x65, 0x72, 0x6e, 0x65, 0x74, 0x65, 0x73, 0x43, 0x6f, + 0x6d, 0x70, 0x6f, 0x6e, 0x65, 0x6e, 0x74, 0x73, 0x12, 0x1f, 0x0a, 0x0b, 0x69, 0x6e, 0x69, 0x74, + 0x5f, 0x73, 0x65, 0x63, 0x72, 0x65, 0x74, 0x18, 0x10, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0a, 0x69, + 0x6e, 0x69, 0x74, 0x53, 0x65, 0x63, 0x72, 0x65, 0x74, 0x12, 0x21, 0x0a, 0x0c, 0x63, 0x6c, 0x75, + 0x73, 0x74, 0x65, 0x72, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x11, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x0b, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x2e, 0x0a, 0x13, + 0x61, 0x70, 0x69, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x5f, 0x63, 0x65, 0x72, 0x74, 0x5f, 0x73, + 0x61, 0x6e, 0x73, 0x18, 0x12, 0x20, 0x03, 0x28, 0x09, 0x52, 0x11, 0x61, 0x70, 0x69, 0x73, 0x65, + 0x72, 0x76, 0x65, 0x72, 0x43, 0x65, 0x72, 0x74, 0x53, 0x61, 0x6e, 0x73, 0x22, 0xc1, 0x01, 0x0a, + 0x0c, 0x49, 0x6e, 0x69, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x3e, 0x0a, + 0x0c, 0x69, 0x6e, 0x69, 0x74, 0x5f, 0x73, 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x69, 0x6e, 0x69, 0x74, 0x2e, 0x49, 0x6e, 0x69, 0x74, 0x53, + 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x48, 0x00, + 0x52, 0x0b, 0x69, 0x6e, 0x69, 0x74, 0x53, 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, 0x12, 0x3e, 0x0a, + 0x0c, 0x69, 0x6e, 0x69, 0x74, 0x5f, 0x66, 0x61, 0x69, 0x6c, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, + 0x01, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x69, 0x6e, 0x69, 0x74, 0x2e, 0x49, 0x6e, 0x69, 0x74, 0x46, + 0x61, 0x69, 0x6c, 0x75, 0x72, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x48, 0x00, + 0x52, 0x0b, 0x69, 0x6e, 0x69, 0x74, 0x46, 0x61, 0x69, 0x6c, 0x75, 0x72, 0x65, 0x12, 0x29, 0x0a, + 0x03, 0x6c, 0x6f, 0x67, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x15, 0x2e, 0x69, 0x6e, 0x69, + 0x74, 0x2e, 0x4c, 0x6f, 0x67, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x54, 0x79, 0x70, + 0x65, 0x48, 0x00, 0x52, 0x03, 0x6c, 0x6f, 0x67, 0x42, 0x06, 0x0a, 0x04, 0x6b, 0x69, 0x6e, 0x64, + 0x22, 0x6f, 0x0a, 0x13, 0x49, 0x6e, 0x69, 0x74, 0x53, 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, 0x52, + 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x1e, 0x0a, 0x0a, 0x6b, 0x75, 0x62, 0x65, 0x63, + 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0a, 0x6b, 0x75, 0x62, + 0x65, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x19, 0x0a, 0x08, 0x6f, 0x77, 0x6e, 0x65, 0x72, + 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x07, 0x6f, 0x77, 0x6e, 0x65, 0x72, + 0x49, 0x64, 0x12, 0x1d, 0x0a, 0x0a, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x69, 0x64, + 0x18, 0x03, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x09, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x49, + 0x64, 0x22, 0x2b, 0x0a, 0x13, 0x49, 0x6e, 0x69, 0x74, 0x46, 0x61, 0x69, 0x6c, 0x75, 0x72, 0x65, + 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x65, 0x72, 0x72, 0x6f, + 0x72, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x22, 0x23, + 0x0a, 0x0f, 0x4c, 0x6f, 0x67, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x54, 0x79, 0x70, + 0x65, 0x12, 0x10, 0x0a, 0x03, 0x6c, 0x6f, 0x67, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x03, + 0x6c, 0x6f, 0x67, 0x22, 0x78, 0x0a, 0x13, 0x4b, 0x75, 0x62, 0x65, 0x72, 0x6e, 0x65, 0x74, 0x65, + 0x73, 0x43, 0x6f, 0x6d, 0x70, 0x6f, 0x6e, 0x65, 0x6e, 0x74, 0x12, 0x10, 0x0a, 0x03, 0x75, 0x72, + 0x6c, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x75, 0x72, 0x6c, 0x12, 0x12, 0x0a, 0x04, + 0x68, 0x61, 0x73, 0x68, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x68, 0x61, 0x73, 0x68, + 0x12, 0x21, 0x0a, 0x0c, 0x69, 0x6e, 0x73, 0x74, 0x61, 0x6c, 0x6c, 0x5f, 0x70, 0x61, 0x74, 0x68, + 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x69, 0x6e, 0x73, 0x74, 0x61, 0x6c, 0x6c, 0x50, + 0x61, 0x74, 0x68, 0x12, 0x18, 0x0a, 0x07, 0x65, 0x78, 0x74, 0x72, 0x61, 0x63, 0x74, 0x18, 0x04, + 0x20, 0x01, 0x28, 0x08, 0x52, 0x07, 0x65, 0x78, 0x74, 0x72, 0x61, 0x63, 0x74, 0x32, 0x36, 0x0a, + 0x03, 0x41, 0x50, 0x49, 0x12, 0x2f, 0x0a, 0x04, 0x49, 0x6e, 0x69, 0x74, 0x12, 0x11, 0x2e, 0x69, + 0x6e, 0x69, 0x74, 0x2e, 0x49, 0x6e, 0x69, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, + 0x12, 0x2e, 0x69, 0x6e, 0x69, 0x74, 0x2e, 0x49, 0x6e, 0x69, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, + 0x6e, 0x73, 0x65, 0x30, 0x01, 0x42, 0x40, 0x5a, 0x3e, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, + 0x63, 0x6f, 0x6d, 0x2f, 0x65, 0x64, 0x67, 0x65, 0x6c, 0x65, 0x73, 0x73, 0x73, 0x79, 0x73, 0x2f, + 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x65, 0x6c, 0x6c, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x2f, 0x76, 0x32, + 0x2f, 0x62, 0x6f, 0x6f, 0x74, 0x73, 0x74, 0x72, 0x61, 0x70, 0x70, 0x65, 0x72, 0x2f, 0x69, 0x6e, + 0x69, 0x74, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( diff --git a/bootstrapper/initproto/init.proto b/bootstrapper/initproto/init.proto index 60ddc3fd3..bbea84717 100644 --- a/bootstrapper/initproto/init.proto +++ b/bootstrapper/initproto/init.proto @@ -19,7 +19,7 @@ message InitRequest { string kubernetes_version = 8; // repeated SSHUserKey ssh_user_keys = 9; removed // bytes salt = 10; removed - bytes helm_deployments = 11; + // bytes helm_deployments = 11; removed // repeated uint32 enforced_pcrs = 12; removed // bool enforce_idkeydigest = 13; removed bool conformance_mode = 14; diff --git a/bootstrapper/internal/initserver/initserver.go b/bootstrapper/internal/initserver/initserver.go index cd822890e..1a5b3ee50 100644 --- a/bootstrapper/internal/initserver/initserver.go +++ b/bootstrapper/internal/initserver/initserver.go @@ -215,7 +215,6 @@ func (s *Server) Init(req *initproto.InitRequest, stream initproto.API_InitServe kubeconfig, err := s.initializer.InitCluster(stream.Context(), req.KubernetesVersion, clusterName, - req.HelmDeployments, req.ConformanceMode, components.NewComponentsFromInitProto(req.KubernetesComponents), req.ApiserverCertSans, @@ -342,7 +341,6 @@ type ClusterInitializer interface { ctx context.Context, k8sVersion string, clusterName string, - helmDeployments []byte, conformanceMode bool, kubernetesComponents components.Components, apiServerCertSANs []string, diff --git a/bootstrapper/internal/initserver/initserver_test.go b/bootstrapper/internal/initserver/initserver_test.go index a2505becd..048927e92 100644 --- a/bootstrapper/internal/initserver/initserver_test.go +++ b/bootstrapper/internal/initserver/initserver_test.go @@ -407,7 +407,7 @@ type stubClusterInitializer struct { func (i *stubClusterInitializer) InitCluster( context.Context, string, string, - []byte, bool, components.Components, []string, *logger.Logger, + bool, components.Components, []string, *logger.Logger, ) ([]byte, error) { return i.initClusterKubeconfig, i.initClusterErr } diff --git a/bootstrapper/internal/kubernetes/BUILD.bazel b/bootstrapper/internal/kubernetes/BUILD.bazel index dcddfe228..c2255148a 100644 --- a/bootstrapper/internal/kubernetes/BUILD.bazel +++ b/bootstrapper/internal/kubernetes/BUILD.bazel @@ -16,7 +16,6 @@ go_library( "//internal/cloud/cloudprovider", "//internal/cloud/metadata", "//internal/constants", - "//internal/deploy/helm", "//internal/kubernetes", "//internal/logger", "//internal/role", @@ -37,7 +36,6 @@ go_test( "//bootstrapper/internal/kubernetes/kubewaiter", "//internal/cloud/metadata", "//internal/constants", - "//internal/deploy/helm", "//internal/kubernetes", "//internal/logger", "//internal/role", diff --git a/bootstrapper/internal/kubernetes/k8sapi/k8sutil.go b/bootstrapper/internal/kubernetes/k8sapi/k8sutil.go index 5f81e0adc..4de0d1bed 100644 --- a/bootstrapper/internal/kubernetes/k8sapi/k8sutil.go +++ b/bootstrapper/internal/kubernetes/k8sapi/k8sutil.go @@ -54,6 +54,7 @@ type Client interface { ListAllNamespaces(ctx context.Context) (*corev1.NamespaceList, error) AnnotateNode(ctx context.Context, nodeName, annotationKey, annotationValue string) error EnforceCoreDNSSpread(ctx context.Context) error + PatchFirstNodePodCIDR(ctx context.Context, firstNodePodCIDR string) error } type componentsInstaller interface { diff --git a/bootstrapper/internal/kubernetes/k8sutil.go b/bootstrapper/internal/kubernetes/k8sutil.go index 3acb4c7bb..d8bf72f23 100644 --- a/bootstrapper/internal/kubernetes/k8sutil.go +++ b/bootstrapper/internal/kubernetes/k8sutil.go @@ -10,7 +10,6 @@ import ( "context" "net" - "github.com/edgelesssys/constellation/v2/internal/deploy/helm" "github.com/edgelesssys/constellation/v2/internal/logger" "github.com/edgelesssys/constellation/v2/internal/role" "github.com/edgelesssys/constellation/v2/internal/versions/components" @@ -24,10 +23,3 @@ type clusterUtil interface { FixCilium(ctx context.Context) error StartKubelet() error } - -// helmClient bundles functions related to microservice deployment. -// Only microservices that can be deployed purely via Helm are deployed with this interface. -type helmClient interface { - InstallChart(context.Context, helm.Release) error - InstallChartWithValues(ctx context.Context, release helm.Release, extraValues map[string]any) error -} diff --git a/bootstrapper/internal/kubernetes/kubernetes.go b/bootstrapper/internal/kubernetes/kubernetes.go index a49b751a3..0e296399e 100644 --- a/bootstrapper/internal/kubernetes/kubernetes.go +++ b/bootstrapper/internal/kubernetes/kubernetes.go @@ -9,11 +9,8 @@ package kubernetes import ( "context" - "encoding/json" - "errors" "fmt" "net" - "os/exec" "regexp" "strings" "time" @@ -22,7 +19,6 @@ import ( "github.com/edgelesssys/constellation/v2/bootstrapper/internal/kubernetes/kubewaiter" "github.com/edgelesssys/constellation/v2/internal/cloud/cloudprovider" "github.com/edgelesssys/constellation/v2/internal/constants" - "github.com/edgelesssys/constellation/v2/internal/deploy/helm" "github.com/edgelesssys/constellation/v2/internal/kubernetes" "github.com/edgelesssys/constellation/v2/internal/logger" "github.com/edgelesssys/constellation/v2/internal/role" @@ -49,7 +45,6 @@ type kubeAPIWaiter interface { type KubeWrapper struct { cloudProvider string clusterUtil clusterUtil - helmClient helmClient kubeAPIWaiter kubeAPIWaiter configProvider configurationProvider client k8sapi.Client @@ -59,12 +54,11 @@ type KubeWrapper struct { // New creates a new KubeWrapper with real values. func New(cloudProvider string, clusterUtil clusterUtil, configProvider configurationProvider, client k8sapi.Client, - providerMetadata ProviderMetadata, helmClient helmClient, kubeAPIWaiter kubeAPIWaiter, + providerMetadata ProviderMetadata, kubeAPIWaiter kubeAPIWaiter, ) *KubeWrapper { return &KubeWrapper{ cloudProvider: cloudProvider, clusterUtil: clusterUtil, - helmClient: helmClient, kubeAPIWaiter: kubeAPIWaiter, configProvider: configProvider, client: client, @@ -75,15 +69,13 @@ func New(cloudProvider string, clusterUtil clusterUtil, configProvider configura // InitCluster initializes a new Kubernetes cluster and applies pod network provider. func (k *KubeWrapper) InitCluster( - ctx context.Context, versionString, clusterName string, - helmReleasesRaw []byte, conformanceMode bool, kubernetesComponents components.Components, apiServerCertSANs []string, log *logger.Logger, + ctx context.Context, versionString, clusterName string, conformanceMode bool, kubernetesComponents components.Components, apiServerCertSANs []string, log *logger.Logger, ) ([]byte, error) { log.With(zap.String("version", versionString)).Infof("Installing Kubernetes components") if err := k.clusterUtil.InstallComponents(ctx, kubernetesComponents); err != nil { return nil, err } - var nodePodCIDR string var validIPs []net.IP // Step 1: retrieve cloud metadata for Kubernetes configuration @@ -102,9 +94,6 @@ func (k *KubeWrapper) InitCluster( nodeIP := instance.VPCIP subnetworkPodCIDR := instance.SecondaryIPRange - if len(instance.AliasIPRanges) > 0 { - nodePodCIDR = instance.AliasIPRanges[0] - } // this is the endpoint in "kubeadm init --control-plane-endpoint=:" // TODO(malt3): switch over to DNS name on AWS and Azure @@ -170,6 +159,17 @@ func (k *KubeWrapper) InitCluster( return nil, fmt.Errorf("failed to setup k8s version ConfigMap: %w", err) } + if cloudprovider.FromString(k.cloudProvider) == cloudprovider.GCP { + // GCP uses direct routing, so we need to set the pod CIDR of the first control-plane node for Cilium. + var nodePodCIDR string + if len(instance.AliasIPRanges) > 0 { + nodePodCIDR = instance.AliasIPRanges[0] + } + if err := k.client.PatchFirstNodePodCIDR(ctx, nodePodCIDR); err != nil { + return nil, fmt.Errorf("patching first node pod CIDR: %w", err) + } + } + // Annotate Node with the hash of the installed components if err := k.client.AnnotateNode(ctx, nodeName, constants.NodeKubernetesComponentsAnnotationKey, k8sComponentsConfigMap, @@ -177,49 +177,6 @@ func (k *KubeWrapper) InitCluster( return nil, fmt.Errorf("annotating node with Kubernetes components hash: %w", err) } - // Step 3: configure & start kubernetes controllers - log.Infof("Starting Kubernetes controllers and deployments") - setupPodNetworkInput := k8sapi.SetupPodNetworkInput{ - CloudProvider: k.cloudProvider, - NodeName: nodeName, - FirstNodePodCIDR: nodePodCIDR, - SubnetworkPodCIDR: subnetworkPodCIDR, - LoadBalancerHost: controlPlaneHost, - LoadBalancerPort: controlPlanePort, - } - - var helmReleases helm.Releases - if err := json.Unmarshal(helmReleasesRaw, &helmReleases); err != nil { - return nil, fmt.Errorf("unmarshalling helm releases: %w", err) - } - - log.Infof("Installing Cilium") - ciliumVals, err := k.setupCiliumVals(ctx, setupPodNetworkInput) - if err != nil { - return nil, fmt.Errorf("setting up cilium vals: %w", err) - } - if err := k.helmClient.InstallChartWithValues(ctx, helmReleases.Cilium, ciliumVals); err != nil { - return nil, fmt.Errorf("installing cilium pod network: %w", err) - } - - log.Infof("Waiting for Cilium to become healthy") - timeToStartWaiting := time.Now() - // TODO(3u13r): Reduce the timeout when we switched the package repository - this is only this high because we once - // saw polling times of ~16 minutes when hitting a slow PoP from Fastly (GitHub's / ghcr.io CDN). - waitCtx, cancel = context.WithTimeout(ctx, 20*time.Minute) - defer cancel() - if err := k.clusterUtil.WaitForCilium(waitCtx, log); err != nil { - return nil, fmt.Errorf("waiting for Cilium to become healthy: %w", err) - } - timeUntilFinishedWaiting := time.Since(timeToStartWaiting) - log.With(zap.Duration("duration", timeUntilFinishedWaiting)).Infof("Cilium became healthy") - - log.Infof("Restarting Cilium") - if err := k.clusterUtil.FixCilium(ctx); err != nil { - log.With(zap.Error(err)).Errorf("FixCilium failed") - // Continue and don't throw an error here - things might be okay. - } - log.Infof("Setting up internal-config ConfigMap") if err := k.setupInternalConfigMap(ctx); err != nil { return nil, fmt.Errorf("failed to setup internal ConfigMap: %w", err) @@ -378,28 +335,3 @@ func getIPAddr() (string, error) { return localAddr.IP.String(), nil } - -func (k *KubeWrapper) setupCiliumVals(ctx context.Context, in k8sapi.SetupPodNetworkInput) (map[string]any, error) { - vals := map[string]any{ - "k8sServiceHost": in.LoadBalancerHost, - "k8sServicePort": in.LoadBalancerPort, - } - - // GCP requires extra configuration for Cilium - if cloudprovider.FromString(k.cloudProvider) == cloudprovider.GCP { - if out, err := exec.CommandContext( - ctx, constants.KubectlPath, - "--kubeconfig", constants.ControlPlaneAdminConfFilename, - "patch", "node", in.NodeName, "-p", "{\"spec\":{\"podCIDR\": \""+in.FirstNodePodCIDR+"\"}}", - ).CombinedOutput(); err != nil { - err = errors.New(string(out)) - return nil, fmt.Errorf("%s: %w", out, err) - } - - vals["ipv4NativeRoutingCIDR"] = in.SubnetworkPodCIDR - vals["strictModeCIDR"] = in.SubnetworkPodCIDR - - } - - return vals, nil -} diff --git a/bootstrapper/internal/kubernetes/kubernetes_test.go b/bootstrapper/internal/kubernetes/kubernetes_test.go index 7bc513907..e0c4f5e4e 100644 --- a/bootstrapper/internal/kubernetes/kubernetes_test.go +++ b/bootstrapper/internal/kubernetes/kubernetes_test.go @@ -17,7 +17,6 @@ import ( "github.com/edgelesssys/constellation/v2/bootstrapper/internal/kubernetes/kubewaiter" "github.com/edgelesssys/constellation/v2/internal/cloud/metadata" "github.com/edgelesssys/constellation/v2/internal/constants" - "github.com/edgelesssys/constellation/v2/internal/deploy/helm" "github.com/edgelesssys/constellation/v2/internal/kubernetes" "github.com/edgelesssys/constellation/v2/internal/logger" "github.com/edgelesssys/constellation/v2/internal/role" @@ -43,7 +42,6 @@ func TestInitCluster(t *testing.T) { testCases := map[string]struct { clusterUtil stubClusterUtil - helmClient stubHelmClient kubectl stubKubectl kubeAPIWaiter stubKubeAPIWaiter providerMetadata ProviderMetadata @@ -131,14 +129,12 @@ func TestInitCluster(t *testing.T) { }, "kubeadm init fails when deploying cilium": { clusterUtil: stubClusterUtil{kubeconfig: []byte("someKubeconfig")}, - helmClient: stubHelmClient{ciliumError: assert.AnError}, providerMetadata: &stubProviderMetadata{}, wantErr: true, k8sVersion: versions.Default, }, "kubeadm init fails when setting up constellation-services chart": { clusterUtil: stubClusterUtil{kubeconfig: []byte("someKubeconfig")}, - helmClient: stubHelmClient{installChartError: assert.AnError}, kubeAPIWaiter: stubKubeAPIWaiter{}, providerMetadata: &stubProviderMetadata{}, wantErr: true, @@ -182,7 +178,6 @@ func TestInitCluster(t *testing.T) { kube := KubeWrapper{ cloudProvider: "aws", // provide a valid cloud provider for cilium installation clusterUtil: &tc.clusterUtil, - helmClient: &tc.helmClient, providerMetadata: tc.providerMetadata, kubeAPIWaiter: &tc.kubeAPIWaiter, configProvider: &stubConfigProvider{initConfig: k8sapi.KubeadmInitYAML{}}, @@ -192,7 +187,7 @@ func TestInitCluster(t *testing.T) { _, err := kube.InitCluster( context.Background(), string(tc.k8sVersion), "kubernetes", - []byte("{}"), false, nil, nil, logger.NewTest(t), + false, nil, nil, logger.NewTest(t), ) if tc.wantErr { @@ -539,6 +534,10 @@ func (s *stubKubectl) AnnotateNode(_ context.Context, _, _, _ string) error { return s.annotateNodeErr } +func (s *stubKubectl) PatchFirstNodePodCIDR(_ context.Context, _ string) error { + return nil +} + func (s *stubKubectl) WaitForCRDs(_ context.Context, _ []string) error { return s.waitForCRDsErr } @@ -551,24 +550,6 @@ func (s *stubKubectl) EnforceCoreDNSSpread(_ context.Context) error { return s.enforceCoreDNSSpreadErr } -type stubHelmClient struct { - ciliumError error - installChartError error -} - -func (s *stubHelmClient) InstallChart(ctx context.Context, release helm.Release) error { - return s.InstallChartWithValues(ctx, release, release.Values) -} - -func (s *stubHelmClient) InstallChartWithValues(_ context.Context, release helm.Release, _ map[string]any) error { - switch release.ReleaseName { - case "cilium": - return s.ciliumError - default: - return s.installChartError - } -} - type stubKubeAPIWaiter struct { waitErr error } diff --git a/cli/internal/cmd/BUILD.bazel b/cli/internal/cmd/BUILD.bazel index 88446a09e..f9e4827e0 100644 --- a/cli/internal/cmd/BUILD.bazel +++ b/cli/internal/cmd/BUILD.bazel @@ -67,7 +67,6 @@ go_library( "//internal/config/migration", "//internal/constants", "//internal/crypto", - "//internal/deploy/helm", "//internal/file", "//internal/grpc/dialer", "//internal/grpc/grpclog", @@ -139,6 +138,7 @@ go_test( "//bootstrapper/initproto", "//cli/internal/cloudcmd", "//cli/internal/clusterid", + "//cli/internal/helm", "//cli/internal/iamid", "//cli/internal/kubernetes", "//cli/internal/terraform", @@ -154,7 +154,6 @@ go_test( "//internal/config", "//internal/constants", "//internal/crypto/testvector", - "//internal/deploy/helm", "//internal/file", "//internal/grpc/atlscredentials", "//internal/grpc/dialer", diff --git a/cli/internal/cmd/init.go b/cli/internal/cmd/init.go index 2f1386521..005c854c7 100644 --- a/cli/internal/cmd/init.go +++ b/cli/internal/cmd/init.go @@ -10,7 +10,6 @@ import ( "bytes" "context" "encoding/hex" - "encoding/json" "errors" "fmt" "io" @@ -45,7 +44,6 @@ import ( "github.com/edgelesssys/constellation/v2/internal/config" "github.com/edgelesssys/constellation/v2/internal/constants" "github.com/edgelesssys/constellation/v2/internal/crypto" - helmdeploy "github.com/edgelesssys/constellation/v2/internal/deploy/helm" "github.com/edgelesssys/constellation/v2/internal/file" "github.com/edgelesssys/constellation/v2/internal/grpc/dialer" "github.com/edgelesssys/constellation/v2/internal/grpc/grpclog" @@ -79,7 +77,7 @@ type initCmd struct { spinner spinnerInterf masterSecret uri.MasterSecret fh *file.Handler - helmInstaller helm.SuiteInstaller + helmInstaller helm.Initializer } // runInitialize runs the initialize command. @@ -103,7 +101,7 @@ func runInitialize(cmd *cobra.Command, _ []string) error { ctx, cancel := context.WithTimeout(cmd.Context(), time.Hour) defer cancel() cmd.SetContext(ctx) - helmInstaller, err := helm.NewInstallationClient(log) + helmInstaller, err := helm.NewInitializer(log) if err != nil { return fmt.Errorf("creating Helm installer: %w", err) } @@ -191,10 +189,6 @@ func (i *initCmd) initialize(cmd *cobra.Command, newDialer func(validator atls.V if err != nil { return fmt.Errorf("loading Helm charts: %w", err) } - helmDeployments, err := json.Marshal(releases) - if err != nil { - return err - } i.log.Debugf("Loaded Helm deployments") if err != nil { return fmt.Errorf("loading Helm charts: %w", err) @@ -208,7 +202,6 @@ func (i *initCmd) initialize(cmd *cobra.Command, newDialer func(validator atls.V CloudServiceAccountUri: serviceAccURI, KubernetesVersion: versions.VersionConfigs[k8sVersion].ClusterVersion, KubernetesComponents: versions.VersionConfigs[k8sVersion].KubernetesComponents.ToInitProto(), - HelmDeployments: helmDeployments, ConformanceMode: flags.conformance, InitSecret: idFile.InitSecret, ClusterName: clusterName, @@ -441,9 +434,9 @@ func (i *initCmd) evalFlagArgs(cmd *cobra.Command) (initFlags, error) { if err != nil { return initFlags{}, fmt.Errorf("parsing skip-helm-wait flag: %w", err) } - helmWaitMode := helmdeploy.WaitModeAtomic + helmWaitMode := helm.WaitModeAtomic if skipHelmWait { - helmWaitMode = helmdeploy.WaitModeNone + helmWaitMode = helm.WaitModeNone } i.log.Debugf("Helm wait flag is %t", skipHelmWait) configPath, err := cmd.Flags().GetString("config") @@ -478,7 +471,7 @@ type initFlags struct { configPath string masterSecretPath string conformance bool - helmWaitMode helmdeploy.WaitMode + helmWaitMode helm.WaitMode force bool mergeConfigs bool } diff --git a/cli/internal/cmd/init_test.go b/cli/internal/cmd/init_test.go index bafe7e4a7..1a1ac03c7 100644 --- a/cli/internal/cmd/init_test.go +++ b/cli/internal/cmd/init_test.go @@ -21,6 +21,7 @@ import ( "github.com/edgelesssys/constellation/v2/bootstrapper/initproto" "github.com/edgelesssys/constellation/v2/cli/internal/clusterid" + "github.com/edgelesssys/constellation/v2/cli/internal/helm" "github.com/edgelesssys/constellation/v2/internal/atls" "github.com/edgelesssys/constellation/v2/internal/attestation/measurements" "github.com/edgelesssys/constellation/v2/internal/attestation/variant" @@ -28,7 +29,6 @@ import ( "github.com/edgelesssys/constellation/v2/internal/cloud/gcpshared" "github.com/edgelesssys/constellation/v2/internal/config" "github.com/edgelesssys/constellation/v2/internal/constants" - helminstaller "github.com/edgelesssys/constellation/v2/internal/deploy/helm" "github.com/edgelesssys/constellation/v2/internal/file" "github.com/edgelesssys/constellation/v2/internal/grpc/atlscredentials" "github.com/edgelesssys/constellation/v2/internal/grpc/dialer" @@ -672,7 +672,7 @@ type stubHelmInstaller struct{} func (i *stubHelmInstaller) Install(_ context.Context, _ cloudprovider.Provider, _ uri.MasterSecret, _ clusterid.File, - _ string, _ *helminstaller.Releases, + _ string, _ *helm.Releases, ) error { return nil } diff --git a/cli/internal/cmd/status.go b/cli/internal/cmd/status.go index 15add6d87..1c00caaa0 100644 --- a/cli/internal/cmd/status.go +++ b/cli/internal/cmd/status.go @@ -74,7 +74,7 @@ func runStatus(cmd *cobra.Command, _ []string) error { } // need helm client to fetch service versions. - helmClient, err := helm.NewClient(kubectl.New(), constants.AdminConfFilename, constants.HelmNamespace, log) + helmClient, err := helm.NewUpgradeClient(kubectl.New(), constants.AdminConfFilename, constants.HelmNamespace, log) if err != nil { return fmt.Errorf("setting up helm client: %w", err) } diff --git a/cli/internal/cmd/upgradecheck.go b/cli/internal/cmd/upgradecheck.go index d5fc5df37..45851c6a5 100644 --- a/cli/internal/cmd/upgradecheck.go +++ b/cli/internal/cmd/upgradecheck.go @@ -383,7 +383,7 @@ type currentVersionInfo struct { } func (v *versionCollector) currentVersions(ctx context.Context) (currentVersionInfo, error) { - helmClient, err := helm.NewClient(kubectl.New(), constants.AdminConfFilename, constants.HelmNamespace, v.log) + helmClient, err := helm.NewUpgradeClient(kubectl.New(), constants.AdminConfFilename, constants.HelmNamespace, v.log) if err != nil { return currentVersionInfo{}, fmt.Errorf("setting up helm client: %w", err) } diff --git a/cli/internal/helm/BUILD.bazel b/cli/internal/helm/BUILD.bazel index 3d4d74ce8..08df2735f 100644 --- a/cli/internal/helm/BUILD.bazel +++ b/cli/internal/helm/BUILD.bazel @@ -5,12 +5,14 @@ go_library( name = "helm", srcs = [ "backup.go", - "client.go", + "ciliumhelper.go", "helm.go", - "helminstaller.go", + "init.go", + "install.go", "loader.go", + "release.go", "serviceversion.go", - "setup.go", + "upgrade.go", "values.go", ], embedsrcs = [ @@ -427,17 +429,21 @@ go_library( "//internal/compatibility", "//internal/config", "//internal/constants", - "//internal/deploy/helm", "//internal/file", "//internal/kms/uri", + "//internal/retry", "//internal/semver", "//internal/versions", "@com_github_pkg_errors//:errors", "@com_github_spf13_afero//:afero", "@io_k8s_apiextensions_apiserver//pkg/apis/apiextensions/v1:apiextensions", "@io_k8s_apimachinery//pkg/api/errors", + "@io_k8s_apimachinery//pkg/apis/meta/v1:meta", "@io_k8s_apimachinery//pkg/apis/meta/v1/unstructured", "@io_k8s_apimachinery//pkg/runtime/schema", + "@io_k8s_apimachinery//pkg/util/wait", + "@io_k8s_client_go//kubernetes", + "@io_k8s_client_go//tools/clientcmd", "@io_k8s_sigs_yaml//:yaml", "@sh_helm_helm//pkg/ignore", "@sh_helm_helm_v3//pkg/action", @@ -453,8 +459,9 @@ go_test( name = "helm_test", srcs = [ "backup_test.go", - "client_test.go", + "helm_test.go", "loader_test.go", + "upgrade_test.go", ], data = glob(["testdata/**"]), embed = [":helm"], @@ -465,7 +472,6 @@ go_test( "//internal/cloud/cloudprovider", "//internal/compatibility", "//internal/config", - "//internal/deploy/helm", "//internal/file", "//internal/logger", "//internal/semver", diff --git a/cli/internal/helm/backup.go b/cli/internal/helm/backup.go index 162c71a9e..aff984ad9 100644 --- a/cli/internal/helm/backup.go +++ b/cli/internal/helm/backup.go @@ -18,7 +18,7 @@ import ( "sigs.k8s.io/yaml" ) -func (c *Client) backupCRDs(ctx context.Context, upgradeID string) ([]apiextensionsv1.CustomResourceDefinition, error) { +func (c *UpgradeClient) backupCRDs(ctx context.Context, upgradeID string) ([]apiextensionsv1.CustomResourceDefinition, error) { c.log.Debugf("Starting CRD backup") crds, err := c.kubectl.GetCRDs(ctx) if err != nil { @@ -53,7 +53,7 @@ func (c *Client) backupCRDs(ctx context.Context, upgradeID string) ([]apiextensi return crds, nil } -func (c *Client) backupCRs(ctx context.Context, crds []apiextensionsv1.CustomResourceDefinition, upgradeID string) error { +func (c *UpgradeClient) backupCRs(ctx context.Context, crds []apiextensionsv1.CustomResourceDefinition, upgradeID string) error { c.log.Debugf("Starting CR backup") for _, crd := range crds { c.log.Debugf("Creating backup for resource type: %s", crd.Name) @@ -99,10 +99,10 @@ func (c *Client) backupCRs(ctx context.Context, crds []apiextensionsv1.CustomRes return nil } -func (c *Client) backupFolder(upgradeID string) string { +func (c *UpgradeClient) backupFolder(upgradeID string) string { return filepath.Join(constants.UpgradeDir, upgradeID, "backups") + string(filepath.Separator) } -func (c *Client) crdBackupFolder(upgradeID string) string { +func (c *UpgradeClient) crdBackupFolder(upgradeID string) string { return filepath.Join(c.backupFolder(upgradeID), "crds") + string(filepath.Separator) } diff --git a/cli/internal/helm/backup_test.go b/cli/internal/helm/backup_test.go index 1533c7b31..46421ba28 100644 --- a/cli/internal/helm/backup_test.go +++ b/cli/internal/helm/backup_test.go @@ -52,7 +52,7 @@ func TestBackupCRDs(t *testing.T) { crd := apiextensionsv1.CustomResourceDefinition{} err := yaml.Unmarshal([]byte(tc.crd), &crd) require.NoError(err) - client := Client{ + client := UpgradeClient{ config: nil, kubectl: stubCrdClient{crds: []apiextensionsv1.CustomResourceDefinition{crd}, getCRDsError: tc.getCRDsError}, fs: file.NewHandler(memFs), @@ -143,7 +143,7 @@ func TestBackupCRs(t *testing.T) { require := require.New(t) memFs := afero.NewMemMapFs() - client := Client{ + client := UpgradeClient{ config: nil, kubectl: stubCrdClient{crs: []unstructured.Unstructured{tc.resource}, getCRsError: tc.getCRsError}, fs: file.NewHandler(memFs), diff --git a/cli/internal/helm/ciliumhelper.go b/cli/internal/helm/ciliumhelper.go new file mode 100644 index 000000000..c36dcdcc6 --- /dev/null +++ b/cli/internal/helm/ciliumhelper.go @@ -0,0 +1,72 @@ +/* +Copyright (c) Edgeless Systems GmbH + +SPDX-License-Identifier: AGPL-3.0-only +*/ + +package helm + +import ( + "context" + "fmt" + "time" + + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/tools/clientcmd" +) + +type k8sDsClient struct { + clientset *kubernetes.Clientset +} + +func newK8sCiliumHelper(kubeconfigPath string) (*k8sDsClient, error) { + config, err := clientcmd.BuildConfigFromFlags("", kubeconfigPath) + if err != nil { + return nil, err + } + clientset, err := kubernetes.NewForConfig(config) + if err != nil { + return nil, err + } + return &k8sDsClient{clientset: clientset}, nil +} + +// WaitForDS waits for a DaemonSet to become ready. +func (h *k8sDsClient) WaitForDS(ctx context.Context, namespace, name string, log debugLog) error { + for { + select { + case <-ctx.Done(): + return fmt.Errorf("context expired before DaemonSet %q became ready", name) + default: + ds, err := h.clientset.AppsV1().DaemonSets(namespace).Get(ctx, name, v1.GetOptions{}) + if err != nil { + return err + } + + if ds.Status.NumberReady == ds.Status.DesiredNumberScheduled { + log.Debugf("DaemonSet %s is ready\n", name) + return nil + } + + log.Debugf("Waiting for DaemonSet %s to become ready...\n", name) + time.Sleep(10 * time.Second) + } + } +} + +// RestartDS restarts all pods of a DaemonSet by updating its template. +func (h *k8sDsClient) RestartDS(namespace, name string) error { + ds, err := h.clientset.AppsV1().DaemonSets(namespace).Get(context.Background(), name, v1.GetOptions{}) + if err != nil { + return err + } + + ds.Spec.Template.ObjectMeta.Annotations["restartTimestamp"] = fmt.Sprintf("%d", time.Now().Unix()) + _, err = h.clientset.AppsV1().DaemonSets(namespace).Update(context.Background(), ds, v1.UpdateOptions{}) + if err != nil { + return err + } + + return nil +} diff --git a/cli/internal/helm/helm.go b/cli/internal/helm/helm.go index dd6487540..51ec7fcff 100644 --- a/cli/internal/helm/helm.go +++ b/cli/internal/helm/helm.go @@ -15,3 +15,25 @@ It is used by the CLI to: - create local backups before running service upgrades */ package helm + +// mergeMaps returns a new map that is the merger of it's inputs. +// Key collisions are resolved by taking the value of the second argument (map b). +// Taken from: https://github.com/helm/helm/blob/dbc6d8e20fe1d58d50e6ed30f09a04a77e4c68db/pkg/cli/values/options.go#L91-L108. +func mergeMaps(a, b map[string]any) map[string]any { + out := make(map[string]any, len(a)) + for k, v := range a { + out[k] = v + } + for k, v := range b { + if v, ok := v.(map[string]any); ok { + if bv, ok := out[k]; ok { + if bv, ok := bv.(map[string]any); ok { + out[k] = mergeMaps(bv, v) + continue + } + } + } + out[k] = v + } + return out +} diff --git a/internal/deploy/helm/helm_test.go b/cli/internal/helm/helm_test.go similarity index 97% rename from internal/deploy/helm/helm_test.go rename to cli/internal/helm/helm_test.go index 78576d5a5..34164ec54 100644 --- a/internal/deploy/helm/helm_test.go +++ b/cli/internal/helm/helm_test.go @@ -102,7 +102,7 @@ func TestMergeMaps(t *testing.T) { for name, tc := range testCases { t.Run(name, func(t *testing.T) { assert := assert.New(t) - newVals := MergeMaps(tc.vals, tc.extraVals) + newVals := mergeMaps(tc.vals, tc.extraVals) assert.Equal(tc.expected, newVals) }) } diff --git a/cli/internal/helm/helminstaller.go b/cli/internal/helm/helminstaller.go deleted file mode 100644 index f8c7489d9..000000000 --- a/cli/internal/helm/helminstaller.go +++ /dev/null @@ -1,100 +0,0 @@ -/* -Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only -*/ - -package helm - -import ( - "context" - "fmt" - - "github.com/edgelesssys/constellation/v2/cli/internal/clusterid" - "github.com/edgelesssys/constellation/v2/internal/cloud/cloudprovider" - "github.com/edgelesssys/constellation/v2/internal/cloud/openstack" - "github.com/edgelesssys/constellation/v2/internal/constants" - helminstaller "github.com/edgelesssys/constellation/v2/internal/deploy/helm" - "github.com/edgelesssys/constellation/v2/internal/kms/uri" -) - -// SuiteInstaller installs all Helm charts required for a constellation cluster. -type SuiteInstaller interface { - Install(ctx context.Context, provider cloudprovider.Provider, masterSecret uri.MasterSecret, - idFile clusterid.File, - serviceAccURI string, releases *helminstaller.Releases, - ) error -} - -type helmInstallationClient struct { - log debugLog - installer helmInstaller -} - -// NewInstallationClient creates a new Helm installation client to install all Helm charts required for a constellation cluster. -func NewInstallationClient(log debugLog) (SuiteInstaller, error) { - installer, err := helminstaller.NewInstaller(constants.AdminConfFilename, log) - if err != nil { - return nil, fmt.Errorf("creating Helm installer: %w", err) - } - return &helmInstallationClient{log: log, installer: installer}, nil -} - -func (h helmInstallationClient) Install(ctx context.Context, provider cloudprovider.Provider, masterSecret uri.MasterSecret, - idFile clusterid.File, - serviceAccURI string, releases *helminstaller.Releases, -) error { - serviceVals, err := setupMicroserviceVals(ctx, provider, masterSecret.Salt, idFile.UID, serviceAccURI) - if err != nil { - return fmt.Errorf("setting up microservice values: %w", err) - } - if err := h.installer.InstallChartWithValues(ctx, releases.ConstellationServices, serviceVals); err != nil { - return fmt.Errorf("installing microservices: %w", err) - } - - h.log.Debugf("Installing cert-manager") - if err := h.installer.InstallChart(ctx, releases.CertManager); err != nil { - return fmt.Errorf("installing cert-manager: %w", err) - } - - if releases.CSI != nil { - var csiVals map[string]any - if provider == cloudprovider.OpenStack { - creds, err := openstack.AccountKeyFromURI(serviceAccURI) - if err != nil { - return err - } - cinderIni := creds.CloudINI().CinderCSIConfiguration() - csiVals = map[string]any{ - "cinder-config": map[string]any{ - "secretData": cinderIni, - }, - } - } - - h.log.Debugf("Installing CSI deployments") - if err := h.installer.InstallChartWithValues(ctx, *releases.CSI, csiVals); err != nil { - return fmt.Errorf("installing CSI snapshot CRDs: %w", err) - } - } - - if releases.AWSLoadBalancerController != nil { - h.log.Debugf("Installing AWS Load Balancer Controller") - if err := h.installer.InstallChart(ctx, *releases.AWSLoadBalancerController); err != nil { - return fmt.Errorf("installing AWS Load Balancer Controller: %w", err) - } - } - - h.log.Debugf("Installing constellation operators") - operatorVals := setupOperatorVals(ctx, idFile.UID) - if err := h.installer.InstallChartWithValues(ctx, releases.ConstellationOperators, operatorVals); err != nil { - return fmt.Errorf("installing constellation operators: %w", err) - } - - // TODO(elchead): AB#3301 do cilium after version upgrade - return nil -} - -type helmInstaller interface { - InstallChart(context.Context, helminstaller.Release) error - InstallChartWithValues(ctx context.Context, release helminstaller.Release, extraValues map[string]any) error -} diff --git a/cli/internal/helm/init.go b/cli/internal/helm/init.go new file mode 100644 index 000000000..99b6503ab --- /dev/null +++ b/cli/internal/helm/init.go @@ -0,0 +1,258 @@ +/* +Copyright (c) Edgeless Systems GmbH +SPDX-License-Identifier: AGPL-3.0-only +*/ + +package helm + +import ( + "context" + "encoding/base64" + "encoding/json" + "fmt" + "time" + + "github.com/edgelesssys/constellation/v2/cli/internal/clusterid" + "github.com/edgelesssys/constellation/v2/cli/internal/terraform" + "github.com/edgelesssys/constellation/v2/internal/cloud/azureshared" + "github.com/edgelesssys/constellation/v2/internal/cloud/cloudprovider" + "github.com/edgelesssys/constellation/v2/internal/cloud/gcpshared" + "github.com/edgelesssys/constellation/v2/internal/cloud/openstack" + "github.com/edgelesssys/constellation/v2/internal/constants" + "github.com/edgelesssys/constellation/v2/internal/kms/uri" +) + +// Initializer installs all Helm charts required for a constellation cluster. +type Initializer interface { + Install(ctx context.Context, provider cloudprovider.Provider, masterSecret uri.MasterSecret, + idFile clusterid.File, + serviceAccURI string, releases *Releases, + ) error +} + +type initializationClient struct { + log debugLog + installer installer +} + +// NewInitializer creates a new client to install all Helm charts required for a constellation cluster. +func NewInitializer(log debugLog) (Initializer, error) { + installer, err := NewInstaller(constants.AdminConfFilename, log) + if err != nil { + return nil, fmt.Errorf("creating Helm installer: %w", err) + } + return &initializationClient{log: log, installer: installer}, nil +} + +// Install installs all Helm charts required for a constellation cluster. +func (h initializationClient) Install(ctx context.Context, provider cloudprovider.Provider, masterSecret uri.MasterSecret, + idFile clusterid.File, + serviceAccURI string, releases *Releases, +) error { + tfClient, err := terraform.New(ctx, constants.TerraformWorkingDir) + if err != nil { + return fmt.Errorf("creating Terraform client: %w", err) + } + output, err := tfClient.ShowCluster(ctx, provider) + if err != nil { + return fmt.Errorf("getting Terraform output: %w", err) + } + ciliumVals := setupCiliumVals(provider, output) + if err := h.installer.InstallChartWithValues(ctx, releases.Cilium, ciliumVals); err != nil { + return fmt.Errorf("installing Cilium: %w", err) + } + h.log.Debugf("Waiting for Cilium to become ready") + helper, err := newK8sCiliumHelper(constants.AdminConfFilename) + if err != nil { + return fmt.Errorf("creating Kubernetes client: %w", err) + } + timeToStartWaiting := time.Now() + // TODO(3u13r): Reduce the timeout when we switched the package repository - this is only this high because we once + // saw polling times of ~16 minutes when hitting a slow PoP from Fastly (GitHub's / ghcr.io CDN). + if err := helper.WaitForDS(ctx, "kube-system", "cilium", h.log); err != nil { + return fmt.Errorf("waiting for Cilium to become healthy: %w", err) + } + timeUntilFinishedWaiting := time.Since(timeToStartWaiting) + h.log.Debugf("Cilium became healthy after %s", timeUntilFinishedWaiting.String()) + + h.log.Debugf("Fix Cilium through restart") + if err := helper.RestartDS("kube-system", "cilium"); err != nil { + return fmt.Errorf("restarting Cilium: %w", err) + } + + h.log.Debugf("Installing microservices") + serviceVals, err := setupMicroserviceVals(provider, masterSecret.Salt, idFile.UID, serviceAccURI, output) + if err != nil { + return fmt.Errorf("setting up microservice values: %w", err) + } + if err := h.installer.InstallChartWithValues(ctx, releases.ConstellationServices, serviceVals); err != nil { + return fmt.Errorf("installing microservices: %w", err) + } + + h.log.Debugf("Installing cert-manager") + if err := h.installer.InstallChart(ctx, releases.CertManager); err != nil { + return fmt.Errorf("installing cert-manager: %w", err) + } + + if releases.CSI != nil { + var csiVals map[string]any + if provider == cloudprovider.OpenStack { + creds, err := openstack.AccountKeyFromURI(serviceAccURI) + if err != nil { + return err + } + cinderIni := creds.CloudINI().CinderCSIConfiguration() + csiVals = map[string]any{ + "cinder-config": map[string]any{ + "secretData": cinderIni, + }, + } + } + + h.log.Debugf("Installing CSI deployments") + if err := h.installer.InstallChartWithValues(ctx, *releases.CSI, csiVals); err != nil { + return fmt.Errorf("installing CSI snapshot CRDs: %w", err) + } + } + + if releases.AWSLoadBalancerController != nil { + h.log.Debugf("Installing AWS Load Balancer Controller") + if err := h.installer.InstallChart(ctx, *releases.AWSLoadBalancerController); err != nil { + return fmt.Errorf("installing AWS Load Balancer Controller: %w", err) + } + } + + h.log.Debugf("Installing constellation operators") + operatorVals := setupOperatorVals(ctx, idFile.UID) + if err := h.installer.InstallChartWithValues(ctx, releases.ConstellationOperators, operatorVals); err != nil { + return fmt.Errorf("installing constellation operators: %w", err) + } + return nil +} + +// installer is the interface for installing a single Helm chart. +type installer interface { + InstallChart(context.Context, Release) error + InstallChartWithValues(ctx context.Context, release Release, extraValues map[string]any) error +} + +// TODO(malt3): switch over to DNS name on AWS and Azure +// soon as every apiserver certificate of every control-plane node +// has the dns endpoint in its SAN list. +func setupCiliumVals(provider cloudprovider.Provider, output terraform.ApplyOutput) map[string]any { + vals := map[string]any{ + "k8sServiceHost": output.IP, + "k8sServicePort": constants.KubernetesPort, + } + if provider == cloudprovider.GCP { + vals["ipv4NativeRoutingCIDR"] = output.GCP.IPCidrPod + vals["strictModeCIDR"] = output.GCP.IPCidrPod + } + return vals +} + +// setupMicroserviceVals returns the values for the microservice chart. +func setupMicroserviceVals(provider cloudprovider.Provider, measurementSalt []byte, uid, serviceAccURI string, output terraform.ApplyOutput) (map[string]any, error) { + extraVals := map[string]any{ + "join-service": map[string]any{ + "measurementSalt": base64.StdEncoding.EncodeToString(measurementSalt), + }, + "verification-service": map[string]any{ + "loadBalancerIP": output.IP, + }, + "konnectivity": map[string]any{ + "loadBalancerIP": output.IP, + }, + } + switch provider { + case cloudprovider.GCP: + serviceAccountKey, err := gcpshared.ServiceAccountKeyFromURI(serviceAccURI) + if err != nil { + return nil, fmt.Errorf("getting service account key: %w", err) + } + rawKey, err := json.Marshal(serviceAccountKey) + if err != nil { + return nil, fmt.Errorf("marshaling service account key: %w", err) + } + if output.GCP == nil { + return nil, fmt.Errorf("no GCP output from Terraform") + } + extraVals["ccm"] = map[string]any{ + "GCP": map[string]any{ + "projectID": output.GCP.ProjectID, + "uid": uid, + "secretData": string(rawKey), + "subnetworkPodCIDR": output.GCP.IPCidrPod, + }, + } + case cloudprovider.Azure: + if output.Azure == nil { + return nil, fmt.Errorf("no Azure output from Terraform") + } + ccmConfig, err := getCCMConfig(*output.Azure, serviceAccURI) + if err != nil { + return nil, fmt.Errorf("getting Azure CCM config: %w", err) + } + extraVals["ccm"] = map[string]any{ + "Azure": map[string]any{ + "azureConfig": string(ccmConfig), + }, + } + } + + return extraVals, nil +} + +// setupOperatorVals returns the values for the constellation-operator chart. +func setupOperatorVals(_ context.Context, uid string) map[string]any { + return map[string]any{ + "constellation-operator": map[string]any{ + "constellationUID": uid, + }, + } +} + +type cloudConfig struct { + Cloud string `json:"cloud,omitempty"` + TenantID string `json:"tenantId,omitempty"` + SubscriptionID string `json:"subscriptionId,omitempty"` + ResourceGroup string `json:"resourceGroup,omitempty"` + Location string `json:"location,omitempty"` + SubnetName string `json:"subnetName,omitempty"` + SecurityGroupName string `json:"securityGroupName,omitempty"` + SecurityGroupResourceGroup string `json:"securityGroupResourceGroup,omitempty"` + LoadBalancerName string `json:"loadBalancerName,omitempty"` + LoadBalancerSku string `json:"loadBalancerSku,omitempty"` + VNetName string `json:"vnetName,omitempty"` + VNetResourceGroup string `json:"vnetResourceGroup,omitempty"` + CloudProviderBackoff bool `json:"cloudProviderBackoff,omitempty"` + UseInstanceMetadata bool `json:"useInstanceMetadata,omitempty"` + VMType string `json:"vmType,omitempty"` + UseManagedIdentityExtension bool `json:"useManagedIdentityExtension,omitempty"` + UserAssignedIdentityID string `json:"userAssignedIdentityID,omitempty"` +} + +// GetCCMConfig returns the configuration needed for the Kubernetes Cloud Controller Manager on Azure. +func getCCMConfig(tfOutput terraform.AzureApplyOutput, serviceAccURI string) ([]byte, error) { + creds, err := azureshared.ApplicationCredentialsFromURI(serviceAccURI) + if err != nil { + return nil, fmt.Errorf("getting service account key: %w", err) + } + useManagedIdentityExtension := creds.PreferredAuthMethod == azureshared.AuthMethodUserAssignedIdentity + config := cloudConfig{ + Cloud: "AzurePublicCloud", + TenantID: creds.TenantID, + SubscriptionID: tfOutput.SubscriptionID, + ResourceGroup: tfOutput.ResourceGroup, + LoadBalancerSku: "standard", + SecurityGroupName: tfOutput.NetworkSecurityGroupName, + LoadBalancerName: tfOutput.LoadBalancerName, + UseInstanceMetadata: true, + VMType: "vmss", + Location: creds.Location, + UseManagedIdentityExtension: useManagedIdentityExtension, + UserAssignedIdentityID: tfOutput.UserAssignedIdentity, + } + + return json.Marshal(config) +} diff --git a/internal/deploy/helm/install.go b/cli/internal/helm/install.go similarity index 98% rename from internal/deploy/helm/install.go rename to cli/internal/helm/install.go index 6e0558a26..14b91dadf 100644 --- a/internal/deploy/helm/install.go +++ b/cli/internal/helm/install.go @@ -68,7 +68,7 @@ func (h *Installer) InstallChart(ctx context.Context, release Release) error { // InstallChartWithValues is the generic install function for helm charts with custom values. func (h *Installer) InstallChartWithValues(ctx context.Context, release Release, extraValues map[string]any) error { - mergedVals := MergeMaps(release.Values, extraValues) + mergedVals := mergeMaps(release.Values, extraValues) h.ReleaseName = release.ReleaseName if err := h.SetWaitMode(release.WaitMode); err != nil { return err diff --git a/cli/internal/helm/loader.go b/cli/internal/helm/loader.go index 622c1de59..0d659cd60 100644 --- a/cli/internal/helm/loader.go +++ b/cli/internal/helm/loader.go @@ -27,7 +27,6 @@ import ( "github.com/edgelesssys/constellation/v2/internal/cloud/cloudprovider" "github.com/edgelesssys/constellation/v2/internal/config" "github.com/edgelesssys/constellation/v2/internal/constants" - "github.com/edgelesssys/constellation/v2/internal/deploy/helm" "github.com/edgelesssys/constellation/v2/internal/semver" "github.com/edgelesssys/constellation/v2/internal/versions" ) @@ -108,21 +107,8 @@ func NewLoader(csp cloudprovider.Provider, k8sVersion versions.ValidK8sVersion, } } -// Load the embedded helm charts. -func (i *ChartLoader) Load(config *config.Config, conformanceMode bool, helmWaitMode helm.WaitMode, masterSecret, salt []byte) ([]byte, error) { - releases, err := i.LoadReleases(config, conformanceMode, helmWaitMode, masterSecret, salt) - if err != nil { - return nil, fmt.Errorf("loading releases: %w", err) - } - rel, err := json.Marshal(releases) - if err != nil { - return nil, err - } - return rel, nil -} - // LoadReleases loads the embedded helm charts and returns them as a HelmReleases object. -func (i *ChartLoader) LoadReleases(config *config.Config, conformanceMode bool, helmWaitMode helm.WaitMode, masterSecret, salt []byte) (*helm.Releases, error) { +func (i *ChartLoader) LoadReleases(config *config.Config, conformanceMode bool, helmWaitMode WaitMode, masterSecret, salt []byte) (*Releases, error) { ciliumRelease, err := i.loadRelease(ciliumInfo, helmWaitMode) if err != nil { return nil, fmt.Errorf("loading cilium: %w", err) @@ -147,7 +133,7 @@ func (i *ChartLoader) LoadReleases(config *config.Config, conformanceMode bool, return nil, fmt.Errorf("extending constellation-services values: %w", err) } - releases := helm.Releases{Cilium: ciliumRelease, CertManager: certManagerRelease, ConstellationOperators: operatorRelease, ConstellationServices: conServicesRelease} + releases := Releases{Cilium: ciliumRelease, CertManager: certManagerRelease, ConstellationOperators: operatorRelease, ConstellationServices: conServicesRelease} if config.HasProvider(cloudprovider.AWS) { awsRelease, err := i.loadRelease(awsLBControllerInfo, helmWaitMode) if err != nil { @@ -168,10 +154,10 @@ func (i *ChartLoader) LoadReleases(config *config.Config, conformanceMode bool, // loadRelease loads the embedded chart and values depending on the given info argument. // IMPORTANT: .helmignore rules specifying files in subdirectories are not applied (e.g. crds/kustomization.yaml). -func (i *ChartLoader) loadRelease(info chartInfo, helmWaitMode helm.WaitMode) (helm.Release, error) { +func (i *ChartLoader) loadRelease(info chartInfo, helmWaitMode WaitMode) (Release, error) { chart, err := loadChartsDir(helmFS, info.path) if err != nil { - return helm.Release{}, fmt.Errorf("loading %s chart: %w", info.releaseName, err) + return Release{}, fmt.Errorf("loading %s chart: %w", info.releaseName, err) } var values map[string]any @@ -181,7 +167,7 @@ func (i *ChartLoader) loadRelease(info chartInfo, helmWaitMode helm.WaitMode) (h var ok bool values, ok = ciliumVals[i.csp.String()] if !ok { - return helm.Release{}, fmt.Errorf("cilium values for csp %q not found", i.csp.String()) + return Release{}, fmt.Errorf("cilium values for csp %q not found", i.csp.String()) } case certManagerInfo.releaseName: values = i.loadCertManagerValues() @@ -200,10 +186,10 @@ func (i *ChartLoader) loadRelease(info chartInfo, helmWaitMode helm.WaitMode) (h chartRaw, err := i.marshalChart(chart) if err != nil { - return helm.Release{}, fmt.Errorf("packaging %s chart: %w", info.releaseName, err) + return Release{}, fmt.Errorf("packaging %s chart: %w", info.releaseName, err) } - return helm.Release{Chart: chartRaw, Values: values, ReleaseName: info.releaseName, WaitMode: helmWaitMode}, nil + return Release{Chart: chartRaw, Values: values, ReleaseName: info.releaseName, WaitMode: helmWaitMode}, nil } func (i *ChartLoader) loadAWSLBControllerValues() map[string]any { diff --git a/cli/internal/helm/loader_test.go b/cli/internal/helm/loader_test.go index 4df745319..b6950f7c0 100644 --- a/cli/internal/helm/loader_test.go +++ b/cli/internal/helm/loader_test.go @@ -8,7 +8,6 @@ package helm import ( "bytes" - "encoding/json" "fmt" "io/fs" "os" @@ -29,21 +28,15 @@ import ( "github.com/edgelesssys/constellation/v2/internal/attestation/measurements" "github.com/edgelesssys/constellation/v2/internal/cloud/cloudprovider" "github.com/edgelesssys/constellation/v2/internal/config" - "github.com/edgelesssys/constellation/v2/internal/deploy/helm" ) -// TestLoad checks if the serialized format that Load returns correctly preserves the dependencies of the loaded chart. -func TestLoad(t *testing.T) { +func TestLoadReleases(t *testing.T) { assert := assert.New(t) require := require.New(t) config := &config.Config{Provider: config.ProviderConfig{GCP: &config.GCPConfig{}}} chartLoader := ChartLoader{csp: config.GetProvider()} - release, err := chartLoader.Load(config, true, helm.WaitModeAtomic, []byte("secret"), []byte("salt")) - require.NoError(err) - - var helmReleases helm.Releases - err = json.Unmarshal(release, &helmReleases) + helmReleases, err := chartLoader.LoadReleases(config, true, WaitModeAtomic, []byte("secret"), []byte("salt")) require.NoError(err) reader := bytes.NewReader(helmReleases.ConstellationServices.Chart) chart, err := loader.LoadArchive(reader) diff --git a/internal/deploy/helm/helm.go b/cli/internal/helm/release.go similarity index 64% rename from internal/deploy/helm/helm.go rename to cli/internal/helm/release.go index dc2a09f38..ccbe7646d 100644 --- a/internal/deploy/helm/helm.go +++ b/cli/internal/helm/release.go @@ -25,28 +25,6 @@ type Releases struct { ConstellationServices Release } -// MergeMaps returns a new map that is the merger of it's inputs. -// Key collisions are resolved by taking the value of the second argument (map b). -// Taken from: https://github.com/helm/helm/blob/dbc6d8e20fe1d58d50e6ed30f09a04a77e4c68db/pkg/cli/values/options.go#L91-L108. -func MergeMaps(a, b map[string]any) map[string]any { - out := make(map[string]any, len(a)) - for k, v := range a { - out[k] = v - } - for k, v := range b { - if v, ok := v.(map[string]any); ok { - if bv, ok := out[k]; ok { - if bv, ok := bv.(map[string]any); ok { - out[k] = MergeMaps(bv, v) - continue - } - } - } - out[k] = v - } - return out -} - // WaitMode specifies the wait mode for a helm release. type WaitMode string diff --git a/cli/internal/helm/setup.go b/cli/internal/helm/setup.go deleted file mode 100644 index 209230a7e..000000000 --- a/cli/internal/helm/setup.go +++ /dev/null @@ -1,134 +0,0 @@ -/* -Copyright (c) Edgeless Systems GmbH - -SPDX-License-Identifier: AGPL-3.0-only -*/ - -package helm - -import ( - "context" - "encoding/base64" - "encoding/json" - "fmt" - - "github.com/edgelesssys/constellation/v2/cli/internal/terraform" - "github.com/edgelesssys/constellation/v2/internal/cloud/azureshared" - "github.com/edgelesssys/constellation/v2/internal/cloud/cloudprovider" - "github.com/edgelesssys/constellation/v2/internal/cloud/gcpshared" - "github.com/edgelesssys/constellation/v2/internal/constants" -) - -// setupMicroserviceVals returns the values for the microservice chart. -func setupMicroserviceVals(ctx context.Context, provider cloudprovider.Provider, measurementSalt []byte, uid, serviceAccURI string) (map[string]any, error) { - tfClient, err := terraform.New(ctx, constants.TerraformWorkingDir) - if err != nil { - return nil, fmt.Errorf("creating Terraform client: %w", err) - } - output, err := tfClient.ShowCluster(ctx, provider) - if err != nil { - return nil, fmt.Errorf("getting Terraform output: %w", err) - } - extraVals := map[string]any{ - "join-service": map[string]any{ - "measurementSalt": base64.StdEncoding.EncodeToString(measurementSalt), - }, - "verification-service": map[string]any{ - "loadBalancerIP": output.IP, - }, - "konnectivity": map[string]any{ - "loadBalancerIP": output.IP, - }, - } - switch provider { - case cloudprovider.GCP: - serviceAccountKey, err := gcpshared.ServiceAccountKeyFromURI(serviceAccURI) - if err != nil { - return nil, fmt.Errorf("getting service account key: %w", err) - } - rawKey, err := json.Marshal(serviceAccountKey) - if err != nil { - return nil, fmt.Errorf("marshaling service account key: %w", err) - } - if output.GCP == nil { - return nil, fmt.Errorf("no GCP output from Terraform") - } - extraVals["ccm"] = map[string]any{ - "GCP": map[string]any{ - "projectID": output.GCP.ProjectID, - "uid": uid, - "secretData": string(rawKey), - "subnetworkPodCIDR": output.GCP.IPCidrPod, - }, - } - case cloudprovider.Azure: - if output.Azure == nil { - return nil, fmt.Errorf("no Azure output from Terraform") - } - ccmConfig, err := getCCMConfig(*output.Azure, serviceAccURI) - if err != nil { - return nil, fmt.Errorf("getting Azure CCM config: %w", err) - } - extraVals["ccm"] = map[string]any{ - "Azure": map[string]any{ - "azureConfig": string(ccmConfig), - }, - } - } - - return extraVals, nil -} - -// setupOperatorVals returns the values for the constellation-operator chart. -func setupOperatorVals(_ context.Context, uid string) map[string]any { - return map[string]any{ - "constellation-operator": map[string]any{ - "constellationUID": uid, - }, - } -} - -type cloudConfig struct { - Cloud string `json:"cloud,omitempty"` - TenantID string `json:"tenantId,omitempty"` - SubscriptionID string `json:"subscriptionId,omitempty"` - ResourceGroup string `json:"resourceGroup,omitempty"` - Location string `json:"location,omitempty"` - SubnetName string `json:"subnetName,omitempty"` - SecurityGroupName string `json:"securityGroupName,omitempty"` - SecurityGroupResourceGroup string `json:"securityGroupResourceGroup,omitempty"` - LoadBalancerName string `json:"loadBalancerName,omitempty"` - LoadBalancerSku string `json:"loadBalancerSku,omitempty"` - VNetName string `json:"vnetName,omitempty"` - VNetResourceGroup string `json:"vnetResourceGroup,omitempty"` - CloudProviderBackoff bool `json:"cloudProviderBackoff,omitempty"` - UseInstanceMetadata bool `json:"useInstanceMetadata,omitempty"` - VMType string `json:"vmType,omitempty"` - UseManagedIdentityExtension bool `json:"useManagedIdentityExtension,omitempty"` - UserAssignedIdentityID string `json:"userAssignedIdentityID,omitempty"` -} - -// GetCCMConfig returns the configuration needed for the Kubernetes Cloud Controller Manager on Azure. -func getCCMConfig(tfOutput terraform.AzureApplyOutput, serviceAccURI string) ([]byte, error) { - creds, err := azureshared.ApplicationCredentialsFromURI(serviceAccURI) - if err != nil { - return nil, fmt.Errorf("getting service account key: %w", err) - } - useManagedIdentityExtension := creds.PreferredAuthMethod == azureshared.AuthMethodUserAssignedIdentity - config := cloudConfig{ - Cloud: "AzurePublicCloud", - TenantID: creds.TenantID, - SubscriptionID: tfOutput.SubscriptionID, - ResourceGroup: tfOutput.ResourceGroup, - LoadBalancerSku: "standard", - SecurityGroupName: tfOutput.NetworkSecurityGroupName, - LoadBalancerName: tfOutput.LoadBalancerName, - UseInstanceMetadata: true, - VMType: "vmss", - Location: creds.Location, - UseManagedIdentityExtension: useManagedIdentityExtension, - UserAssignedIdentityID: tfOutput.UserAssignedIdentity, - } - - return json.Marshal(config) -} diff --git a/cli/internal/helm/client.go b/cli/internal/helm/upgrade.go similarity index 91% rename from cli/internal/helm/client.go rename to cli/internal/helm/upgrade.go index f0200f531..88d679722 100644 --- a/cli/internal/helm/client.go +++ b/cli/internal/helm/upgrade.go @@ -13,20 +13,20 @@ import ( "strings" "time" + "helm.sh/helm/v3/pkg/action" + "helm.sh/helm/v3/pkg/chart" + "helm.sh/helm/v3/pkg/cli" + "helm.sh/helm/v3/pkg/release" + "github.com/edgelesssys/constellation/v2/cli/internal/clusterid" "github.com/edgelesssys/constellation/v2/internal/cloud/cloudprovider" "github.com/edgelesssys/constellation/v2/internal/compatibility" "github.com/edgelesssys/constellation/v2/internal/config" "github.com/edgelesssys/constellation/v2/internal/constants" - "github.com/edgelesssys/constellation/v2/internal/deploy/helm" "github.com/edgelesssys/constellation/v2/internal/file" "github.com/edgelesssys/constellation/v2/internal/semver" "github.com/edgelesssys/constellation/v2/internal/versions" "github.com/spf13/afero" - "helm.sh/helm/v3/pkg/action" - "helm.sh/helm/v3/pkg/chart" - "helm.sh/helm/v3/pkg/cli" - "helm.sh/helm/v3/pkg/release" apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime/schema" @@ -44,8 +44,8 @@ var ErrConfirmationMissing = errors.New("action requires user confirmation") var errReleaseNotFound = errors.New("release not found") -// Client handles interaction with helm and the cluster. -type Client struct { +// UpgradeClient handles interaction with helm and the cluster. +type UpgradeClient struct { config *action.Configuration kubectl crdClient fs file.Handler @@ -53,8 +53,8 @@ type Client struct { log debugLog } -// NewClient returns a new initializes client for the namespace Client. -func NewClient(client crdClient, kubeConfigPath, helmNamespace string, log debugLog) (*Client, error) { +// NewUpgradeClient returns a new initializes upgrade client for the given namespace. +func NewUpgradeClient(client crdClient, kubeConfigPath, helmNamespace string, log debugLog) (*UpgradeClient, error) { settings := cli.New() settings.KubeConfig = kubeConfigPath // constants.AdminConfFilename @@ -74,10 +74,10 @@ func NewClient(client crdClient, kubeConfigPath, helmNamespace string, log debug return nil, fmt.Errorf("initializing kubectl: %w", err) } - return &Client{kubectl: client, fs: fileHandler, actions: actions{config: actionConfig}, log: log}, nil + return &UpgradeClient{kubectl: client, fs: fileHandler, actions: actions{config: actionConfig}, log: log}, nil } -func (c *Client) shouldUpgrade(releaseName string, newVersion semver.Semver, force bool) error { +func (c *UpgradeClient) shouldUpgrade(releaseName string, newVersion semver.Semver, force bool) error { currentVersion, err := c.currentVersion(releaseName) if err != nil { return fmt.Errorf("getting version for %s: %w", releaseName, err) @@ -107,7 +107,7 @@ func (c *Client) shouldUpgrade(releaseName string, newVersion semver.Semver, for // Upgrade runs a helm-upgrade on all deployments that are managed via Helm. // If the CLI receives an interrupt signal it will cancel the context. // Canceling the context will prompt helm to abort and roll back the ongoing upgrade. -func (c *Client) Upgrade(ctx context.Context, config *config.Config, idFile clusterid.File, timeout time.Duration, allowDestructive, force bool, upgradeID string) error { +func (c *UpgradeClient) Upgrade(ctx context.Context, config *config.Config, idFile clusterid.File, timeout time.Duration, allowDestructive, force bool, upgradeID string) error { upgradeErrs := []error{} upgradeReleases := []*chart.Chart{} newReleases := []*chart.Chart{} @@ -206,7 +206,7 @@ func getManagedCharts(config *config.Config) []chartInfo { } // Versions queries the cluster for running versions and returns a map of releaseName -> version. -func (c *Client) Versions() (ServiceVersions, error) { +func (c *UpgradeClient) Versions() (ServiceVersions, error) { ciliumVersion, err := c.currentVersion(ciliumInfo.releaseName) if err != nil { return ServiceVersions{}, fmt.Errorf("getting %s version: %w", ciliumInfo.releaseName, err) @@ -246,7 +246,7 @@ func (c *Client) Versions() (ServiceVersions, error) { } // currentVersion returns the version of the currently installed helm release. -func (c *Client) currentVersion(release string) (semver.Semver, error) { +func (c *UpgradeClient) currentVersion(release string) (semver.Semver, error) { rel, err := c.actions.listAction(release) if err != nil { return semver.Semver{}, err @@ -266,7 +266,7 @@ func (c *Client) currentVersion(release string) (semver.Semver, error) { return semver.New(rel[0].Chart.Metadata.Version) } -func (c *Client) csiVersions() (map[string]semver.Semver, error) { +func (c *UpgradeClient) csiVersions() (map[string]semver.Semver, error) { packedChartRelease, err := c.actions.listAction(csiInfo.releaseName) if err != nil { return nil, fmt.Errorf("listing %s: %w", csiInfo.releaseName, err) @@ -299,7 +299,7 @@ func (c *Client) csiVersions() (map[string]semver.Semver, error) { } // installNewRelease installs a previously not installed release on the cluster. -func (c *Client) installNewRelease( +func (c *UpgradeClient) installNewRelease( ctx context.Context, timeout time.Duration, conf *config.Config, idFile clusterid.File, chart *chart.Chart, ) error { releaseName, values, err := c.loadUpgradeValues(ctx, conf, idFile, chart) @@ -310,7 +310,7 @@ func (c *Client) installNewRelease( } // upgradeRelease upgrades a release running on the cluster. -func (c *Client) upgradeRelease( +func (c *UpgradeClient) upgradeRelease( ctx context.Context, timeout time.Duration, conf *config.Config, idFile clusterid.File, chart *chart.Chart, ) error { releaseName, values, err := c.loadUpgradeValues(ctx, conf, idFile, chart) @@ -327,7 +327,7 @@ func (c *Client) upgradeRelease( } // loadUpgradeValues loads values for a chart required for running an upgrade. -func (c *Client) loadUpgradeValues(ctx context.Context, conf *config.Config, idFile clusterid.File, chart *chart.Chart, +func (c *UpgradeClient) loadUpgradeValues(ctx context.Context, conf *config.Config, idFile clusterid.File, chart *chart.Chart, ) (string, map[string]any, error) { // We need to load all values that can be statically loaded before merging them with the cluster // values. Otherwise the templates are not rendered correctly. @@ -385,7 +385,7 @@ func (c *Client) loadUpgradeValues(ctx context.Context, conf *config.Config, idF // applyMigrations checks the from version and applies the necessary migrations. // The function assumes the caller has verified that our version drift restriction is not violated, // Currently, this is done during config validation. -func (c *Client) applyMigrations(ctx context.Context, releaseName string, values map[string]any, conf *config.Config) error { +func (c *UpgradeClient) applyMigrations(ctx context.Context, releaseName string, values map[string]any, conf *config.Config) error { current, err := c.currentVersion(releaseName) if err != nil { return fmt.Errorf("getting %s version: %w", releaseName, err) @@ -413,7 +413,7 @@ func migrateFrom2_8(_ context.Context, _ map[string]any, _ *config.Config, _ crd // and merging the fetched values with the locally found values. // This is done to ensure that new values (from upgrades of the local files) end up in the cluster. // reuse-values does not ensure this. -func (c *Client) mergeClusterValues(localValues map[string]any, releaseName string) (map[string]any, error) { +func (c *UpgradeClient) mergeClusterValues(localValues map[string]any, releaseName string) (map[string]any, error) { // Ensure installCRDs is set for cert-manager chart. if releaseName == certManagerInfo.releaseName { localValues["installCRDs"] = true @@ -423,11 +423,11 @@ func (c *Client) mergeClusterValues(localValues map[string]any, releaseName stri return nil, fmt.Errorf("getting values for %s: %w", releaseName, err) } - return helm.MergeMaps(clusterValues, localValues), nil + return mergeMaps(clusterValues, localValues), nil } // GetValues queries the cluster for the values of the given release. -func (c *Client) GetValues(release string) (map[string]any, error) { +func (c *UpgradeClient) GetValues(release string) (map[string]any, error) { client := action.NewGetValues(c.config) // Version corresponds to the releases revision. Specifying a Version <= 0 yields the latest release. client.Version = 0 @@ -441,7 +441,7 @@ func (c *Client) GetValues(release string) (map[string]any, error) { // updateCRDs walks through the dependencies of the given chart and applies // the files in the dependencie's 'crds' folder. // This function is NOT recursive! -func (c *Client) updateCRDs(ctx context.Context, chart *chart.Chart) error { +func (c *UpgradeClient) updateCRDs(ctx context.Context, chart *chart.Chart) error { for _, dep := range chart.Dependencies() { for _, crdFile := range dep.Files { if strings.HasPrefix(crdFile.Name, "crds/") { @@ -456,11 +456,6 @@ func (c *Client) updateCRDs(ctx context.Context, chart *chart.Chart) error { return nil } -type debugLog interface { - Debugf(format string, args ...any) - Sync() -} - type crdClient interface { Initialize(kubeconfig []byte) error ApplyCRD(ctx context.Context, rawCRD []byte) error diff --git a/cli/internal/helm/client_test.go b/cli/internal/helm/upgrade_test.go similarity index 92% rename from cli/internal/helm/client_test.go rename to cli/internal/helm/upgrade_test.go index 285def10a..819371a5a 100644 --- a/cli/internal/helm/client_test.go +++ b/cli/internal/helm/upgrade_test.go @@ -46,7 +46,7 @@ func TestShouldUpgrade(t *testing.T) { assert := assert.New(t) require := require.New(t) - client := Client{kubectl: nil, actions: &stubActionWrapper{version: tc.version}, log: logger.NewTest(t)} + client := UpgradeClient{kubectl: nil, actions: &stubActionWrapper{version: tc.version}, log: logger.NewTest(t)} chart, err := loadChartsDir(helmFS, certManagerInfo.path) require.NoError(err) @@ -77,7 +77,7 @@ func TestUpgradeRelease(t *testing.T) { assert := assert.New(t) require := require.New(t) - client := Client{kubectl: nil, actions: &stubActionWrapper{version: tc.version}, log: logger.NewTest(t)} + client := UpgradeClient{kubectl: nil, actions: &stubActionWrapper{version: tc.version}, log: logger.NewTest(t)} chart, err := loadChartsDir(helmFS, certManagerInfo.path) require.NoError(err) diff --git a/cli/internal/kubernetes/upgrade.go b/cli/internal/kubernetes/upgrade.go index dfb9ade9b..c50ac9e4e 100644 --- a/cli/internal/kubernetes/upgrade.go +++ b/cli/internal/kubernetes/upgrade.go @@ -136,7 +136,7 @@ func NewUpgrader(ctx context.Context, outWriter io.Writer, fileHandler file.Hand } u.dynamicInterface = &NodeVersionClient{client: unstructuredClient} - helmClient, err := helm.NewClient(kubectl.New(), constants.AdminConfFilename, constants.HelmNamespace, log) + helmClient, err := helm.NewUpgradeClient(kubectl.New(), constants.AdminConfFilename, constants.HelmNamespace, log) if err != nil { return nil, fmt.Errorf("setting up helm client: %w", err) } diff --git a/internal/deploy/helm/BUILD.bazel b/internal/deploy/helm/BUILD.bazel deleted file mode 100644 index 5cb17b315..000000000 --- a/internal/deploy/helm/BUILD.bazel +++ /dev/null @@ -1,28 +0,0 @@ -load("@io_bazel_rules_go//go:def.bzl", "go_library") -load("//bazel/go:go_test.bzl", "go_test") - -go_library( - name = "helm", - srcs = [ - "helm.go", - "install.go", - ], - importpath = "github.com/edgelesssys/constellation/v2/internal/deploy/helm", - visibility = ["//:__subpackages__"], - deps = [ - "//internal/constants", - "//internal/retry", - "@io_k8s_apimachinery//pkg/util/wait", - "@sh_helm_helm_v3//pkg/action", - "@sh_helm_helm_v3//pkg/chart", - "@sh_helm_helm_v3//pkg/chart/loader", - "@sh_helm_helm_v3//pkg/cli", - ], -) - -go_test( - name = "helm_test", - srcs = ["helm_test.go"], - embed = [":helm"], - deps = ["@com_github_stretchr_testify//assert"], -) diff --git a/internal/kubernetes/kubectl/BUILD.bazel b/internal/kubernetes/kubectl/BUILD.bazel index 9af1ccb90..51cfe9e90 100644 --- a/internal/kubernetes/kubectl/BUILD.bazel +++ b/internal/kubernetes/kubectl/BUILD.bazel @@ -12,9 +12,11 @@ go_library( "@io_k8s_apiextensions_apiserver//pkg/client/clientset/clientset/typed/apiextensions/v1:apiextensions", "@io_k8s_apimachinery//pkg/apis/meta/v1:meta", "@io_k8s_apimachinery//pkg/apis/meta/v1/unstructured", + "@io_k8s_apimachinery//pkg/labels", "@io_k8s_apimachinery//pkg/runtime", "@io_k8s_apimachinery//pkg/runtime/schema", "@io_k8s_apimachinery//pkg/runtime/serializer", + "@io_k8s_apimachinery//pkg/types", "@io_k8s_client_go//dynamic", "@io_k8s_client_go//kubernetes", "@io_k8s_client_go//scale/scheme", diff --git a/internal/kubernetes/kubectl/kubectl.go b/internal/kubernetes/kubectl/kubectl.go index bfafd9492..04f2ee224 100644 --- a/internal/kubernetes/kubectl/kubectl.go +++ b/internal/kubernetes/kubectl/kubectl.go @@ -19,9 +19,11 @@ import ( apiextensionsclientv1 "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset/typed/apiextensions/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/runtime/serializer" + "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/dynamic" "k8s.io/client-go/kubernetes" "k8s.io/client-go/scale/scheme" @@ -130,6 +132,22 @@ func (k *Kubectl) AnnotateNode(ctx context.Context, nodeName, annotationKey, ann }) } +// PatchFirstNodePodCIDR patches the firstNodePodCIDR of the first control-plane node for Cilium. +func (k *Kubectl) PatchFirstNodePodCIDR(ctx context.Context, firstNodePodCIDR string) error { + selector := labels.Set{"node-role.kubernetes.io/control-plane": ""}.AsSelector() + controlPlaneList, err := k.CoreV1().Nodes().List(ctx, metav1.ListOptions{LabelSelector: selector.String()}) + if err != nil { + return err + } + if len(controlPlaneList.Items) != 1 { + return fmt.Errorf("expected 1 control-plane node, got %d", len(controlPlaneList.Items)) + } + nodeName := controlPlaneList.Items[0].Name + // Update the node's spec + _, err = k.CoreV1().Nodes().Patch(context.Background(), nodeName, types.MergePatchType, []byte(fmt.Sprintf(`{"spec":{"podCIDR":"%s"}}`, firstNodePodCIDR)), metav1.PatchOptions{}) + return err +} + // ListAllNamespaces returns all namespaces in the cluster. func (k *Kubectl) ListAllNamespaces(ctx context.Context) (*corev1.NamespaceList, error) { return k.CoreV1().Namespaces().List(ctx, metav1.ListOptions{})