Git-Mirrors/constellation
1
0
Fork 0
mirror of https://github.com/edgelesssys/constellation.git synced 2025-02-02 02:24:56 -05:00
Code Issues Packages Projects Releases Wiki Activity

helm: remove konnectivity from control-planes

Browse Source 
This is the first step in our migration off of
konnectivity. Before node-to-node encryption
we used konnectivity to route some KubeAPI
to kubelet traffic over the pod network which then
would be encrypted.

Since we enabled node-to-node encryption this has no
security upsides anymore. Note that we still deploy
the konnectivity agents via helm and still have the
load balancer for konnectivity.

In the following releases we will remove both.
This commit is contained in:
Leonard Cohnen 2023-11-15 12:38:01 +01:00 committed by 3u13r
parent 79f562374a
commit cfcc0898b2
10 changed files with 23 additions and 374 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
bootstrapper/internal/kubernetes/k8sapi/BUILD.bazel
View File

@ -20,7 +20,6 @@ go_library(
"//internal/installer",
"//internal/kubernetes",
"//internal/logger",
"//internal/role",
"//internal/versions/components",
"@com_github_coreos_go_systemd_v22//dbus",
"@com_github_spf13_afero//:afero",

123
bootstrapper/internal/kubernetes/k8sapi/k8sutil.go
View File

@ -17,14 +17,11 @@ import (
"os"
"os/exec"
"path/filepath"
"strconv"
"strings"
"time"
"github.com/edgelesssys/constellation/v2/bootstrapper/internal/certificate"
"github.com/edgelesssys/constellation/v2/bootstrapper/internal/kubernetes/k8sapi/resources"
"github.com/edgelesssys/constellation/v2/internal/constants"
"github.com/edgelesssys/constellation/v2/internal/role"
"github.com/edgelesssys/constellation/v2/internal/versions/components"
corev1 "k8s.io/api/core/v1"
"k8s.io/apiserver/pkg/authentication/user"
@ -90,7 +87,7 @@ func (k *KubernetesUtil) InstallComponents(ctx context.Context, kubernetesCompon
// InitCluster instruments kubeadm to initialize the K8s cluster.
// On success an admin kubeconfig file is returned.
func (k *KubernetesUtil) InitCluster(
ctx context.Context, initConfig []byte, nodeName, clusterName string, ips []net.IP, controlPlaneHost, controlPlanePort string, conformanceMode bool, log *logger.Logger,
ctx context.Context, initConfig []byte, nodeName, clusterName string, ips []net.IP, conformanceMode bool, log *logger.Logger,
) ([]byte, error) {
// TODO(3u13r): audit policy should be user input
auditPolicy, err := resources.NewDefaultAuditPolicy().Marshal()
@ -146,12 +143,6 @@ func (k *KubernetesUtil) InitCluster(
return nil, fmt.Errorf("creating static pods directory: %w", err)
}
log.Infof("Preparing node for Konnectivity")
controlPlaneEndpoint := net.JoinHostPort(controlPlaneHost, controlPlanePort)
if err := k.prepareControlPlaneForKonnectivity(ctx, controlPlaneEndpoint); err != nil {
return nil, fmt.Errorf("setup konnectivity: %w", err)
}
// initialize the cluster
log.Infof("Initializing the cluster using kubeadm init")
skipPhases := "--skip-phases=preflight,certs"
@ -189,56 +180,6 @@ func (k *KubernetesUtil) InitCluster(
return out, nil
}
func (k *KubernetesUtil) prepareControlPlaneForKonnectivity(ctx context.Context, loadBalancerEndpoint string) error {
if !strings.Contains(loadBalancerEndpoint, ":") {
loadBalancerEndpoint = net.JoinHostPort(loadBalancerEndpoint, strconv.Itoa(constants.KubernetesPort))
}
konnectivityServerYaml, err := resources.NewKonnectivityServerStaticPod().Marshal()
if err != nil {
return fmt.Errorf("generating konnectivity server static pod: %w", err)
}
if err := os.WriteFile("/etc/kubernetes/manifests/konnectivity-server.yaml", konnectivityServerYaml, 0o644); err != nil {
return fmt.Errorf("writing konnectivity server pod: %w", err)
}
egressConfigYaml, err := resources.NewEgressSelectorConfiguration().Marshal()
if err != nil {
return fmt.Errorf("generating egress selector configuration: %w", err)
}
if err := os.WriteFile("/etc/kubernetes/egress-selector-configuration.yaml", egressConfigYaml, 0o644); err != nil {
return fmt.Errorf("writing egress selector config: %w", err)
}
if err := k.createSignedKonnectivityCert(); err != nil {
return fmt.Errorf("generating konnectivity server certificate: %w", err)
}
if out, err := exec.CommandContext(ctx, constants.KubectlPath, "config", "set-credentials", "--kubeconfig", "/etc/kubernetes/konnectivity-server.conf", "system:konnectivity-server",
"--client-certificate", "/etc/kubernetes/konnectivity.crt", "--client-key", "/etc/kubernetes/konnectivity.key", "--embed-certs=true").CombinedOutput(); err != nil {
return fmt.Errorf("konnectivity kubeconfig set-credentials: %w, %s", err, string(out))
}
if out, err := exec.CommandContext(ctx, constants.KubectlPath, "--kubeconfig", "/etc/kubernetes/konnectivity-server.conf", "config", "set-cluster", "kubernetes", "--server", "https://"+loadBalancerEndpoint,
"--certificate-authority", "/etc/kubernetes/pki/ca.crt", "--embed-certs=true").CombinedOutput(); err != nil {
return fmt.Errorf("konnectivity kubeconfig set-cluster: %w, %s", err, string(out))
}
if out, err := exec.CommandContext(ctx, constants.KubectlPath, "--kubeconfig", "/etc/kubernetes/konnectivity-server.conf", "config", "set-context", "system:konnectivity-server@kubernetes",
"--cluster", "kubernetes", "--user", "system:konnectivity-server").CombinedOutput(); err != nil {
return fmt.Errorf("konnectivity kubeconfig set-context: %w, %s", err, string(out))
}
if out, err := exec.CommandContext(ctx, constants.KubectlPath, "--kubeconfig", "/etc/kubernetes/konnectivity-server.conf", "config", "use-context", "system:konnectivity-server@kubernetes").CombinedOutput(); err != nil {
return fmt.Errorf("konnectivity kubeconfig use-context: %w, %s", err, string(out))
}
// cleanup
if err := os.Remove("/etc/kubernetes/konnectivity.crt"); err != nil {
return fmt.Errorf("removing konnectivity certificate: %w", err)
}
if err := os.Remove("/etc/kubernetes/konnectivity.key"); err != nil {
return fmt.Errorf("removing konnectivity key: %w", err)
}
return nil
}
// SetupPodNetworkInput holds all configuration options to setup the pod network.
type SetupPodNetworkInput struct {
CloudProvider string
@ -250,7 +191,7 @@ type SetupPodNetworkInput struct {
}
// JoinCluster joins existing Kubernetes cluster using kubeadm join.
func (k *KubernetesUtil) JoinCluster(ctx context.Context, joinConfig []byte, peerRole role.Role, controlPlaneHost, controlPlanePort string, log *logger.Logger) error {
func (k *KubernetesUtil) JoinCluster(ctx context.Context, joinConfig []byte, log *logger.Logger) error {
// TODO(3u13r): audit policy should be user input
auditPolicy, err := resources.NewDefaultAuditPolicy().Marshal()
if err != nil {
@ -275,14 +216,6 @@ func (k *KubernetesUtil) JoinCluster(ctx context.Context, joinConfig []byte, pee
return fmt.Errorf("creating static pods directory: %w", err)
}
if peerRole == role.ControlPlane {
log.Infof("Prep Init Kubernetes cluster")
controlPlaneEndpoint := net.JoinHostPort(controlPlaneHost, controlPlanePort)
if err := k.prepareControlPlaneForKonnectivity(ctx, controlPlaneEndpoint); err != nil {
return fmt.Errorf("setup konnectivity: %w", err)
}
}
// run `kubeadm join` to join a worker node to an existing Kubernetes cluster
cmd := exec.CommandContext(ctx, constants.KubeadmPath, "join", "-v=5", "--config", joinConfigFile.Name())
out, err := cmd.CombinedOutput()
@ -369,58 +302,6 @@ func (k *KubernetesUtil) createSignedKubeletCert(nodeName string, ips []net.IP)
return k.file.Write(certificate.CertificateFilename, kubeletCert, file.OptMkdirAll)
}
// createSignedKonnectivityCert manually creates a Kubernetes CA signed certificate for the Konnectivity server.
func (k *KubernetesUtil) createSignedKonnectivityCert() error {
// Create CSR
certRequestRaw, keyPem, err := resources.GetKonnectivityCertificateRequest()
if err != nil {
return err
}
if err := k.file.Write(resources.KonnectivityKeyFilename, keyPem, file.OptMkdirAll); err != nil {
return err
}
certRequest, err := x509.ParseCertificateRequest(certRequestRaw)
if err != nil {
return err
}
// Prepare certificate signing
serialNumber, err := crypto.GenerateCertificateSerialNumber()
if err != nil {
return err
}
now := time.Now()
// Create the kubelet certificate
// For a reference on the certificate fields, see: https://kubernetes.io/docs/setup/best-practices/certificates/
certTmpl := &x509.Certificate{
SerialNumber: serialNumber,
NotBefore: now.Add(-2 * time.Hour),
NotAfter: now.Add(24 * 365 * time.Hour),
Subject: certRequest.Subject,
}
parentCert, parentKey, err := k.getKubernetesCACertAndKey()
if err != nil {
return err
}
// Sign the certificate
certRaw, err := x509.CreateCertificate(rand.Reader, certTmpl, parentCert, certRequest.PublicKey, parentKey)
if err != nil {
return err
}
// Write the certificate
konnectivityCert := pem.EncodeToMemory(&pem.Block{
Type: "CERTIFICATE",
Bytes: certRaw,
})
return k.file.Write(resources.KonnectivityCertificateFilename, konnectivityCert, file.OptMkdirAll)
}
// getKubernetesCACertAndKey returns the Kubernetes CA certificate and key.
// The key of type `any` can be consumed by `x509.CreateCertificate()`.
func (k *KubernetesUtil) getKubernetesCACertAndKey() (*x509.Certificate, any, error) {

21
bootstrapper/internal/kubernetes/k8sapi/kubeadm_config.go
View File

@ -71,10 +71,9 @@ func (c *KubdeadmConfiguration) InitConfiguration(externalCloudProvider bool, cl
"audit-log-path": filepath.Join(auditLogDir, auditLogFile), // CIS benchmark
"audit-log-maxage": "30", // CIS benchmark - Default value of Rancher
// log size = 10 files * 100MB + 100 MB (which is currently being written) = 1.1GB
"audit-log-maxbackup": "10", // CIS benchmark - Default value of Rancher
"audit-log-maxsize": "100", // CIS benchmark - Default value of Rancher
"profiling": "false", // CIS benchmark
"egress-selector-config-file": "/etc/kubernetes/egress-selector-configuration.yaml",
"audit-log-maxbackup": "10", // CIS benchmark - Default value of Rancher
"audit-log-maxsize": "100", // CIS benchmark - Default value of Rancher
"profiling": "false", // CIS benchmark
"kubelet-certificate-authority": filepath.Join(
kubeconstants.KubernetesDir,
kubeconstants.DefaultCertificateDir,
@ -104,20 +103,6 @@ func (c *KubdeadmConfiguration) InitConfiguration(externalCloudProvider bool, cl
ReadOnly: true,
PathType: corev1.HostPathFile,
},
{
Name: "egress-config",
HostPath: "/etc/kubernetes/egress-selector-configuration.yaml",
MountPath: "/etc/kubernetes/egress-selector-configuration.yaml",
ReadOnly: true,
PathType: corev1.HostPathFile,
},
{
Name: "konnectivity-uds",
HostPath: "/run/konnectivity-server",
MountPath: "/run/konnectivity-server",
ReadOnly: false,
PathType: corev1.HostPathDirectoryOrCreate,
},
},
},
CertSANs: []string{"127.0.0.1"},

6
bootstrapper/internal/kubernetes/k8sapi/resources/BUILD.bazel
View File

@ -5,19 +5,13 @@ go_library(
name = "resources",
srcs = [
"auditpolicy.go",
"konnectivity.go",
"resources.go",
],
importpath = "github.com/edgelesssys/constellation/v2/bootstrapper/internal/kubernetes/k8sapi/resources",
visibility = ["//bootstrapper:__subpackages__"],
deps = [
"//bootstrapper/internal/certificate",
"//internal/kubernetes",
"//internal/versions",
"@io_k8s_api//core/v1:core",
"@io_k8s_apimachinery//pkg/apis/meta/v1:meta",
"@io_k8s_apimachinery//pkg/util/intstr",
"@io_k8s_apiserver//pkg/apis/apiserver",
"@io_k8s_apiserver//pkg/apis/audit/v1:audit",
],
)

205
bootstrapper/internal/kubernetes/k8sapi/resources/konnectivity.go
View File

@ -1,205 +0,0 @@
/*
Copyright (c) Edgeless Systems GmbH
SPDX-License-Identifier: AGPL-3.0-only
*/
package resources
import (
"crypto/x509"
"crypto/x509/pkix"
"github.com/edgelesssys/constellation/v2/bootstrapper/internal/certificate"
"github.com/edgelesssys/constellation/v2/internal/kubernetes"
"github.com/edgelesssys/constellation/v2/internal/versions"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/intstr"
"k8s.io/apiserver/pkg/apis/apiserver"
)
const (
// KonnectivityCertificateFilename is the path to the kubelets certificate.
KonnectivityCertificateFilename = "/etc/kubernetes/konnectivity.crt"
// KonnectivityKeyFilename is the path to the kubelets private key.
KonnectivityKeyFilename = "/etc/kubernetes/konnectivity.key"
)
// KonnectivityServerStaticPod deployment.
type KonnectivityServerStaticPod struct {
StaticPod corev1.Pod
}
// EgressSelectorConfiguration deployment.
type EgressSelectorConfiguration struct {
EgressSelectorConfiguration apiserver.EgressSelectorConfiguration
}
// NewKonnectivityServerStaticPod create a new KonnectivityServerStaticPod.
func NewKonnectivityServerStaticPod() *KonnectivityServerStaticPod {
udsHostPathType := corev1.HostPathDirectoryOrCreate
return &KonnectivityServerStaticPod{
StaticPod: corev1.Pod{
TypeMeta: metav1.TypeMeta{
APIVersion: "v1",
Kind: "Pod",
},
ObjectMeta: metav1.ObjectMeta{
Name: "konnectivity-server",
Namespace: "kube-system",
},
Spec: corev1.PodSpec{
PriorityClassName: "system-cluster-critical",
HostNetwork: true,
Containers: []corev1.Container{
{
Name: "konnectivity-server-container",
Image: versions.KonnectivityServerImage,
Command: []string{"/proxy-server"},
Args: []string{
"--logtostderr=true",
// This needs to be consistent with the value set in egressSelectorConfiguration.
"--uds-name=/run/konnectivity-server/konnectivity-server.socket",
// Clean up existing UDS file before starting the server in case the server crashed at some point.
"--delete-existing-uds-file=true",
// The following two lines assume the Konnectivity server is
// deployed on the same machine as the apiserver, and the certs and
// key of the API Server are at the specified location.
"--cluster-cert=/etc/kubernetes/pki/apiserver.crt",
"--cluster-key=/etc/kubernetes/pki/apiserver.key",
// This needs to be consistent with the value set in egressSelectorConfiguration.
"--mode=grpc",
"--server-port=0",
"--agent-port=8132",
"--admin-port=8133",
"--health-port=8134",
"--v=5",
"--agent-namespace=kube-system",
"--agent-service-account=konnectivity-agent",
"--kubeconfig=/etc/kubernetes/konnectivity-server.conf",
"--authentication-audience=system:konnectivity-server",
"--proxy-strategies=default",
},
LivenessProbe: &corev1.Probe{
ProbeHandler: corev1.ProbeHandler{
HTTPGet: &corev1.HTTPGetAction{
Path: "/healthz",
Port: intstr.FromInt(8134),
},
},
InitialDelaySeconds: 30,
TimeoutSeconds: 60,
},
Ports: []corev1.ContainerPort{
{
Name: "agent-port",
ContainerPort: 8132,
HostPort: 8132,
},
{
Name: "admin-port",
ContainerPort: 8133,
HostPort: 8133,
},
{
Name: "health-port",
ContainerPort: 8134,
HostPort: 8134,
},
},
VolumeMounts: []corev1.VolumeMount{
{
Name: "k8s-certs",
MountPath: "/etc/kubernetes/pki",
ReadOnly: true,
},
{
Name: "kubeconfig",
MountPath: "/etc/kubernetes/konnectivity-server.conf",
ReadOnly: true,
},
{
Name: "konnectivity-uds",
MountPath: "/run/konnectivity-server",
ReadOnly: false,
},
},
},
},
Volumes: []corev1.Volume{
{
Name: "k8s-certs",
VolumeSource: corev1.VolumeSource{
HostPath: &corev1.HostPathVolumeSource{
Path: "/etc/kubernetes/pki",
},
},
},
{
Name: "kubeconfig",
VolumeSource: corev1.VolumeSource{
HostPath: &corev1.HostPathVolumeSource{
Path: "/etc/kubernetes/konnectivity-server.conf",
},
},
},
{
Name: "konnectivity-uds",
VolumeSource: corev1.VolumeSource{
HostPath: &corev1.HostPathVolumeSource{
Path: "/run/konnectivity-server",
Type: &udsHostPathType,
},
},
},
},
},
},
}
}
// NewEgressSelectorConfiguration creates a new EgressSelectorConfiguration.
func NewEgressSelectorConfiguration() *EgressSelectorConfiguration {
return &EgressSelectorConfiguration{
EgressSelectorConfiguration: apiserver.EgressSelectorConfiguration{
TypeMeta: metav1.TypeMeta{
APIVersion: "apiserver.k8s.io/v1beta1",
Kind: "EgressSelectorConfiguration",
},
EgressSelections: []apiserver.EgressSelection{
{
Name: "cluster",
Connection: apiserver.Connection{
ProxyProtocol: "GRPC",
Transport: &apiserver.Transport{
UDS: &apiserver.UDSTransport{
UDSName: "/run/konnectivity-server/konnectivity-server.socket",
},
},
},
},
},
},
}
}
// Marshal to Kubernetes YAML.
func (v *KonnectivityServerStaticPod) Marshal() ([]byte, error) {
return kubernetes.MarshalK8SResources(v)
}
// Marshal to Kubernetes YAML.
func (v *EgressSelectorConfiguration) Marshal() ([]byte, error) {
return kubernetes.MarshalK8SResources(v)
}
// GetKonnectivityCertificateRequest returns a certificate request and matching private key for the konnectivity server.
func GetKonnectivityCertificateRequest() (certificateRequest []byte, privateKey []byte, err error) {
csrTemplate := &x509.CertificateRequest{
Subject: pkix.Name{
CommonName: "system:konnectivity-server",
},
}
return certificate.GetCertificateRequest(csrTemplate)
}

5
bootstrapper/internal/kubernetes/k8sutil.go
View File

@ -11,13 +11,12 @@ import (
"net"
"github.com/edgelesssys/constellation/v2/internal/logger"
"github.com/edgelesssys/constellation/v2/internal/role"
"github.com/edgelesssys/constellation/v2/internal/versions/components"
)
type clusterUtil interface {
InstallComponents(ctx context.Context, kubernetesComponents components.Components) error
InitCluster(ctx context.Context, initConfig []byte, nodeName, clusterName string, ips []net.IP, controlPlaneHost, controlPlanePort string, conformanceMode bool, log *logger.Logger) ([]byte, error)
JoinCluster(ctx context.Context, joinConfig []byte, peerRole role.Role, controlPlaneHost, controlPlanePort string, log *logger.Logger) error
InitCluster(ctx context.Context, initConfig []byte, nodeName, clusterName string, ips []net.IP, conformanceMode bool, log *logger.Logger) ([]byte, error)
JoinCluster(ctx context.Context, joinConfig []byte, log *logger.Logger) error
StartKubelet() error
}

4
bootstrapper/internal/kubernetes/kubernetes.go
View File

@ -133,7 +133,7 @@ func (k *KubeWrapper) InitCluster(
return nil, fmt.Errorf("encoding kubeadm init configuration as YAML: %w", err)
}
log.Infof("Initializing Kubernetes cluster")
kubeConfig, err := k.clusterUtil.InitCluster(ctx, initConfigYAML, nodeName, clusterName, validIPs, controlPlaneHost, controlPlanePort, conformanceMode, log)
kubeConfig, err := k.clusterUtil.InitCluster(ctx, initConfigYAML, nodeName, clusterName, validIPs, conformanceMode, log)
if err != nil {
return nil, fmt.Errorf("kubeadm init: %w", err)
}
@ -238,7 +238,7 @@ func (k *KubeWrapper) JoinCluster(ctx context.Context, args *kubeadm.BootstrapTo
return fmt.Errorf("encoding kubeadm join configuration as YAML: %w", err)
}
log.With(zap.String("apiServerEndpoint", args.APIServerEndpoint)).Infof("Joining Kubernetes cluster")
if err := k.clusterUtil.JoinCluster(ctx, joinConfigYAML, peerRole, loadBalancerHost, loadBalancerPort, log); err != nil {
if err := k.clusterUtil.JoinCluster(ctx, joinConfigYAML, log); err != nil {
return fmt.Errorf("joining cluster: %v; %w ", string(joinConfigYAML), err)
}

27
bootstrapper/internal/kubernetes/kubernetes_test.go
View File

@ -420,16 +420,15 @@ func TestK8sCompliantHostname(t *testing.T) {
}
type stubClusterUtil struct {
installComponentsErr error
initClusterErr error
setupAutoscalingError error
setupKonnectivityError error
setupGCPGuestAgentErr error
setupOLMErr error
setupNMOErr error
setupNodeOperatorErr error
joinClusterErr error
startKubeletErr error
installComponentsErr error
initClusterErr error
setupAutoscalingError error
setupGCPGuestAgentErr error
setupOLMErr error
setupNMOErr error
setupNodeOperatorErr error
joinClusterErr error
startKubeletErr error
kubeconfig []byte
@ -437,15 +436,11 @@ type stubClusterUtil struct {
joinConfigs [][]byte
}
func (s *stubClusterUtil) SetupKonnectivity(_ k8sapi.Client, _ kubernetes.Marshaler) error {
return s.setupKonnectivityError
}
func (s *stubClusterUtil) InstallComponents(_ context.Context, _ components.Components) error {
return s.installComponentsErr
}
func (s *stubClusterUtil) InitCluster(_ context.Context, initConfig []byte, _, _ string, _ []net.IP, _, _ string, _ bool, _ *logger.Logger) ([]byte, error) {
func (s *stubClusterUtil) InitCluster(_ context.Context, initConfig []byte, _, _ string, _ []net.IP, _ bool, _ *logger.Logger) ([]byte, error) {
s.initConfigs = append(s.initConfigs, initConfig)
return s.kubeconfig, s.initClusterErr
}
@ -470,7 +465,7 @@ func (s *stubClusterUtil) SetupNodeOperator(_ context.Context, _ k8sapi.Client,
return s.setupNodeOperatorErr
}
func (s *stubClusterUtil) JoinCluster(_ context.Context, joinConfig []byte, _ role.Role, _, _ string, _ *logger.Logger) error {
func (s *stubClusterUtil) JoinCluster(_ context.Context, joinConfig []byte, _ *logger.Logger) error {
s.joinConfigs = append(s.joinConfigs, joinConfig)
return s.joinClusterErr
}

3
cli/internal/helm/overrides.go
View File

@ -73,6 +73,9 @@ func extraConstellationServicesValues(
extraVals["verification-service"] = map[string]any{
"attestationVariant": cfg.GetAttestationConfig().GetVariant().String(),
}
extraVals["konnectivity"] = map[string]any{
"loadBalancerIP": output.ClusterEndpoint,
}
extraVals["key-service"] = map[string]any{
"masterSecret": base64.StdEncoding.EncodeToString(masterSecret.Key),

2
internal/constants/constants.go
View File

@ -67,8 +67,6 @@ const (
RecoveryPort = 9999
// DebugdPort port for debugd process.
DebugdPort = 4000
// KonnectivityPort port for konnectivity k8s service.
KonnectivityPort = 8132
//
// Filenames.