mirror of
https://github.com/edgelesssys/constellation.git
synced 2025-05-07 16:55:15 -04:00
helm: remove konnectivity from control-planes
This is the first step in our migration off of konnectivity. Before node-to-node encryption we used konnectivity to route some KubeAPI to kubelet traffic over the pod network which then would be encrypted. Since we enabled node-to-node encryption this has no security upsides anymore. Note that we still deploy the konnectivity agents via helm and still have the load balancer for konnectivity. In the following releases we will remove both.
This commit is contained in:
Leonard Cohnen
3u13r
parent
79f562374a
commit
cfcc0898b2
10 changed files with 23 additions and 374 deletions
|
|
@ -17,14 +17,11 @@ import (
|
|||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/edgelesssys/constellation/v2/bootstrapper/internal/certificate"
|
||||
"github.com/edgelesssys/constellation/v2/bootstrapper/internal/kubernetes/k8sapi/resources"
|
||||
"github.com/edgelesssys/constellation/v2/internal/constants"
|
||||
"github.com/edgelesssys/constellation/v2/internal/role"
|
||||
"github.com/edgelesssys/constellation/v2/internal/versions/components"
|
||||
corev1 "k8s.io/api/core/v1"
|
||||
"k8s.io/apiserver/pkg/authentication/user"
|
||||
|
|
@ -90,7 +87,7 @@ func (k *KubernetesUtil) InstallComponents(ctx context.Context, kubernetesCompon
|
|||
// InitCluster instruments kubeadm to initialize the K8s cluster.
|
||||
// On success an admin kubeconfig file is returned.
|
||||
func (k *KubernetesUtil) InitCluster(
|
||||
ctx context.Context, initConfig []byte, nodeName, clusterName string, ips []net.IP, controlPlaneHost, controlPlanePort string, conformanceMode bool, log *logger.Logger,
|
||||
ctx context.Context, initConfig []byte, nodeName, clusterName string, ips []net.IP, conformanceMode bool, log *logger.Logger,
|
||||
) ([]byte, error) {
|
||||
// TODO(3u13r): audit policy should be user input
|
||||
auditPolicy, err := resources.NewDefaultAuditPolicy().Marshal()
|
||||
|
|
@ -146,12 +143,6 @@ func (k *KubernetesUtil) InitCluster(
|
|||
return nil, fmt.Errorf("creating static pods directory: %w", err)
|
||||
}
|
||||
|
||||
log.Infof("Preparing node for Konnectivity")
|
||||
controlPlaneEndpoint := net.JoinHostPort(controlPlaneHost, controlPlanePort)
|
||||
if err := k.prepareControlPlaneForKonnectivity(ctx, controlPlaneEndpoint); err != nil {
|
||||
return nil, fmt.Errorf("setup konnectivity: %w", err)
|
||||
}
|
||||
|
||||
// initialize the cluster
|
||||
log.Infof("Initializing the cluster using kubeadm init")
|
||||
skipPhases := "--skip-phases=preflight,certs"
|
||||
|
|
@ -189,56 +180,6 @@ func (k *KubernetesUtil) InitCluster(
|
|||
return out, nil
|
||||
}
|
||||
|
||||
func (k *KubernetesUtil) prepareControlPlaneForKonnectivity(ctx context.Context, loadBalancerEndpoint string) error {
|
||||
if !strings.Contains(loadBalancerEndpoint, ":") {
|
||||
loadBalancerEndpoint = net.JoinHostPort(loadBalancerEndpoint, strconv.Itoa(constants.KubernetesPort))
|
||||
}
|
||||
|
||||
konnectivityServerYaml, err := resources.NewKonnectivityServerStaticPod().Marshal()
|
||||
if err != nil {
|
||||
return fmt.Errorf("generating konnectivity server static pod: %w", err)
|
||||
}
|
||||
if err := os.WriteFile("/etc/kubernetes/manifests/konnectivity-server.yaml", konnectivityServerYaml, 0o644); err != nil {
|
||||
return fmt.Errorf("writing konnectivity server pod: %w", err)
|
||||
}
|
||||
|
||||
egressConfigYaml, err := resources.NewEgressSelectorConfiguration().Marshal()
|
||||
if err != nil {
|
||||
return fmt.Errorf("generating egress selector configuration: %w", err)
|
||||
}
|
||||
if err := os.WriteFile("/etc/kubernetes/egress-selector-configuration.yaml", egressConfigYaml, 0o644); err != nil {
|
||||
return fmt.Errorf("writing egress selector config: %w", err)
|
||||
}
|
||||
|
||||
if err := k.createSignedKonnectivityCert(); err != nil {
|
||||
return fmt.Errorf("generating konnectivity server certificate: %w", err)
|
||||
}
|
||||
|
||||
if out, err := exec.CommandContext(ctx, constants.KubectlPath, "config", "set-credentials", "--kubeconfig", "/etc/kubernetes/konnectivity-server.conf", "system:konnectivity-server",
|
||||
"--client-certificate", "/etc/kubernetes/konnectivity.crt", "--client-key", "/etc/kubernetes/konnectivity.key", "--embed-certs=true").CombinedOutput(); err != nil {
|
||||
return fmt.Errorf("konnectivity kubeconfig set-credentials: %w, %s", err, string(out))
|
||||
}
|
||||
if out, err := exec.CommandContext(ctx, constants.KubectlPath, "--kubeconfig", "/etc/kubernetes/konnectivity-server.conf", "config", "set-cluster", "kubernetes", "--server", "https://"+loadBalancerEndpoint,
|
||||
"--certificate-authority", "/etc/kubernetes/pki/ca.crt", "--embed-certs=true").CombinedOutput(); err != nil {
|
||||
return fmt.Errorf("konnectivity kubeconfig set-cluster: %w, %s", err, string(out))
|
||||
}
|
||||
if out, err := exec.CommandContext(ctx, constants.KubectlPath, "--kubeconfig", "/etc/kubernetes/konnectivity-server.conf", "config", "set-context", "system:konnectivity-server@kubernetes",
|
||||
"--cluster", "kubernetes", "--user", "system:konnectivity-server").CombinedOutput(); err != nil {
|
||||
return fmt.Errorf("konnectivity kubeconfig set-context: %w, %s", err, string(out))
|
||||
}
|
||||
if out, err := exec.CommandContext(ctx, constants.KubectlPath, "--kubeconfig", "/etc/kubernetes/konnectivity-server.conf", "config", "use-context", "system:konnectivity-server@kubernetes").CombinedOutput(); err != nil {
|
||||
return fmt.Errorf("konnectivity kubeconfig use-context: %w, %s", err, string(out))
|
||||
}
|
||||
// cleanup
|
||||
if err := os.Remove("/etc/kubernetes/konnectivity.crt"); err != nil {
|
||||
return fmt.Errorf("removing konnectivity certificate: %w", err)
|
||||
}
|
||||
if err := os.Remove("/etc/kubernetes/konnectivity.key"); err != nil {
|
||||
return fmt.Errorf("removing konnectivity key: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// SetupPodNetworkInput holds all configuration options to setup the pod network.
|
||||
type SetupPodNetworkInput struct {
|
||||
CloudProvider string
|
||||
|
|
@ -250,7 +191,7 @@ type SetupPodNetworkInput struct {
|
|||
}
|
||||
|
||||
// JoinCluster joins existing Kubernetes cluster using kubeadm join.
|
||||
func (k *KubernetesUtil) JoinCluster(ctx context.Context, joinConfig []byte, peerRole role.Role, controlPlaneHost, controlPlanePort string, log *logger.Logger) error {
|
||||
func (k *KubernetesUtil) JoinCluster(ctx context.Context, joinConfig []byte, log *logger.Logger) error {
|
||||
// TODO(3u13r): audit policy should be user input
|
||||
auditPolicy, err := resources.NewDefaultAuditPolicy().Marshal()
|
||||
if err != nil {
|
||||
|
|
@ -275,14 +216,6 @@ func (k *KubernetesUtil) JoinCluster(ctx context.Context, joinConfig []byte, pee
|
|||
return fmt.Errorf("creating static pods directory: %w", err)
|
||||
}
|
||||
|
||||
if peerRole == role.ControlPlane {
|
||||
log.Infof("Prep Init Kubernetes cluster")
|
||||
controlPlaneEndpoint := net.JoinHostPort(controlPlaneHost, controlPlanePort)
|
||||
if err := k.prepareControlPlaneForKonnectivity(ctx, controlPlaneEndpoint); err != nil {
|
||||
return fmt.Errorf("setup konnectivity: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
// run `kubeadm join` to join a worker node to an existing Kubernetes cluster
|
||||
cmd := exec.CommandContext(ctx, constants.KubeadmPath, "join", "-v=5", "--config", joinConfigFile.Name())
|
||||
out, err := cmd.CombinedOutput()
|
||||
|
|
@ -369,58 +302,6 @@ func (k *KubernetesUtil) createSignedKubeletCert(nodeName string, ips []net.IP)
|
|||
return k.file.Write(certificate.CertificateFilename, kubeletCert, file.OptMkdirAll)
|
||||
}
|
||||
|
||||
// createSignedKonnectivityCert manually creates a Kubernetes CA signed certificate for the Konnectivity server.
|
||||
func (k *KubernetesUtil) createSignedKonnectivityCert() error {
|
||||
// Create CSR
|
||||
certRequestRaw, keyPem, err := resources.GetKonnectivityCertificateRequest()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := k.file.Write(resources.KonnectivityKeyFilename, keyPem, file.OptMkdirAll); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
certRequest, err := x509.ParseCertificateRequest(certRequestRaw)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Prepare certificate signing
|
||||
serialNumber, err := crypto.GenerateCertificateSerialNumber()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
now := time.Now()
|
||||
// Create the kubelet certificate
|
||||
// For a reference on the certificate fields, see: https://kubernetes.io/docs/setup/best-practices/certificates/
|
||||
certTmpl := &x509.Certificate{
|
||||
SerialNumber: serialNumber,
|
||||
NotBefore: now.Add(-2 * time.Hour),
|
||||
NotAfter: now.Add(24 * 365 * time.Hour),
|
||||
Subject: certRequest.Subject,
|
||||
}
|
||||
|
||||
parentCert, parentKey, err := k.getKubernetesCACertAndKey()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Sign the certificate
|
||||
certRaw, err := x509.CreateCertificate(rand.Reader, certTmpl, parentCert, certRequest.PublicKey, parentKey)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Write the certificate
|
||||
konnectivityCert := pem.EncodeToMemory(&pem.Block{
|
||||
Type: "CERTIFICATE",
|
||||
Bytes: certRaw,
|
||||
})
|
||||
|
||||
return k.file.Write(resources.KonnectivityCertificateFilename, konnectivityCert, file.OptMkdirAll)
|
||||
}
|
||||
|
||||
// getKubernetesCACertAndKey returns the Kubernetes CA certificate and key.
|
||||
// The key of type `any` can be consumed by `x509.CreateCertificate()`.
|
||||
func (k *KubernetesUtil) getKubernetesCACertAndKey() (*x509.Certificate, any, error) {
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue