mirror of
https://github.com/edgelesssys/constellation.git
synced 2025-08-02 20:16:15 -04:00
Deploy Konnectivity
This commit is contained in:
parent
15592e8f3f
commit
7163c161b6
32 changed files with 1243 additions and 496 deletions
|
@ -66,12 +66,13 @@ func (c *CoreOSConfiguration) InitConfiguration(externalCloudProvider bool, k8sV
|
|||
APIServer: kubeadm.APIServer{
|
||||
ControlPlaneComponent: kubeadm.ControlPlaneComponent{
|
||||
ExtraArgs: map[string]string{
|
||||
"audit-policy-file": auditPolicyPath,
|
||||
"audit-log-path": filepath.Join(auditLogDir, auditLogFile), // CIS benchmark
|
||||
"audit-log-maxage": "30", // CIS benchmark - Default value of Rancher
|
||||
"audit-log-maxbackup": "10", // CIS benchmark - Default value of Rancher
|
||||
"audit-log-maxsize": "100", // CIS benchmark - Default value of Rancher
|
||||
"profiling": "false", // CIS benchmark
|
||||
"audit-policy-file": auditPolicyPath,
|
||||
"audit-log-path": filepath.Join(auditLogDir, auditLogFile), // CIS benchmark
|
||||
"audit-log-maxage": "30", // CIS benchmark - Default value of Rancher
|
||||
"audit-log-maxbackup": "10", // CIS benchmark - Default value of Rancher
|
||||
"audit-log-maxsize": "100", // CIS benchmark - Default value of Rancher
|
||||
"profiling": "false", // CIS benchmark
|
||||
"egress-selector-config-file": "/etc/kubernetes/egress-selector-configuration.yaml",
|
||||
"kubelet-certificate-authority": filepath.Join(
|
||||
kubeconstants.KubernetesDir,
|
||||
kubeconstants.DefaultCertificateDir,
|
||||
|
@ -101,6 +102,20 @@ func (c *CoreOSConfiguration) InitConfiguration(externalCloudProvider bool, k8sV
|
|||
ReadOnly: true,
|
||||
PathType: corev1.HostPathFile,
|
||||
},
|
||||
{
|
||||
Name: "egress-config",
|
||||
HostPath: "/etc/kubernetes/egress-selector-configuration.yaml",
|
||||
MountPath: "/etc/kubernetes/egress-selector-configuration.yaml",
|
||||
ReadOnly: true,
|
||||
PathType: corev1.HostPathFile,
|
||||
},
|
||||
{
|
||||
Name: "konnectivity-uds",
|
||||
HostPath: "/etc/kubernetes/konnectivity-server",
|
||||
MountPath: "/etc/kubernetes/konnectivity-server",
|
||||
ReadOnly: false,
|
||||
PathType: corev1.HostPathDirectoryOrCreate,
|
||||
},
|
||||
},
|
||||
},
|
||||
CertSANs: []string{"127.0.0.1"},
|
||||
|
@ -133,6 +148,7 @@ func (c *CoreOSConfiguration) InitConfiguration(externalCloudProvider bool, k8sV
|
|||
"TLS_RSA_WITH_AES_256_GCM_SHA384",
|
||||
"TLS_RSA_WITH_AES_128_GCM_SHA256",
|
||||
}, // CIS benchmark
|
||||
StaticPodPath: "/etc/kubernetes/manifests",
|
||||
TypeMeta: metav1.TypeMeta{
|
||||
APIVersion: kubeletconf.SchemeGroupVersion.String(),
|
||||
Kind: "KubeletConfiguration",
|
||||
|
|
|
@ -0,0 +1,380 @@
|
|||
/*
|
||||
Copyright (c) Edgeless Systems GmbH
|
||||
|
||||
SPDX-License-Identifier: AGPL-3.0-only
|
||||
*/
|
||||
|
||||
package resources
|
||||
|
||||
import (
|
||||
"crypto/x509"
|
||||
"crypto/x509/pkix"
|
||||
|
||||
"github.com/edgelesssys/constellation/bootstrapper/internal/certificate"
|
||||
"github.com/edgelesssys/constellation/internal/kubernetes"
|
||||
"github.com/edgelesssys/constellation/internal/versions"
|
||||
appsv1 "k8s.io/api/apps/v1"
|
||||
corev1 "k8s.io/api/core/v1"
|
||||
rbacv1 "k8s.io/api/rbac/v1"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/util/intstr"
|
||||
"k8s.io/apiserver/pkg/apis/apiserver"
|
||||
)
|
||||
|
||||
const (
|
||||
// KonnectivityCertificateFilename is the path to the kubelets certificate.
|
||||
KonnectivityCertificateFilename = "/etc/kubernetes/konnectivity.crt"
|
||||
// KonnectivityKeyFilename is the path to the kubelets private key.
|
||||
KonnectivityKeyFilename = "/etc/kubernetes/konnectivity.key"
|
||||
)
|
||||
|
||||
type konnectivityAgents struct {
|
||||
DaemonSet appsv1.DaemonSet
|
||||
ClusterRoleBinding rbacv1.ClusterRoleBinding
|
||||
ServiceAccount corev1.ServiceAccount
|
||||
}
|
||||
|
||||
type konnectivityServerStaticPod struct {
|
||||
StaticPod corev1.Pod
|
||||
}
|
||||
|
||||
type egressSelectorConfiguration struct {
|
||||
EgressSelectorConfiguration apiserver.EgressSelectorConfiguration
|
||||
}
|
||||
|
||||
func NewKonnectivityAgents(konnectivityServerAddress string) *konnectivityAgents {
|
||||
return &konnectivityAgents{
|
||||
DaemonSet: appsv1.DaemonSet{
|
||||
TypeMeta: metav1.TypeMeta{
|
||||
APIVersion: "apps/v1",
|
||||
Kind: "DaemonSet",
|
||||
},
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: "konnectivity-agent",
|
||||
Namespace: "kube-system",
|
||||
Labels: map[string]string{
|
||||
"k8s-app": "konnectivity-agent",
|
||||
"addonmanager.kubernetes.io/mode": "Reconcile",
|
||||
},
|
||||
},
|
||||
Spec: appsv1.DaemonSetSpec{
|
||||
Selector: &metav1.LabelSelector{
|
||||
MatchLabels: map[string]string{
|
||||
"k8s-app": "konnectivity-agent",
|
||||
},
|
||||
},
|
||||
Template: corev1.PodTemplateSpec{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Labels: map[string]string{
|
||||
"k8s-app": "konnectivity-agent",
|
||||
},
|
||||
},
|
||||
Spec: corev1.PodSpec{
|
||||
PriorityClassName: "system-cluster-critical",
|
||||
Tolerations: []corev1.Toleration{
|
||||
{
|
||||
Key: "node-role.kubernetes.io/master",
|
||||
Operator: corev1.TolerationOpExists,
|
||||
Effect: corev1.TaintEffectNoSchedule,
|
||||
},
|
||||
{
|
||||
Key: "node-role.kubernetes.io/control-plane",
|
||||
Operator: corev1.TolerationOpExists,
|
||||
Effect: corev1.TaintEffectNoSchedule,
|
||||
},
|
||||
{
|
||||
Key: "CriticalAddonsOnly",
|
||||
Operator: corev1.TolerationOpExists,
|
||||
},
|
||||
{
|
||||
Key: "node.kubernetes.io/not-ready",
|
||||
Operator: corev1.TolerationOpExists,
|
||||
Effect: corev1.TaintEffectNoExecute,
|
||||
},
|
||||
},
|
||||
Containers: []corev1.Container{
|
||||
{
|
||||
Name: "konnectivity-agent",
|
||||
Image: versions.KonnectivityAgentImage,
|
||||
Command: []string{
|
||||
"/proxy-agent",
|
||||
},
|
||||
Args: []string{
|
||||
"--logtostderr=true",
|
||||
"--proxy-server-host=" + konnectivityServerAddress,
|
||||
"--ca-cert=/var/run/secrets/kubernetes.io/serviceaccount/ca.crt",
|
||||
"--proxy-server-port=8132",
|
||||
"--admin-server-port=8133",
|
||||
"--health-server-port=8134",
|
||||
"--service-account-token-path=/var/run/secrets/tokens/konnectivity-agent-token",
|
||||
"--agent-identifiers=host=$(HOST_IP)",
|
||||
// we will be able to avoid constant polling when either one is done:
|
||||
// https://github.com/kubernetes-sigs/apiserver-network-proxy/issues/358
|
||||
// https://github.com/kubernetes-sigs/apiserver-network-proxy/issues/273
|
||||
"--sync-forever=true",
|
||||
// Ensure stable connection to the konnectivity server.
|
||||
"--keepalive-time=60s",
|
||||
"--sync-interval=1s",
|
||||
"--sync-interval-cap=3s",
|
||||
"--probe-interval=1s",
|
||||
"--v=3",
|
||||
},
|
||||
Env: []corev1.EnvVar{
|
||||
{
|
||||
Name: "HOST_IP",
|
||||
ValueFrom: &corev1.EnvVarSource{
|
||||
FieldRef: &corev1.ObjectFieldSelector{
|
||||
APIVersion: "v1",
|
||||
FieldPath: "status.hostIP",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
VolumeMounts: []corev1.VolumeMount{
|
||||
{
|
||||
Name: "konnectivity-agent-token",
|
||||
MountPath: "/var/run/secrets/tokens",
|
||||
ReadOnly: true,
|
||||
},
|
||||
},
|
||||
LivenessProbe: &corev1.Probe{
|
||||
ProbeHandler: corev1.ProbeHandler{
|
||||
HTTPGet: &corev1.HTTPGetAction{
|
||||
Path: "/healthz",
|
||||
Port: intstr.FromInt(8134),
|
||||
},
|
||||
},
|
||||
InitialDelaySeconds: 15,
|
||||
TimeoutSeconds: 15,
|
||||
},
|
||||
},
|
||||
},
|
||||
ServiceAccountName: "konnectivity-agent",
|
||||
Volumes: []corev1.Volume{
|
||||
{
|
||||
Name: "konnectivity-agent-token",
|
||||
VolumeSource: corev1.VolumeSource{
|
||||
Projected: &corev1.ProjectedVolumeSource{
|
||||
Sources: []corev1.VolumeProjection{
|
||||
{
|
||||
ServiceAccountToken: &corev1.ServiceAccountTokenProjection{
|
||||
Audience: "system:konnectivity-server",
|
||||
Path: "konnectivity-agent-token",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
ClusterRoleBinding: rbacv1.ClusterRoleBinding{
|
||||
TypeMeta: metav1.TypeMeta{
|
||||
APIVersion: "rbac.authorization.k8s.io/v1",
|
||||
Kind: "ClusterRoleBinding",
|
||||
},
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: "system:konnectivity-server",
|
||||
Labels: map[string]string{
|
||||
"kubernetes.io/cluster-service": "true",
|
||||
"addonmanager.kubernetes.io/mode": "Reconcile",
|
||||
},
|
||||
},
|
||||
RoleRef: rbacv1.RoleRef{
|
||||
APIGroup: "rbac.authorization.k8s.io",
|
||||
Kind: "ClusterRole",
|
||||
Name: "system:auth-delegator",
|
||||
},
|
||||
Subjects: []rbacv1.Subject{
|
||||
{
|
||||
APIGroup: "rbac.authorization.k8s.io",
|
||||
Kind: "User",
|
||||
Name: "system:konnectivity-server",
|
||||
},
|
||||
},
|
||||
},
|
||||
ServiceAccount: corev1.ServiceAccount{
|
||||
TypeMeta: metav1.TypeMeta{
|
||||
APIVersion: "v1",
|
||||
Kind: "ServiceAccount",
|
||||
},
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: "konnectivity-agent",
|
||||
Namespace: "kube-system",
|
||||
Labels: map[string]string{
|
||||
"kubernetes.io/cluster-service": "true",
|
||||
"addonmanager.kubernetes.io/mode": "Reconcile",
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func NewKonnectivityServerStaticPod() *konnectivityServerStaticPod {
|
||||
udsHostPathType := corev1.HostPathDirectoryOrCreate
|
||||
return &konnectivityServerStaticPod{
|
||||
StaticPod: corev1.Pod{
|
||||
TypeMeta: metav1.TypeMeta{
|
||||
APIVersion: "v1",
|
||||
Kind: "Pod",
|
||||
},
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: "konnectivity-server",
|
||||
Namespace: "kube-system",
|
||||
},
|
||||
Spec: corev1.PodSpec{
|
||||
PriorityClassName: "system-cluster-critical",
|
||||
HostNetwork: true,
|
||||
Containers: []corev1.Container{
|
||||
{
|
||||
Name: "konnectivity-server-container",
|
||||
Image: versions.KonnectivityServerImage,
|
||||
Command: []string{"/proxy-server"},
|
||||
Args: []string{
|
||||
"--logtostderr=true",
|
||||
// This needs to be consistent with the value set in egressSelectorConfiguration.
|
||||
"--uds-name=/etc/kubernetes/konnectivity-server/konnectivity-server.socket",
|
||||
// The following two lines assume the Konnectivity server is
|
||||
// deployed on the same machine as the apiserver, and the certs and
|
||||
// key of the API Server are at the specified location.
|
||||
"--cluster-cert=/etc/kubernetes/pki/apiserver.crt",
|
||||
"--cluster-key=/etc/kubernetes/pki/apiserver.key",
|
||||
// This needs to be consistent with the value set in egressSelectorConfiguration.
|
||||
"--mode=grpc",
|
||||
"--server-port=0",
|
||||
"--agent-port=8132",
|
||||
"--admin-port=8133",
|
||||
"--health-port=8134",
|
||||
"--v=5",
|
||||
"--agent-namespace=kube-system",
|
||||
"--agent-service-account=konnectivity-agent",
|
||||
"--kubeconfig=/etc/kubernetes/konnectivity-server.conf",
|
||||
"--authentication-audience=system:konnectivity-server",
|
||||
"--proxy-strategies=destHost",
|
||||
},
|
||||
LivenessProbe: &corev1.Probe{
|
||||
ProbeHandler: corev1.ProbeHandler{
|
||||
HTTPGet: &corev1.HTTPGetAction{
|
||||
Path: "/healthz",
|
||||
Port: intstr.FromInt(8134),
|
||||
},
|
||||
},
|
||||
InitialDelaySeconds: 30,
|
||||
TimeoutSeconds: 60,
|
||||
},
|
||||
Ports: []corev1.ContainerPort{
|
||||
{
|
||||
Name: "agent-port",
|
||||
ContainerPort: 8132,
|
||||
HostPort: 8132,
|
||||
},
|
||||
{
|
||||
Name: "admin-port",
|
||||
ContainerPort: 8133,
|
||||
HostPort: 8133,
|
||||
},
|
||||
{
|
||||
Name: "health-port",
|
||||
ContainerPort: 8134,
|
||||
HostPort: 8134,
|
||||
},
|
||||
},
|
||||
VolumeMounts: []corev1.VolumeMount{
|
||||
{
|
||||
Name: "k8s-certs",
|
||||
MountPath: "/etc/kubernetes/pki",
|
||||
ReadOnly: true,
|
||||
},
|
||||
{
|
||||
Name: "kubeconfig",
|
||||
MountPath: "/etc/kubernetes/konnectivity-server.conf",
|
||||
ReadOnly: true,
|
||||
},
|
||||
{
|
||||
Name: "konnectivity-uds",
|
||||
MountPath: "/etc/kubernetes/konnectivity-server",
|
||||
ReadOnly: false,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
Volumes: []corev1.Volume{
|
||||
{
|
||||
Name: "k8s-certs",
|
||||
VolumeSource: corev1.VolumeSource{
|
||||
HostPath: &corev1.HostPathVolumeSource{
|
||||
Path: "/etc/kubernetes/pki",
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: "kubeconfig",
|
||||
VolumeSource: corev1.VolumeSource{
|
||||
HostPath: &corev1.HostPathVolumeSource{
|
||||
Path: "/etc/kubernetes/konnectivity-server.conf",
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: "konnectivity-uds",
|
||||
VolumeSource: corev1.VolumeSource{
|
||||
HostPath: &corev1.HostPathVolumeSource{
|
||||
Path: "/etc/kubernetes/konnectivity-server",
|
||||
Type: &udsHostPathType,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func NewEgressSelectorConfiguration() *egressSelectorConfiguration {
|
||||
return &egressSelectorConfiguration{
|
||||
EgressSelectorConfiguration: apiserver.EgressSelectorConfiguration{
|
||||
TypeMeta: metav1.TypeMeta{
|
||||
APIVersion: "apiserver.k8s.io/v1beta1",
|
||||
Kind: "EgressSelectorConfiguration",
|
||||
},
|
||||
EgressSelections: []apiserver.EgressSelection{
|
||||
{
|
||||
Name: "cluster",
|
||||
Connection: apiserver.Connection{
|
||||
ProxyProtocol: "GRPC",
|
||||
Transport: &apiserver.Transport{
|
||||
UDS: &apiserver.UDSTransport{
|
||||
UDSName: "/etc/kubernetes/konnectivity-server/konnectivity-server.socket",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func (v *konnectivityAgents) Marshal() ([]byte, error) {
|
||||
return kubernetes.MarshalK8SResources(v)
|
||||
}
|
||||
|
||||
func (v *konnectivityServerStaticPod) Marshal() ([]byte, error) {
|
||||
return kubernetes.MarshalK8SResources(v)
|
||||
}
|
||||
|
||||
func (v *egressSelectorConfiguration) Marshal() ([]byte, error) {
|
||||
return kubernetes.MarshalK8SResources(v)
|
||||
}
|
||||
|
||||
// GetCertificateRequest returns a certificate request and matching private key for the konnectivity server.
|
||||
func GetKonnectivityCertificateRequest() (certificateRequest []byte, privateKey []byte, err error) {
|
||||
csrTemplate := &x509.CertificateRequest{
|
||||
Subject: pkix.Name{
|
||||
CommonName: "system:konnectivity-server",
|
||||
},
|
||||
}
|
||||
return certificate.GetCertificateRequest(csrTemplate)
|
||||
}
|
|
@ -0,0 +1,28 @@
|
|||
/*
|
||||
Copyright (c) Edgeless Systems GmbH
|
||||
|
||||
SPDX-License-Identifier: AGPL-3.0-only
|
||||
*/
|
||||
|
||||
package resources
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/edgelesssys/constellation/internal/kubernetes"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestKonnectivityMarshalUnmarshal(t *testing.T) {
|
||||
require := require.New(t)
|
||||
assert := assert.New(t)
|
||||
|
||||
kmsDepl := NewKonnectivityAgents("192.168.2.1")
|
||||
data, err := kmsDepl.Marshal()
|
||||
require.NoError(err)
|
||||
|
||||
var recreated konnectivityAgents
|
||||
require.NoError(kubernetes.UnmarshalK8SResources(data, &recreated))
|
||||
assert.Equal(kmsDepl, &recreated)
|
||||
}
|
|
@ -28,6 +28,7 @@ import (
|
|||
"github.com/edgelesssys/constellation/bootstrapper/internal/kubernetes/k8sapi/resources"
|
||||
"github.com/edgelesssys/constellation/internal/constants"
|
||||
"github.com/edgelesssys/constellation/internal/kubernetes"
|
||||
"github.com/edgelesssys/constellation/internal/role"
|
||||
kubeconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
|
||||
|
||||
"github.com/edgelesssys/constellation/internal/crypto"
|
||||
|
@ -130,7 +131,7 @@ func (k *KubernetesUtil) InstallComponents(ctx context.Context, version versions
|
|||
}
|
||||
|
||||
func (k *KubernetesUtil) InitCluster(
|
||||
ctx context.Context, initConfig []byte, nodeName string, ips []net.IP, log *logger.Logger,
|
||||
ctx context.Context, initConfig []byte, nodeName string, ips []net.IP, controlPlaneEndpoint string, log *logger.Logger,
|
||||
) error {
|
||||
// TODO: audit policy should be user input
|
||||
auditPolicy, err := resources.NewDefaultAuditPolicy().Marshal()
|
||||
|
@ -180,6 +181,11 @@ func (k *KubernetesUtil) InitCluster(
|
|||
return err
|
||||
}
|
||||
|
||||
log.Infof("Preparing node for Konnectivity")
|
||||
if err := k.prepareControlPlaneForKonnectivity(ctx, controlPlaneEndpoint); err != nil {
|
||||
return fmt.Errorf("setup konnectivity: %w", err)
|
||||
}
|
||||
|
||||
// initialize the cluster
|
||||
log.Infof("Initializing the cluster using kubeadm init")
|
||||
cmd = exec.CommandContext(ctx, kubeadmPath, "init", "-v=5", "--skip-phases=preflight,certs,addon/kube-proxy", "--config", initConfigFile.Name())
|
||||
|
@ -195,6 +201,64 @@ func (k *KubernetesUtil) InitCluster(
|
|||
return nil
|
||||
}
|
||||
|
||||
func (k *KubernetesUtil) prepareControlPlaneForKonnectivity(ctx context.Context, loadBalancerEndpoint string) error {
|
||||
if !strings.Contains(loadBalancerEndpoint, ":") {
|
||||
loadBalancerEndpoint = net.JoinHostPort(loadBalancerEndpoint, strconv.Itoa(constants.KubernetesPort))
|
||||
}
|
||||
|
||||
if err := os.MkdirAll("/etc/kubernetes/manifests", os.ModePerm); err != nil {
|
||||
return fmt.Errorf("creating static pods directory: %w", err)
|
||||
}
|
||||
|
||||
konnectivityServerYaml, err := resources.NewKonnectivityServerStaticPod().Marshal()
|
||||
if err != nil {
|
||||
return fmt.Errorf("generating konnectivity server static pod: %w", err)
|
||||
}
|
||||
if err := os.WriteFile("/etc/kubernetes/manifests/konnectivity-server.yaml", konnectivityServerYaml, 0o644); err != nil {
|
||||
return fmt.Errorf("writing konnectivity server pod: %w", err)
|
||||
}
|
||||
|
||||
egressConfigYaml, err := resources.NewEgressSelectorConfiguration().Marshal()
|
||||
if err != nil {
|
||||
return fmt.Errorf("generating egress selector configuration: %w", err)
|
||||
}
|
||||
if err := os.WriteFile("/etc/kubernetes/egress-selector-configuration.yaml", egressConfigYaml, 0o644); err != nil {
|
||||
return fmt.Errorf("writing egress selector config: %w", err)
|
||||
}
|
||||
|
||||
if err := k.createSignedKonnectivityCert(); err != nil {
|
||||
return fmt.Errorf("generating konnectivity server certificate: %w", err)
|
||||
}
|
||||
|
||||
if out, err := exec.CommandContext(ctx, kubectlPath, "config", "set-credentials", "--kubeconfig", "/etc/kubernetes/konnectivity-server.conf", "system:konnectivity-server",
|
||||
"--client-certificate", "/etc/kubernetes/konnectivity.crt", "--client-key", "/etc/kubernetes/konnectivity.key", "--embed-certs=true").CombinedOutput(); err != nil {
|
||||
return fmt.Errorf("konnectivity kubeconfig set-credentials: %w, %s", err, string(out))
|
||||
}
|
||||
if out, err := exec.CommandContext(ctx, kubectlPath, "--kubeconfig", "/etc/kubernetes/konnectivity-server.conf", "config", "set-cluster", "kubernetes", "--server", "https://"+loadBalancerEndpoint,
|
||||
"--certificate-authority", "/etc/kubernetes/pki/ca.crt", "--embed-certs=true").CombinedOutput(); err != nil {
|
||||
return fmt.Errorf("konnectivity kubeconfig set-cluster: %w, %s", err, string(out))
|
||||
}
|
||||
if out, err := exec.CommandContext(ctx, kubectlPath, "--kubeconfig", "/etc/kubernetes/konnectivity-server.conf", "config", "set-context", "system:konnectivity-server@kubernetes",
|
||||
"--cluster", "kubernetes", "--user", "system:konnectivity-server").CombinedOutput(); err != nil {
|
||||
return fmt.Errorf("konnectivity kubeconfig set-context: %w, %s", err, string(out))
|
||||
}
|
||||
if out, err := exec.CommandContext(ctx, kubectlPath, "--kubeconfig", "/etc/kubernetes/konnectivity-server.conf", "config", "use-context", "system:konnectivity-server@kubernetes").CombinedOutput(); err != nil {
|
||||
return fmt.Errorf("konnectivity kubeconfig use-context: %w, %s", err, string(out))
|
||||
}
|
||||
// cleanup
|
||||
if err := os.Remove("/etc/kubernetes/konnectivity.crt"); err != nil {
|
||||
return fmt.Errorf("removing konnectivity certificate: %w", err)
|
||||
}
|
||||
if err := os.Remove("/etc/kubernetes/konnectivity.key"); err != nil {
|
||||
return fmt.Errorf("removing konnectivity key: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (k *KubernetesUtil) SetupKonnectivity(kubectl Client, konnectivityAgentsDaemonSet kubernetes.Marshaler) error {
|
||||
return kubectl.Apply(konnectivityAgentsDaemonSet, true)
|
||||
}
|
||||
|
||||
func (k *KubernetesUtil) SetupHelmDeployments(ctx context.Context, kubectl Client, helmDeployments []byte, in SetupPodNetworkInput, log *logger.Logger) error {
|
||||
var helmDeploy helm.Deployments
|
||||
if err := json.Unmarshal(helmDeployments, &helmDeploy); err != nil {
|
||||
|
@ -450,7 +514,7 @@ func (k *KubernetesUtil) SetupNodeOperator(ctx context.Context, kubectl Client,
|
|||
}
|
||||
|
||||
// JoinCluster joins existing Kubernetes cluster using kubeadm join.
|
||||
func (k *KubernetesUtil) JoinCluster(ctx context.Context, joinConfig []byte, log *logger.Logger) error {
|
||||
func (k *KubernetesUtil) JoinCluster(ctx context.Context, joinConfig []byte, peerRole role.Role, controlPlaneEndpoint string, log *logger.Logger) error {
|
||||
// TODO: audit policy should be user input
|
||||
auditPolicy, err := resources.NewDefaultAuditPolicy().Marshal()
|
||||
if err != nil {
|
||||
|
@ -469,6 +533,13 @@ func (k *KubernetesUtil) JoinCluster(ctx context.Context, joinConfig []byte, log
|
|||
return fmt.Errorf("writing kubeadm init yaml config %v: %w", joinConfigFile.Name(), err)
|
||||
}
|
||||
|
||||
if peerRole == role.ControlPlane {
|
||||
log.Infof("Prep Init Kubernetes cluster")
|
||||
if err := k.prepareControlPlaneForKonnectivity(ctx, controlPlaneEndpoint); err != nil {
|
||||
return fmt.Errorf("setup konnectivity: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
// run `kubeadm join` to join a worker node to an existing Kubernetes cluster
|
||||
cmd := exec.CommandContext(ctx, kubeadmPath, "join", "-v=5", "--config", joinConfigFile.Name())
|
||||
out, err := cmd.CombinedOutput()
|
||||
|
@ -497,6 +568,7 @@ func (k *KubernetesUtil) StartKubelet() error {
|
|||
// createSignedKubeletCert manually creates a Kubernetes CA signed kubelet certificate for the bootstrapper node.
|
||||
// This is necessary because this node does not request a certificate from the join service.
|
||||
func (k *KubernetesUtil) createSignedKubeletCert(nodeName string, ips []net.IP) error {
|
||||
// Create CSR
|
||||
certRequestRaw, kubeletKey, err := kubelet.GetCertificateRequest(nodeName, ips)
|
||||
if err != nil {
|
||||
return err
|
||||
|
@ -505,48 +577,12 @@ func (k *KubernetesUtil) createSignedKubeletCert(nodeName string, ips []net.IP)
|
|||
return err
|
||||
}
|
||||
|
||||
parentCertRaw, err := k.file.Read(filepath.Join(
|
||||
kubeconstants.KubernetesDir,
|
||||
kubeconstants.DefaultCertificateDir,
|
||||
kubeconstants.CACertName,
|
||||
))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
parentCert, err := crypto.PemToX509Cert(parentCertRaw)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
parentKeyRaw, err := k.file.Read(filepath.Join(
|
||||
kubeconstants.KubernetesDir,
|
||||
kubeconstants.DefaultCertificateDir,
|
||||
kubeconstants.CAKeyName,
|
||||
))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
parentKeyPEM, _ := pem.Decode(parentKeyRaw)
|
||||
var parentKey any
|
||||
switch parentKeyPEM.Type {
|
||||
case "EC PRIVATE KEY":
|
||||
parentKey, err = x509.ParseECPrivateKey(parentKeyPEM.Bytes)
|
||||
case "RSA PRIVATE KEY":
|
||||
parentKey, err = x509.ParsePKCS1PrivateKey(parentKeyPEM.Bytes)
|
||||
case "PRIVATE KEY":
|
||||
parentKey, err = x509.ParsePKCS8PrivateKey(parentKeyPEM.Bytes)
|
||||
default:
|
||||
err = fmt.Errorf("unsupported key type %q", parentKeyPEM.Type)
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
certRequest, err := x509.ParseCertificateRequest(certRequestRaw)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Prepare certificate signing
|
||||
serialNumber, err := crypto.GenerateCertificateSerialNumber()
|
||||
if err != nil {
|
||||
return err
|
||||
|
@ -570,10 +606,18 @@ func (k *KubernetesUtil) createSignedKubeletCert(nodeName string, ips []net.IP)
|
|||
IPAddresses: certRequest.IPAddresses,
|
||||
}
|
||||
|
||||
parentCert, parentKey, err := k.getKubernetesCACertAndKey()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Sign the certificate
|
||||
certRaw, err := x509.CreateCertificate(rand.Reader, certTmpl, parentCert, certRequest.PublicKey, parentKey)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Write the certificate
|
||||
kubeletCert := pem.EncodeToMemory(&pem.Block{
|
||||
Type: "CERTIFICATE",
|
||||
Bytes: certRaw,
|
||||
|
@ -581,3 +625,97 @@ func (k *KubernetesUtil) createSignedKubeletCert(nodeName string, ips []net.IP)
|
|||
|
||||
return k.file.Write(kubelet.CertificateFilename, kubeletCert, file.OptMkdirAll)
|
||||
}
|
||||
|
||||
// createSignedKonnectivityCert manually creates a Kubernetes CA signed certificate for the Konnectivity server.
|
||||
func (k *KubernetesUtil) createSignedKonnectivityCert() error {
|
||||
// Create CSR
|
||||
certRequestRaw, keyPem, err := resources.GetKonnectivityCertificateRequest()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := k.file.Write(resources.KonnectivityKeyFilename, keyPem, file.OptMkdirAll); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
certRequest, err := x509.ParseCertificateRequest(certRequestRaw)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Prepare certificate signing
|
||||
serialNumber, err := crypto.GenerateCertificateSerialNumber()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
now := time.Now()
|
||||
// Create the kubelet certificate
|
||||
// For a reference on the certificate fields, see: https://kubernetes.io/docs/setup/best-practices/certificates/
|
||||
certTmpl := &x509.Certificate{
|
||||
SerialNumber: serialNumber,
|
||||
NotBefore: now.Add(-2 * time.Hour),
|
||||
NotAfter: now.Add(24 * 365 * time.Hour),
|
||||
Subject: certRequest.Subject,
|
||||
}
|
||||
|
||||
parentCert, parentKey, err := k.getKubernetesCACertAndKey()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Sign the certificate
|
||||
certRaw, err := x509.CreateCertificate(rand.Reader, certTmpl, parentCert, certRequest.PublicKey, parentKey)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Write the certificate
|
||||
konnectivityCert := pem.EncodeToMemory(&pem.Block{
|
||||
Type: "CERTIFICATE",
|
||||
Bytes: certRaw,
|
||||
})
|
||||
|
||||
return k.file.Write(resources.KonnectivityCertificateFilename, konnectivityCert, file.OptMkdirAll)
|
||||
}
|
||||
|
||||
// getKubernetesCACertAndKey returns the Kubernetes CA certificate and key.
|
||||
// The key of type `any` can be consumed by `x509.CreateCertificate()`.
|
||||
func (k *KubernetesUtil) getKubernetesCACertAndKey() (*x509.Certificate, any, error) {
|
||||
parentCertRaw, err := k.file.Read(filepath.Join(
|
||||
kubeconstants.KubernetesDir,
|
||||
kubeconstants.DefaultCertificateDir,
|
||||
kubeconstants.CACertName,
|
||||
))
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
parentCert, err := crypto.PemToX509Cert(parentCertRaw)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
parentKeyRaw, err := k.file.Read(filepath.Join(
|
||||
kubeconstants.KubernetesDir,
|
||||
kubeconstants.DefaultCertificateDir,
|
||||
kubeconstants.CAKeyName,
|
||||
))
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
parentKeyPEM, _ := pem.Decode(parentKeyRaw)
|
||||
var parentKey any
|
||||
switch parentKeyPEM.Type {
|
||||
case "EC PRIVATE KEY":
|
||||
parentKey, err = x509.ParseECPrivateKey(parentKeyPEM.Bytes)
|
||||
case "RSA PRIVATE KEY":
|
||||
parentKey, err = x509.ParsePKCS1PrivateKey(parentKeyPEM.Bytes)
|
||||
case "PRIVATE KEY":
|
||||
parentKey, err = x509.ParsePKCS8PrivateKey(parentKeyPEM.Bytes)
|
||||
default:
|
||||
err = fmt.Errorf("unsupported key type %q", parentKeyPEM.Type)
|
||||
}
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
return parentCert, parentKey, nil
|
||||
}
|
||||
|
|
Loading…
Add table
Add a link
Reference in a new issue