/* Copyright (c) Edgeless Systems GmbH SPDX-License-Identifier: AGPL-3.0-only */ package resources import ( "crypto/x509" "crypto/x509/pkix" "github.com/edgelesssys/constellation/bootstrapper/internal/certificate" "github.com/edgelesssys/constellation/internal/kubernetes" "github.com/edgelesssys/constellation/internal/versions" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/apiserver/pkg/apis/apiserver" ) const ( // KonnectivityCertificateFilename is the path to the kubelets certificate. KonnectivityCertificateFilename = "/etc/kubernetes/konnectivity.crt" // KonnectivityKeyFilename is the path to the kubelets private key. KonnectivityKeyFilename = "/etc/kubernetes/konnectivity.key" ) type konnectivityAgents struct { DaemonSet appsv1.DaemonSet ClusterRoleBinding rbacv1.ClusterRoleBinding ServiceAccount corev1.ServiceAccount } type konnectivityServerStaticPod struct { StaticPod corev1.Pod } type egressSelectorConfiguration struct { EgressSelectorConfiguration apiserver.EgressSelectorConfiguration } func NewKonnectivityAgents(konnectivityServerAddress string) *konnectivityAgents { return &konnectivityAgents{ DaemonSet: appsv1.DaemonSet{ TypeMeta: metav1.TypeMeta{ APIVersion: "apps/v1", Kind: "DaemonSet", }, ObjectMeta: metav1.ObjectMeta{ Name: "konnectivity-agent", Namespace: "kube-system", Labels: map[string]string{ "k8s-app": "konnectivity-agent", "addonmanager.kubernetes.io/mode": "Reconcile", }, }, Spec: appsv1.DaemonSetSpec{ Selector: &metav1.LabelSelector{ MatchLabels: map[string]string{ "k8s-app": "konnectivity-agent", }, }, Template: corev1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ Labels: map[string]string{ "k8s-app": "konnectivity-agent", }, }, Spec: corev1.PodSpec{ PriorityClassName: "system-cluster-critical", Tolerations: []corev1.Toleration{ { Key: "node-role.kubernetes.io/master", Operator: corev1.TolerationOpExists, Effect: corev1.TaintEffectNoSchedule, }, { Key: "node-role.kubernetes.io/control-plane", Operator: corev1.TolerationOpExists, Effect: corev1.TaintEffectNoSchedule, }, { Key: "CriticalAddonsOnly", Operator: corev1.TolerationOpExists, }, { Key: "node.kubernetes.io/not-ready", Operator: corev1.TolerationOpExists, Effect: corev1.TaintEffectNoExecute, }, }, Containers: []corev1.Container{ { Name: "konnectivity-agent", Image: versions.KonnectivityAgentImage, Command: []string{ "/proxy-agent", }, Args: []string{ "--logtostderr=true", "--proxy-server-host=" + konnectivityServerAddress, "--ca-cert=/var/run/secrets/kubernetes.io/serviceaccount/ca.crt", "--proxy-server-port=8132", "--admin-server-port=8133", "--health-server-port=8134", "--service-account-token-path=/var/run/secrets/tokens/konnectivity-agent-token", "--agent-identifiers=host=$(HOST_IP)", // we will be able to avoid constant polling when either one is done: // https://github.com/kubernetes-sigs/apiserver-network-proxy/issues/358 // https://github.com/kubernetes-sigs/apiserver-network-proxy/issues/273 "--sync-forever=true", // Ensure stable connection to the konnectivity server. "--keepalive-time=60s", "--sync-interval=1s", "--sync-interval-cap=3s", "--probe-interval=1s", "--v=3", }, Env: []corev1.EnvVar{ { Name: "HOST_IP", ValueFrom: &corev1.EnvVarSource{ FieldRef: &corev1.ObjectFieldSelector{ APIVersion: "v1", FieldPath: "status.hostIP", }, }, }, }, VolumeMounts: []corev1.VolumeMount{ { Name: "konnectivity-agent-token", MountPath: "/var/run/secrets/tokens", ReadOnly: true, }, }, LivenessProbe: &corev1.Probe{ ProbeHandler: corev1.ProbeHandler{ HTTPGet: &corev1.HTTPGetAction{ Path: "/healthz", Port: intstr.FromInt(8134), }, }, InitialDelaySeconds: 15, TimeoutSeconds: 15, }, }, }, ServiceAccountName: "konnectivity-agent", Volumes: []corev1.Volume{ { Name: "konnectivity-agent-token", VolumeSource: corev1.VolumeSource{ Projected: &corev1.ProjectedVolumeSource{ Sources: []corev1.VolumeProjection{ { ServiceAccountToken: &corev1.ServiceAccountTokenProjection{ Audience: "system:konnectivity-server", Path: "konnectivity-agent-token", }, }, }, }, }, }, }, }, }, }, }, ClusterRoleBinding: rbacv1.ClusterRoleBinding{ TypeMeta: metav1.TypeMeta{ APIVersion: "rbac.authorization.k8s.io/v1", Kind: "ClusterRoleBinding", }, ObjectMeta: metav1.ObjectMeta{ Name: "system:konnectivity-server", Labels: map[string]string{ "kubernetes.io/cluster-service": "true", "addonmanager.kubernetes.io/mode": "Reconcile", }, }, RoleRef: rbacv1.RoleRef{ APIGroup: "rbac.authorization.k8s.io", Kind: "ClusterRole", Name: "system:auth-delegator", }, Subjects: []rbacv1.Subject{ { APIGroup: "rbac.authorization.k8s.io", Kind: "User", Name: "system:konnectivity-server", }, }, }, ServiceAccount: corev1.ServiceAccount{ TypeMeta: metav1.TypeMeta{ APIVersion: "v1", Kind: "ServiceAccount", }, ObjectMeta: metav1.ObjectMeta{ Name: "konnectivity-agent", Namespace: "kube-system", Labels: map[string]string{ "kubernetes.io/cluster-service": "true", "addonmanager.kubernetes.io/mode": "Reconcile", }, }, }, } } func NewKonnectivityServerStaticPod() *konnectivityServerStaticPod { udsHostPathType := corev1.HostPathDirectoryOrCreate return &konnectivityServerStaticPod{ StaticPod: corev1.Pod{ TypeMeta: metav1.TypeMeta{ APIVersion: "v1", Kind: "Pod", }, ObjectMeta: metav1.ObjectMeta{ Name: "konnectivity-server", Namespace: "kube-system", }, Spec: corev1.PodSpec{ PriorityClassName: "system-cluster-critical", HostNetwork: true, Containers: []corev1.Container{ { Name: "konnectivity-server-container", Image: versions.KonnectivityServerImage, Command: []string{"/proxy-server"}, Args: []string{ "--logtostderr=true", // This needs to be consistent with the value set in egressSelectorConfiguration. "--uds-name=/etc/kubernetes/konnectivity-server/konnectivity-server.socket", // The following two lines assume the Konnectivity server is // deployed on the same machine as the apiserver, and the certs and // key of the API Server are at the specified location. "--cluster-cert=/etc/kubernetes/pki/apiserver.crt", "--cluster-key=/etc/kubernetes/pki/apiserver.key", // This needs to be consistent with the value set in egressSelectorConfiguration. "--mode=grpc", "--server-port=0", "--agent-port=8132", "--admin-port=8133", "--health-port=8134", "--v=5", "--agent-namespace=kube-system", "--agent-service-account=konnectivity-agent", "--kubeconfig=/etc/kubernetes/konnectivity-server.conf", "--authentication-audience=system:konnectivity-server", "--proxy-strategies=destHost", }, LivenessProbe: &corev1.Probe{ ProbeHandler: corev1.ProbeHandler{ HTTPGet: &corev1.HTTPGetAction{ Path: "/healthz", Port: intstr.FromInt(8134), }, }, InitialDelaySeconds: 30, TimeoutSeconds: 60, }, Ports: []corev1.ContainerPort{ { Name: "agent-port", ContainerPort: 8132, HostPort: 8132, }, { Name: "admin-port", ContainerPort: 8133, HostPort: 8133, }, { Name: "health-port", ContainerPort: 8134, HostPort: 8134, }, }, VolumeMounts: []corev1.VolumeMount{ { Name: "k8s-certs", MountPath: "/etc/kubernetes/pki", ReadOnly: true, }, { Name: "kubeconfig", MountPath: "/etc/kubernetes/konnectivity-server.conf", ReadOnly: true, }, { Name: "konnectivity-uds", MountPath: "/etc/kubernetes/konnectivity-server", ReadOnly: false, }, }, }, }, Volumes: []corev1.Volume{ { Name: "k8s-certs", VolumeSource: corev1.VolumeSource{ HostPath: &corev1.HostPathVolumeSource{ Path: "/etc/kubernetes/pki", }, }, }, { Name: "kubeconfig", VolumeSource: corev1.VolumeSource{ HostPath: &corev1.HostPathVolumeSource{ Path: "/etc/kubernetes/konnectivity-server.conf", }, }, }, { Name: "konnectivity-uds", VolumeSource: corev1.VolumeSource{ HostPath: &corev1.HostPathVolumeSource{ Path: "/etc/kubernetes/konnectivity-server", Type: &udsHostPathType, }, }, }, }, }, }, } } func NewEgressSelectorConfiguration() *egressSelectorConfiguration { return &egressSelectorConfiguration{ EgressSelectorConfiguration: apiserver.EgressSelectorConfiguration{ TypeMeta: metav1.TypeMeta{ APIVersion: "apiserver.k8s.io/v1beta1", Kind: "EgressSelectorConfiguration", }, EgressSelections: []apiserver.EgressSelection{ { Name: "cluster", Connection: apiserver.Connection{ ProxyProtocol: "GRPC", Transport: &apiserver.Transport{ UDS: &apiserver.UDSTransport{ UDSName: "/etc/kubernetes/konnectivity-server/konnectivity-server.socket", }, }, }, }, }, }, } } func (v *konnectivityAgents) Marshal() ([]byte, error) { return kubernetes.MarshalK8SResources(v) } func (v *konnectivityServerStaticPod) Marshal() ([]byte, error) { return kubernetes.MarshalK8SResources(v) } func (v *egressSelectorConfiguration) Marshal() ([]byte, error) { return kubernetes.MarshalK8SResources(v) } // GetCertificateRequest returns a certificate request and matching private key for the konnectivity server. func GetKonnectivityCertificateRequest() (certificateRequest []byte, privateKey []byte, err error) { csrTemplate := &x509.CertificateRequest{ Subject: pkix.Name{ CommonName: "system:konnectivity-server", }, } return certificate.GetCertificateRequest(csrTemplate) }