2022-08-31 21:40:29 -04:00
|
|
|
/*
|
|
|
|
Copyright (c) Edgeless Systems GmbH
|
|
|
|
|
|
|
|
SPDX-License-Identifier: AGPL-3.0-only
|
|
|
|
*/
|
|
|
|
|
|
|
|
package resources
|
|
|
|
|
|
|
|
import (
|
|
|
|
"crypto/x509"
|
|
|
|
"crypto/x509/pkix"
|
|
|
|
|
|
|
|
"github.com/edgelesssys/constellation/bootstrapper/internal/certificate"
|
|
|
|
"github.com/edgelesssys/constellation/internal/kubernetes"
|
|
|
|
"github.com/edgelesssys/constellation/internal/versions"
|
|
|
|
appsv1 "k8s.io/api/apps/v1"
|
|
|
|
corev1 "k8s.io/api/core/v1"
|
|
|
|
rbacv1 "k8s.io/api/rbac/v1"
|
|
|
|
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
|
|
|
"k8s.io/apimachinery/pkg/util/intstr"
|
|
|
|
"k8s.io/apiserver/pkg/apis/apiserver"
|
|
|
|
)
|
|
|
|
|
|
|
|
const (
|
|
|
|
// KonnectivityCertificateFilename is the path to the kubelets certificate.
|
|
|
|
KonnectivityCertificateFilename = "/etc/kubernetes/konnectivity.crt"
|
|
|
|
// KonnectivityKeyFilename is the path to the kubelets private key.
|
|
|
|
KonnectivityKeyFilename = "/etc/kubernetes/konnectivity.key"
|
|
|
|
)
|
|
|
|
|
|
|
|
type konnectivityAgents struct {
|
|
|
|
DaemonSet appsv1.DaemonSet
|
|
|
|
ClusterRoleBinding rbacv1.ClusterRoleBinding
|
|
|
|
ServiceAccount corev1.ServiceAccount
|
|
|
|
}
|
|
|
|
|
|
|
|
type konnectivityServerStaticPod struct {
|
|
|
|
StaticPod corev1.Pod
|
|
|
|
}
|
|
|
|
|
|
|
|
type egressSelectorConfiguration struct {
|
|
|
|
EgressSelectorConfiguration apiserver.EgressSelectorConfiguration
|
|
|
|
}
|
|
|
|
|
|
|
|
func NewKonnectivityAgents(konnectivityServerAddress string) *konnectivityAgents {
|
|
|
|
return &konnectivityAgents{
|
|
|
|
DaemonSet: appsv1.DaemonSet{
|
|
|
|
TypeMeta: metav1.TypeMeta{
|
|
|
|
APIVersion: "apps/v1",
|
|
|
|
Kind: "DaemonSet",
|
|
|
|
},
|
|
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
|
|
Name: "konnectivity-agent",
|
|
|
|
Namespace: "kube-system",
|
|
|
|
Labels: map[string]string{
|
|
|
|
"k8s-app": "konnectivity-agent",
|
|
|
|
"addonmanager.kubernetes.io/mode": "Reconcile",
|
|
|
|
},
|
|
|
|
},
|
|
|
|
Spec: appsv1.DaemonSetSpec{
|
|
|
|
Selector: &metav1.LabelSelector{
|
|
|
|
MatchLabels: map[string]string{
|
|
|
|
"k8s-app": "konnectivity-agent",
|
|
|
|
},
|
|
|
|
},
|
|
|
|
Template: corev1.PodTemplateSpec{
|
|
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
|
|
Labels: map[string]string{
|
|
|
|
"k8s-app": "konnectivity-agent",
|
|
|
|
},
|
|
|
|
},
|
|
|
|
Spec: corev1.PodSpec{
|
|
|
|
PriorityClassName: "system-cluster-critical",
|
|
|
|
Tolerations: []corev1.Toleration{
|
|
|
|
{
|
|
|
|
Key: "node-role.kubernetes.io/master",
|
|
|
|
Operator: corev1.TolerationOpExists,
|
|
|
|
Effect: corev1.TaintEffectNoSchedule,
|
|
|
|
},
|
|
|
|
{
|
|
|
|
Key: "node-role.kubernetes.io/control-plane",
|
|
|
|
Operator: corev1.TolerationOpExists,
|
|
|
|
Effect: corev1.TaintEffectNoSchedule,
|
|
|
|
},
|
|
|
|
{
|
|
|
|
Key: "CriticalAddonsOnly",
|
|
|
|
Operator: corev1.TolerationOpExists,
|
|
|
|
},
|
|
|
|
{
|
|
|
|
Key: "node.kubernetes.io/not-ready",
|
|
|
|
Operator: corev1.TolerationOpExists,
|
|
|
|
Effect: corev1.TaintEffectNoExecute,
|
|
|
|
},
|
|
|
|
},
|
|
|
|
Containers: []corev1.Container{
|
|
|
|
{
|
|
|
|
Name: "konnectivity-agent",
|
|
|
|
Image: versions.KonnectivityAgentImage,
|
|
|
|
Command: []string{
|
|
|
|
"/proxy-agent",
|
|
|
|
},
|
|
|
|
Args: []string{
|
|
|
|
"--logtostderr=true",
|
|
|
|
"--proxy-server-host=" + konnectivityServerAddress,
|
|
|
|
"--ca-cert=/var/run/secrets/kubernetes.io/serviceaccount/ca.crt",
|
|
|
|
"--proxy-server-port=8132",
|
|
|
|
"--admin-server-port=8133",
|
|
|
|
"--health-server-port=8134",
|
|
|
|
"--service-account-token-path=/var/run/secrets/tokens/konnectivity-agent-token",
|
|
|
|
"--agent-identifiers=host=$(HOST_IP)",
|
|
|
|
// we will be able to avoid constant polling when either one is done:
|
|
|
|
// https://github.com/kubernetes-sigs/apiserver-network-proxy/issues/358
|
|
|
|
// https://github.com/kubernetes-sigs/apiserver-network-proxy/issues/273
|
|
|
|
"--sync-forever=true",
|
|
|
|
// Ensure stable connection to the konnectivity server.
|
|
|
|
"--keepalive-time=60s",
|
|
|
|
"--sync-interval=1s",
|
|
|
|
"--sync-interval-cap=3s",
|
|
|
|
"--probe-interval=1s",
|
|
|
|
"--v=3",
|
|
|
|
},
|
|
|
|
Env: []corev1.EnvVar{
|
|
|
|
{
|
|
|
|
Name: "HOST_IP",
|
|
|
|
ValueFrom: &corev1.EnvVarSource{
|
|
|
|
FieldRef: &corev1.ObjectFieldSelector{
|
|
|
|
APIVersion: "v1",
|
|
|
|
FieldPath: "status.hostIP",
|
|
|
|
},
|
|
|
|
},
|
|
|
|
},
|
|
|
|
},
|
|
|
|
VolumeMounts: []corev1.VolumeMount{
|
|
|
|
{
|
|
|
|
Name: "konnectivity-agent-token",
|
|
|
|
MountPath: "/var/run/secrets/tokens",
|
|
|
|
ReadOnly: true,
|
|
|
|
},
|
|
|
|
},
|
|
|
|
LivenessProbe: &corev1.Probe{
|
|
|
|
ProbeHandler: corev1.ProbeHandler{
|
|
|
|
HTTPGet: &corev1.HTTPGetAction{
|
|
|
|
Path: "/healthz",
|
|
|
|
Port: intstr.FromInt(8134),
|
|
|
|
},
|
|
|
|
},
|
|
|
|
InitialDelaySeconds: 15,
|
|
|
|
TimeoutSeconds: 15,
|
|
|
|
},
|
|
|
|
},
|
|
|
|
},
|
|
|
|
ServiceAccountName: "konnectivity-agent",
|
|
|
|
Volumes: []corev1.Volume{
|
|
|
|
{
|
|
|
|
Name: "konnectivity-agent-token",
|
|
|
|
VolumeSource: corev1.VolumeSource{
|
|
|
|
Projected: &corev1.ProjectedVolumeSource{
|
|
|
|
Sources: []corev1.VolumeProjection{
|
|
|
|
{
|
|
|
|
ServiceAccountToken: &corev1.ServiceAccountTokenProjection{
|
|
|
|
Audience: "system:konnectivity-server",
|
|
|
|
Path: "konnectivity-agent-token",
|
|
|
|
},
|
|
|
|
},
|
|
|
|
},
|
|
|
|
},
|
|
|
|
},
|
|
|
|
},
|
|
|
|
},
|
|
|
|
},
|
|
|
|
},
|
|
|
|
},
|
|
|
|
},
|
|
|
|
ClusterRoleBinding: rbacv1.ClusterRoleBinding{
|
|
|
|
TypeMeta: metav1.TypeMeta{
|
|
|
|
APIVersion: "rbac.authorization.k8s.io/v1",
|
|
|
|
Kind: "ClusterRoleBinding",
|
|
|
|
},
|
|
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
|
|
Name: "system:konnectivity-server",
|
|
|
|
Labels: map[string]string{
|
|
|
|
"kubernetes.io/cluster-service": "true",
|
|
|
|
"addonmanager.kubernetes.io/mode": "Reconcile",
|
|
|
|
},
|
|
|
|
},
|
|
|
|
RoleRef: rbacv1.RoleRef{
|
|
|
|
APIGroup: "rbac.authorization.k8s.io",
|
|
|
|
Kind: "ClusterRole",
|
|
|
|
Name: "system:auth-delegator",
|
|
|
|
},
|
|
|
|
Subjects: []rbacv1.Subject{
|
|
|
|
{
|
|
|
|
APIGroup: "rbac.authorization.k8s.io",
|
|
|
|
Kind: "User",
|
|
|
|
Name: "system:konnectivity-server",
|
|
|
|
},
|
|
|
|
},
|
|
|
|
},
|
|
|
|
ServiceAccount: corev1.ServiceAccount{
|
|
|
|
TypeMeta: metav1.TypeMeta{
|
|
|
|
APIVersion: "v1",
|
|
|
|
Kind: "ServiceAccount",
|
|
|
|
},
|
|
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
|
|
Name: "konnectivity-agent",
|
|
|
|
Namespace: "kube-system",
|
|
|
|
Labels: map[string]string{
|
|
|
|
"kubernetes.io/cluster-service": "true",
|
|
|
|
"addonmanager.kubernetes.io/mode": "Reconcile",
|
|
|
|
},
|
|
|
|
},
|
|
|
|
},
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
func NewKonnectivityServerStaticPod() *konnectivityServerStaticPod {
|
|
|
|
udsHostPathType := corev1.HostPathDirectoryOrCreate
|
|
|
|
return &konnectivityServerStaticPod{
|
|
|
|
StaticPod: corev1.Pod{
|
|
|
|
TypeMeta: metav1.TypeMeta{
|
|
|
|
APIVersion: "v1",
|
|
|
|
Kind: "Pod",
|
|
|
|
},
|
|
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
|
|
Name: "konnectivity-server",
|
|
|
|
Namespace: "kube-system",
|
|
|
|
},
|
|
|
|
Spec: corev1.PodSpec{
|
|
|
|
PriorityClassName: "system-cluster-critical",
|
|
|
|
HostNetwork: true,
|
|
|
|
Containers: []corev1.Container{
|
|
|
|
{
|
|
|
|
Name: "konnectivity-server-container",
|
|
|
|
Image: versions.KonnectivityServerImage,
|
|
|
|
Command: []string{"/proxy-server"},
|
|
|
|
Args: []string{
|
|
|
|
"--logtostderr=true",
|
|
|
|
// This needs to be consistent with the value set in egressSelectorConfiguration.
|
|
|
|
"--uds-name=/etc/kubernetes/konnectivity-server/konnectivity-server.socket",
|
|
|
|
// The following two lines assume the Konnectivity server is
|
|
|
|
// deployed on the same machine as the apiserver, and the certs and
|
|
|
|
// key of the API Server are at the specified location.
|
|
|
|
"--cluster-cert=/etc/kubernetes/pki/apiserver.crt",
|
|
|
|
"--cluster-key=/etc/kubernetes/pki/apiserver.key",
|
|
|
|
// This needs to be consistent with the value set in egressSelectorConfiguration.
|
|
|
|
"--mode=grpc",
|
|
|
|
"--server-port=0",
|
|
|
|
"--agent-port=8132",
|
|
|
|
"--admin-port=8133",
|
|
|
|
"--health-port=8134",
|
|
|
|
"--v=5",
|
|
|
|
"--agent-namespace=kube-system",
|
|
|
|
"--agent-service-account=konnectivity-agent",
|
|
|
|
"--kubeconfig=/etc/kubernetes/konnectivity-server.conf",
|
|
|
|
"--authentication-audience=system:konnectivity-server",
|
2022-09-12 08:55:10 -04:00
|
|
|
"--proxy-strategies=destHost,default",
|
2022-08-31 21:40:29 -04:00
|
|
|
},
|
|
|
|
LivenessProbe: &corev1.Probe{
|
|
|
|
ProbeHandler: corev1.ProbeHandler{
|
|
|
|
HTTPGet: &corev1.HTTPGetAction{
|
|
|
|
Path: "/healthz",
|
|
|
|
Port: intstr.FromInt(8134),
|
|
|
|
},
|
|
|
|
},
|
|
|
|
InitialDelaySeconds: 30,
|
|
|
|
TimeoutSeconds: 60,
|
|
|
|
},
|
|
|
|
Ports: []corev1.ContainerPort{
|
|
|
|
{
|
|
|
|
Name: "agent-port",
|
|
|
|
ContainerPort: 8132,
|
|
|
|
HostPort: 8132,
|
|
|
|
},
|
|
|
|
{
|
|
|
|
Name: "admin-port",
|
|
|
|
ContainerPort: 8133,
|
|
|
|
HostPort: 8133,
|
|
|
|
},
|
|
|
|
{
|
|
|
|
Name: "health-port",
|
|
|
|
ContainerPort: 8134,
|
|
|
|
HostPort: 8134,
|
|
|
|
},
|
|
|
|
},
|
|
|
|
VolumeMounts: []corev1.VolumeMount{
|
|
|
|
{
|
|
|
|
Name: "k8s-certs",
|
|
|
|
MountPath: "/etc/kubernetes/pki",
|
|
|
|
ReadOnly: true,
|
|
|
|
},
|
|
|
|
{
|
|
|
|
Name: "kubeconfig",
|
|
|
|
MountPath: "/etc/kubernetes/konnectivity-server.conf",
|
|
|
|
ReadOnly: true,
|
|
|
|
},
|
|
|
|
{
|
|
|
|
Name: "konnectivity-uds",
|
|
|
|
MountPath: "/etc/kubernetes/konnectivity-server",
|
|
|
|
ReadOnly: false,
|
|
|
|
},
|
|
|
|
},
|
|
|
|
},
|
|
|
|
},
|
|
|
|
Volumes: []corev1.Volume{
|
|
|
|
{
|
|
|
|
Name: "k8s-certs",
|
|
|
|
VolumeSource: corev1.VolumeSource{
|
|
|
|
HostPath: &corev1.HostPathVolumeSource{
|
|
|
|
Path: "/etc/kubernetes/pki",
|
|
|
|
},
|
|
|
|
},
|
|
|
|
},
|
|
|
|
{
|
|
|
|
Name: "kubeconfig",
|
|
|
|
VolumeSource: corev1.VolumeSource{
|
|
|
|
HostPath: &corev1.HostPathVolumeSource{
|
|
|
|
Path: "/etc/kubernetes/konnectivity-server.conf",
|
|
|
|
},
|
|
|
|
},
|
|
|
|
},
|
|
|
|
{
|
|
|
|
Name: "konnectivity-uds",
|
|
|
|
VolumeSource: corev1.VolumeSource{
|
|
|
|
HostPath: &corev1.HostPathVolumeSource{
|
|
|
|
Path: "/etc/kubernetes/konnectivity-server",
|
|
|
|
Type: &udsHostPathType,
|
|
|
|
},
|
|
|
|
},
|
|
|
|
},
|
|
|
|
},
|
|
|
|
},
|
|
|
|
},
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
func NewEgressSelectorConfiguration() *egressSelectorConfiguration {
|
|
|
|
return &egressSelectorConfiguration{
|
|
|
|
EgressSelectorConfiguration: apiserver.EgressSelectorConfiguration{
|
|
|
|
TypeMeta: metav1.TypeMeta{
|
|
|
|
APIVersion: "apiserver.k8s.io/v1beta1",
|
|
|
|
Kind: "EgressSelectorConfiguration",
|
|
|
|
},
|
|
|
|
EgressSelections: []apiserver.EgressSelection{
|
|
|
|
{
|
|
|
|
Name: "cluster",
|
|
|
|
Connection: apiserver.Connection{
|
|
|
|
ProxyProtocol: "GRPC",
|
|
|
|
Transport: &apiserver.Transport{
|
|
|
|
UDS: &apiserver.UDSTransport{
|
|
|
|
UDSName: "/etc/kubernetes/konnectivity-server/konnectivity-server.socket",
|
|
|
|
},
|
|
|
|
},
|
|
|
|
},
|
|
|
|
},
|
|
|
|
},
|
|
|
|
},
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
func (v *konnectivityAgents) Marshal() ([]byte, error) {
|
|
|
|
return kubernetes.MarshalK8SResources(v)
|
|
|
|
}
|
|
|
|
|
|
|
|
func (v *konnectivityServerStaticPod) Marshal() ([]byte, error) {
|
|
|
|
return kubernetes.MarshalK8SResources(v)
|
|
|
|
}
|
|
|
|
|
|
|
|
func (v *egressSelectorConfiguration) Marshal() ([]byte, error) {
|
|
|
|
return kubernetes.MarshalK8SResources(v)
|
|
|
|
}
|
|
|
|
|
|
|
|
// GetCertificateRequest returns a certificate request and matching private key for the konnectivity server.
|
|
|
|
func GetKonnectivityCertificateRequest() (certificateRequest []byte, privateKey []byte, err error) {
|
|
|
|
csrTemplate := &x509.CertificateRequest{
|
|
|
|
Subject: pkix.Name{
|
|
|
|
CommonName: "system:konnectivity-server",
|
|
|
|
},
|
|
|
|
}
|
|
|
|
return certificate.GetCertificateRequest(csrTemplate)
|
|
|
|
}
|