From 6dc97590fe54491287e62a1c568d7a05b87dc794 Mon Sep 17 00:00:00 2001 From: Moritz Eckert Date: Fri, 20 May 2022 17:30:37 +0200 Subject: [PATCH] Enable and configure k8s audit-log (#160) * Enable and configure k8s audit-log * Update coordinator/kubernetes/k8sapi/kubeadm_config.go Co-authored-by: Malte Poll * add mount point for audit log dir in kubeadm conf * Mount audit policy into kube-apiserver static pod * Write default auditpolicy on cluster init / cluster join Co-authored-by: Malte Poll --- .../kubernetes/k8sapi/kubeadm_config.go | 53 ++++++++++++++----- .../k8sapi/resources/auditpolicy.go | 33 ++++++++++++ .../k8sapi/resources/auditpolicy_test.go | 21 ++++++++ coordinator/kubernetes/k8sapi/util.go | 18 +++++++ go.mod | 1 + go.sum | 1 + 6 files changed, 114 insertions(+), 13 deletions(-) create mode 100644 coordinator/kubernetes/k8sapi/resources/auditpolicy.go create mode 100644 coordinator/kubernetes/k8sapi/resources/auditpolicy_test.go diff --git a/coordinator/kubernetes/k8sapi/kubeadm_config.go b/coordinator/kubernetes/k8sapi/kubeadm_config.go index 35e325c76..56e29f1c4 100644 --- a/coordinator/kubernetes/k8sapi/kubeadm_config.go +++ b/coordinator/kubernetes/k8sapi/kubeadm_config.go @@ -1,9 +1,12 @@ package k8sapi import ( + "path/filepath" + "github.com/edgelesssys/constellation/coordinator/kubernetes/k8sapi/resources" "github.com/edgelesssys/constellation/internal/constants" - v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" kubeletconf "k8s.io/kubelet/config/v1beta1" kubeadm "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta3" ) @@ -12,7 +15,10 @@ import ( // Slimmed down to the fields we require const ( - bindPort = 6443 + bindPort = 6443 + auditLogDir = "/var/log/kubernetes/audit/" + auditLogFile = "audit.log" + auditPolicyPath = "/etc/kubernetes/audit-policy.yaml" ) type CoreOSConfiguration struct{} @@ -24,7 +30,7 @@ func (c *CoreOSConfiguration) InitConfiguration(externalCloudProvider bool) Kube } return KubeadmInitYAML{ InitConfiguration: kubeadm.InitConfiguration{ - TypeMeta: v1.TypeMeta{ + TypeMeta: metav1.TypeMeta{ APIVersion: kubeadm.SchemeGroupVersion.String(), Kind: "InitConfiguration", }, @@ -41,7 +47,7 @@ func (c *CoreOSConfiguration) InitConfiguration(externalCloudProvider bool) Kube }, }, ClusterConfiguration: kubeadm.ClusterConfiguration{ - TypeMeta: v1.TypeMeta{ + TypeMeta: metav1.TypeMeta{ Kind: "ClusterConfiguration", APIVersion: kubeadm.SchemeGroupVersion.String(), }, @@ -50,7 +56,12 @@ func (c *CoreOSConfiguration) InitConfiguration(externalCloudProvider bool) Kube APIServer: kubeadm.APIServer{ ControlPlaneComponent: kubeadm.ControlPlaneComponent{ ExtraArgs: map[string]string{ - "profiling": "false", // CIS benchmark + "audit-policy-file": auditPolicyPath, + "audit-log-path": filepath.Join(auditLogDir, auditLogFile), // CIS benchmark + "audit-log-maxage": "30", // CIS benchmark - Default value of Rancher + "audit-log-maxbackup": "10", // CIS benchmark - Default value of Rancher + "audit-log-maxsize": "100", // CIS benchmark - Default value of Rancher + "profiling": "false", // CIS benchmark "tls-cipher-suites": "TLS_AES_128_GCM_SHA256,TLS_AES_256_GCM_SHA384,TLS_CHACHA20_POLY1305_SHA256," + "TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256," + "TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384," + @@ -60,6 +71,22 @@ func (c *CoreOSConfiguration) InitConfiguration(externalCloudProvider bool) Kube "TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256,TLS_RSA_WITH_3DES_EDE_CBC_SHA,TLS_RSA_WITH_AES_128_CBC_SHA," + "TLS_RSA_WITH_AES_128_GCM_SHA256,TLS_RSA_WITH_AES_256_CBC_SHA,TLS_RSA_WITH_AES_256_GCM_SHA384", // CIS benchmark }, + ExtraVolumes: []kubeadm.HostPathMount{ + { + Name: "audit-log", + HostPath: auditLogDir, + MountPath: auditLogDir, + ReadOnly: false, + PathType: corev1.HostPathDirectoryOrCreate, + }, + { + Name: "audit", + HostPath: auditPolicyPath, + MountPath: auditPolicyPath, + ReadOnly: true, + PathType: corev1.HostPathFile, + }, + }, }, CertSANs: []string{"127.0.0.1", "10.118.0.1"}, }, @@ -92,7 +119,7 @@ func (c *CoreOSConfiguration) InitConfiguration(externalCloudProvider bool) Kube "TLS_RSA_WITH_AES_256_GCM_SHA384", "TLS_RSA_WITH_AES_128_GCM_SHA256", }, // CIS benchmark - TypeMeta: v1.TypeMeta{ + TypeMeta: metav1.TypeMeta{ APIVersion: kubeletconf.SchemeGroupVersion.String(), Kind: "KubeletConfiguration", }, @@ -107,7 +134,7 @@ func (c *CoreOSConfiguration) JoinConfiguration(externalCloudProvider bool) Kube } return KubeadmJoinYAML{ JoinConfiguration: kubeadm.JoinConfiguration{ - TypeMeta: v1.TypeMeta{ + TypeMeta: metav1.TypeMeta{ APIVersion: kubeadm.SchemeGroupVersion.String(), Kind: "JoinConfiguration", }, @@ -122,7 +149,7 @@ func (c *CoreOSConfiguration) JoinConfiguration(externalCloudProvider bool) Kube }, }, KubeletConfiguration: kubeletconf.KubeletConfiguration{ - TypeMeta: v1.TypeMeta{ + TypeMeta: metav1.TypeMeta{ APIVersion: kubeletconf.SchemeGroupVersion.String(), Kind: "KubeletConfiguration", }, @@ -135,7 +162,7 @@ type AWSConfiguration struct{} func (a *AWSConfiguration) InitConfiguration() KubeadmInitYAML { return KubeadmInitYAML{ InitConfiguration: kubeadm.InitConfiguration{ - TypeMeta: v1.TypeMeta{ + TypeMeta: metav1.TypeMeta{ APIVersion: kubeadm.SchemeGroupVersion.String(), Kind: "InitConfiguration", }, @@ -146,7 +173,7 @@ func (a *AWSConfiguration) InitConfiguration() KubeadmInitYAML { LocalAPIEndpoint: kubeadm.APIEndpoint{BindPort: bindPort}, }, ClusterConfiguration: kubeadm.ClusterConfiguration{ - TypeMeta: v1.TypeMeta{ + TypeMeta: metav1.TypeMeta{ APIVersion: kubeadm.SchemeGroupVersion.String(), Kind: "ClusterConfiguration", }, @@ -155,7 +182,7 @@ func (a *AWSConfiguration) InitConfiguration() KubeadmInitYAML { }, }, KubeletConfiguration: kubeletconf.KubeletConfiguration{ - TypeMeta: v1.TypeMeta{ + TypeMeta: metav1.TypeMeta{ APIVersion: kubeletconf.SchemeGroupVersion.String(), Kind: "KubeletConfiguration", }, @@ -166,7 +193,7 @@ func (a *AWSConfiguration) InitConfiguration() KubeadmInitYAML { func (a *AWSConfiguration) JoinConfiguration() KubeadmJoinYAML { return KubeadmJoinYAML{ JoinConfiguration: kubeadm.JoinConfiguration{ - TypeMeta: v1.TypeMeta{ + TypeMeta: metav1.TypeMeta{ APIVersion: kubeadm.SchemeGroupVersion.String(), Kind: "JoinConfiguration", }, @@ -179,7 +206,7 @@ func (a *AWSConfiguration) JoinConfiguration() KubeadmJoinYAML { }, }, KubeletConfiguration: kubeletconf.KubeletConfiguration{ - TypeMeta: v1.TypeMeta{ + TypeMeta: metav1.TypeMeta{ APIVersion: kubeletconf.SchemeGroupVersion.String(), Kind: "KubeletConfiguration", }, diff --git a/coordinator/kubernetes/k8sapi/resources/auditpolicy.go b/coordinator/kubernetes/k8sapi/resources/auditpolicy.go new file mode 100644 index 000000000..56a82f013 --- /dev/null +++ b/coordinator/kubernetes/k8sapi/resources/auditpolicy.go @@ -0,0 +1,33 @@ +package resources + +import ( + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + auditv1 "k8s.io/apiserver/pkg/apis/audit/v1" +) + +// AuditPolicy defines rulesets for what should be logged in the kube-apiserver audit log. +// reference: https://kubernetes.io/docs/tasks/debug/debug-cluster/audit/ . +type AuditPolicy struct { + Policy auditv1.Policy +} + +func NewDefaultAuditPolicy() *AuditPolicy { + return &AuditPolicy{ + Policy: auditv1.Policy{ + TypeMeta: v1.TypeMeta{ + APIVersion: "audit.k8s.io/v1", + Kind: "Policy", + }, + Rules: []auditv1.PolicyRule{ + { + Level: auditv1.LevelMetadata, + }, + }, + }, + } +} + +// Marshal marshals the audit policy as a YAML document. +func (p *AuditPolicy) Marshal() ([]byte, error) { + return MarshalK8SResources(p) +} diff --git a/coordinator/kubernetes/k8sapi/resources/auditpolicy_test.go b/coordinator/kubernetes/k8sapi/resources/auditpolicy_test.go new file mode 100644 index 000000000..eb90e1ebf --- /dev/null +++ b/coordinator/kubernetes/k8sapi/resources/auditpolicy_test.go @@ -0,0 +1,21 @@ +package resources + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestAuditPolicyMarshalUnmarshal(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + + auditPolicy := NewDefaultAuditPolicy() + data, err := auditPolicy.Marshal() + require.NoError(err) + + var recreated AuditPolicy + require.NoError(UnmarshalK8SResources(data, &recreated)) + assert.Equal(auditPolicy, &recreated) +} diff --git a/coordinator/kubernetes/k8sapi/util.go b/coordinator/kubernetes/k8sapi/util.go index cfcb021ab..2b9b24a33 100644 --- a/coordinator/kubernetes/k8sapi/util.go +++ b/coordinator/kubernetes/k8sapi/util.go @@ -37,6 +37,15 @@ type ClusterUtil interface { type KubernetesUtil struct{} func (k *KubernetesUtil) InitCluster(initConfig []byte) error { + // TODO: audit policy should be user input + auditPolicy, err := resources.NewDefaultAuditPolicy().Marshal() + if err != nil { + return fmt.Errorf("failed to generate default audit policy: %w", err) + } + if err := os.WriteFile(auditPolicyPath, auditPolicy, 0o644); err != nil { + return fmt.Errorf("failed to write default audit policy: %w", err) + } + initConfigFile, err := os.CreateTemp("", "kubeadm-init.*.yaml") if err != nil { return fmt.Errorf("failed to create init config file %v: %w", initConfigFile.Name(), err) @@ -102,6 +111,15 @@ func (k *KubernetesUtil) SetupCloudNodeManager(kubectl Client, cloudNodeManagerC // JoinCluster joins existing Kubernetes cluster using kubeadm join. func (k *KubernetesUtil) JoinCluster(joinConfig []byte) error { + // TODO: audit policy should be user input + auditPolicy, err := resources.NewDefaultAuditPolicy().Marshal() + if err != nil { + return fmt.Errorf("failed to generate default audit policy: %w", err) + } + if err := os.WriteFile(auditPolicyPath, auditPolicy, 0o644); err != nil { + return fmt.Errorf("failed to write default audit policy: %w", err) + } + joinConfigFile, err := os.CreateTemp("", "kubeadm-join.*.yaml") if err != nil { return fmt.Errorf("failed to create join config file %v: %w", joinConfigFile.Name(), err) diff --git a/go.mod b/go.mod index ffa1bda9f..12b89462c 100644 --- a/go.mod +++ b/go.mod @@ -208,6 +208,7 @@ require ( google.golang.org/appengine v1.6.7 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect + k8s.io/apiserver v0.24.0 k8s.io/cluster-bootstrap v0.0.0 // indirect k8s.io/component-base v0.24.0 // indirect k8s.io/kube-openapi v0.0.0-20220328201542-3ee0da9b0b42 // indirect diff --git a/go.sum b/go.sum index a575f75ee..d891e68ac 100644 --- a/go.sum +++ b/go.sum @@ -2301,6 +2301,7 @@ k8s.io/api v0.24.0/go.mod h1:5Jl90IUrJHUJYEMANRURMiVvJ0g7Ax7r3R1bqO8zx8I= k8s.io/apiextensions-apiserver v0.24.0/go.mod h1:iuVe4aEpe6827lvO6yWQVxiPSpPoSKVjkq+MIdg84cM= k8s.io/apimachinery v0.24.0 h1:ydFCyC/DjCvFCHK5OPMKBlxayQytB8pxy8YQInd5UyQ= k8s.io/apimachinery v0.24.0/go.mod h1:82Bi4sCzVBdpYjyI4jY6aHX+YCUchUIrZrXKedjd2UM= +k8s.io/apiserver v0.24.0 h1:GR7kGsjOMfilRvlG3Stxv/3uz/ryvJ/aZXc5pqdsNV0= k8s.io/apiserver v0.24.0/go.mod h1:WFx2yiOMawnogNToVvUYT9nn1jaIkMKj41ZYCVycsBA= k8s.io/cli-runtime v0.24.0 h1:ot3Qf49T852uEyNApABO1UHHpFIckKK/NqpheZYN2gM= k8s.io/cli-runtime v0.24.0/go.mod h1:9XxoZDsEkRFUThnwqNviqzljtT/LdHtNWvcNFrAXl0A=