Enable and configure k8s audit-log (#160)

* Enable and configure k8s audit-log

* Update coordinator/kubernetes/k8sapi/kubeadm_config.go

Co-authored-by: Malte Poll <mp@edgeless.systems>

* add mount point for audit log dir in kubeadm conf

* Mount audit policy into kube-apiserver static pod

* Write default auditpolicy on cluster init / cluster join

Co-authored-by: Malte Poll <mp@edgeless.systems>

coordinator/kubernetes/k8sapi/kubeadm_config.go

@@ -1,9 +1,12 @@
 package k8sapi
 
 import (
+	"path/filepath"
+
 	"github.com/edgelesssys/constellation/coordinator/kubernetes/k8sapi/resources"
 	"github.com/edgelesssys/constellation/internal/constants"
-	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	kubeletconf "k8s.io/kubelet/config/v1beta1"
 	kubeadm "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta3"
 )
@@ -12,7 +15,10 @@ import (
 // Slimmed down to the fields we require
 
 const (
-	bindPort = 6443
+	bindPort        = 6443
+	auditLogDir     = "/var/log/kubernetes/audit/"
+	auditLogFile    = "audit.log"
+	auditPolicyPath = "/etc/kubernetes/audit-policy.yaml"
 )
 
 type CoreOSConfiguration struct{}
@@ -24,7 +30,7 @@ func (c *CoreOSConfiguration) InitConfiguration(externalCloudProvider bool) Kube
 	}
 	return KubeadmInitYAML{
 		InitConfiguration: kubeadm.InitConfiguration{
-			TypeMeta: v1.TypeMeta{
+			TypeMeta: metav1.TypeMeta{
 				APIVersion: kubeadm.SchemeGroupVersion.String(),
 				Kind:       "InitConfiguration",
 			},
@@ -41,7 +47,7 @@ func (c *CoreOSConfiguration) InitConfiguration(externalCloudProvider bool) Kube
 			},
 		},
 		ClusterConfiguration: kubeadm.ClusterConfiguration{
-			TypeMeta: v1.TypeMeta{
+			TypeMeta: metav1.TypeMeta{
 				Kind:       "ClusterConfiguration",
 				APIVersion: kubeadm.SchemeGroupVersion.String(),
 			},
@@ -50,7 +56,12 @@ func (c *CoreOSConfiguration) InitConfiguration(externalCloudProvider bool) Kube
 			APIServer: kubeadm.APIServer{
 				ControlPlaneComponent: kubeadm.ControlPlaneComponent{
 					ExtraArgs: map[string]string{
-						"profiling": "false", // CIS benchmark
+						"audit-policy-file":   auditPolicyPath,
+						"audit-log-path":      filepath.Join(auditLogDir, auditLogFile), // CIS benchmark
+						"audit-log-maxage":    "30",                                     // CIS benchmark - Default value of Rancher
+						"audit-log-maxbackup": "10",                                     // CIS benchmark - Default value of Rancher
+						"audit-log-maxsize":   "100",                                    // CIS benchmark - Default value of Rancher
+						"profiling":           "false",                                  // CIS benchmark
 						"tls-cipher-suites": "TLS_AES_128_GCM_SHA256,TLS_AES_256_GCM_SHA384,TLS_CHACHA20_POLY1305_SHA256," +
 							"TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256," +
 							"TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384," +
@@ -60,6 +71,22 @@ func (c *CoreOSConfiguration) InitConfiguration(externalCloudProvider bool) Kube
 							"TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256,TLS_RSA_WITH_3DES_EDE_CBC_SHA,TLS_RSA_WITH_AES_128_CBC_SHA," +
 							"TLS_RSA_WITH_AES_128_GCM_SHA256,TLS_RSA_WITH_AES_256_CBC_SHA,TLS_RSA_WITH_AES_256_GCM_SHA384", // CIS benchmark
 					},
+					ExtraVolumes: []kubeadm.HostPathMount{
+						{
+							Name:      "audit-log",
+							HostPath:  auditLogDir,
+							MountPath: auditLogDir,
+							ReadOnly:  false,
+							PathType:  corev1.HostPathDirectoryOrCreate,
+						},
+						{
+							Name:      "audit",
+							HostPath:  auditPolicyPath,
+							MountPath: auditPolicyPath,
+							ReadOnly:  true,
+							PathType:  corev1.HostPathFile,
+						},
+					},
 				},
 				CertSANs: []string{"127.0.0.1", "10.118.0.1"},
 			},
@@ -92,7 +119,7 @@ func (c *CoreOSConfiguration) InitConfiguration(externalCloudProvider bool) Kube
 				"TLS_RSA_WITH_AES_256_GCM_SHA384",
 				"TLS_RSA_WITH_AES_128_GCM_SHA256",
 			}, // CIS benchmark
-			TypeMeta: v1.TypeMeta{
+			TypeMeta: metav1.TypeMeta{
 				APIVersion: kubeletconf.SchemeGroupVersion.String(),
 				Kind:       "KubeletConfiguration",
 			},
@@ -107,7 +134,7 @@ func (c *CoreOSConfiguration) JoinConfiguration(externalCloudProvider bool) Kube
 	}
 	return KubeadmJoinYAML{
 		JoinConfiguration: kubeadm.JoinConfiguration{
-			TypeMeta: v1.TypeMeta{
+			TypeMeta: metav1.TypeMeta{
 				APIVersion: kubeadm.SchemeGroupVersion.String(),
 				Kind:       "JoinConfiguration",
 			},
@@ -122,7 +149,7 @@ func (c *CoreOSConfiguration) JoinConfiguration(externalCloudProvider bool) Kube
 			},
 		},
 		KubeletConfiguration: kubeletconf.KubeletConfiguration{
-			TypeMeta: v1.TypeMeta{
+			TypeMeta: metav1.TypeMeta{
 				APIVersion: kubeletconf.SchemeGroupVersion.String(),
 				Kind:       "KubeletConfiguration",
 			},
@@ -135,7 +162,7 @@ type AWSConfiguration struct{}
 func (a *AWSConfiguration) InitConfiguration() KubeadmInitYAML {
 	return KubeadmInitYAML{
 		InitConfiguration: kubeadm.InitConfiguration{
-			TypeMeta: v1.TypeMeta{
+			TypeMeta: metav1.TypeMeta{
 				APIVersion: kubeadm.SchemeGroupVersion.String(),
 				Kind:       "InitConfiguration",
 			},
@@ -146,7 +173,7 @@ func (a *AWSConfiguration) InitConfiguration() KubeadmInitYAML {
 			LocalAPIEndpoint: kubeadm.APIEndpoint{BindPort: bindPort},
 		},
 		ClusterConfiguration: kubeadm.ClusterConfiguration{
-			TypeMeta: v1.TypeMeta{
+			TypeMeta: metav1.TypeMeta{
 				APIVersion: kubeadm.SchemeGroupVersion.String(),
 				Kind:       "ClusterConfiguration",
 			},
@@ -155,7 +182,7 @@ func (a *AWSConfiguration) InitConfiguration() KubeadmInitYAML {
 			},
 		},
 		KubeletConfiguration: kubeletconf.KubeletConfiguration{
-			TypeMeta: v1.TypeMeta{
+			TypeMeta: metav1.TypeMeta{
 				APIVersion: kubeletconf.SchemeGroupVersion.String(),
 				Kind:       "KubeletConfiguration",
 			},
@@ -166,7 +193,7 @@ func (a *AWSConfiguration) InitConfiguration() KubeadmInitYAML {
 func (a *AWSConfiguration) JoinConfiguration() KubeadmJoinYAML {
 	return KubeadmJoinYAML{
 		JoinConfiguration: kubeadm.JoinConfiguration{
-			TypeMeta: v1.TypeMeta{
+			TypeMeta: metav1.TypeMeta{
 				APIVersion: kubeadm.SchemeGroupVersion.String(),
 				Kind:       "JoinConfiguration",
 			},
@@ -179,7 +206,7 @@ func (a *AWSConfiguration) JoinConfiguration() KubeadmJoinYAML {
 			},
 		},
 		KubeletConfiguration: kubeletconf.KubeletConfiguration{
-			TypeMeta: v1.TypeMeta{
+			TypeMeta: metav1.TypeMeta{
 				APIVersion: kubeletconf.SchemeGroupVersion.String(),
 				Kind:       "KubeletConfiguration",
 			},

coordinator/kubernetes/k8sapi/resources/auditpolicy.go

@@ -0,0 +1,33 @@
+package resources
+
+import (
+	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	auditv1 "k8s.io/apiserver/pkg/apis/audit/v1"
+)
+
+// AuditPolicy defines rulesets for what should be logged in the kube-apiserver audit log.
+// reference: https://kubernetes.io/docs/tasks/debug/debug-cluster/audit/ .
+type AuditPolicy struct {
+	Policy auditv1.Policy
+}
+
+func NewDefaultAuditPolicy() *AuditPolicy {
+	return &AuditPolicy{
+		Policy: auditv1.Policy{
+			TypeMeta: v1.TypeMeta{
+				APIVersion: "audit.k8s.io/v1",
+				Kind:       "Policy",
+			},
+			Rules: []auditv1.PolicyRule{
+				{
+					Level: auditv1.LevelMetadata,
+				},
+			},
+		},
+	}
+}
+
+// Marshal marshals the audit policy as a YAML document.
+func (p *AuditPolicy) Marshal() ([]byte, error) {
+	return MarshalK8SResources(p)
+}

coordinator/kubernetes/k8sapi/resources/auditpolicy_test.go

@@ -0,0 +1,21 @@
+package resources
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestAuditPolicyMarshalUnmarshal(t *testing.T) {
+	require := require.New(t)
+	assert := assert.New(t)
+
+	auditPolicy := NewDefaultAuditPolicy()
+	data, err := auditPolicy.Marshal()
+	require.NoError(err)
+
+	var recreated AuditPolicy
+	require.NoError(UnmarshalK8SResources(data, &recreated))
+	assert.Equal(auditPolicy, &recreated)
+}

coordinator/kubernetes/k8sapi/util.go

@@ -37,6 +37,15 @@ type ClusterUtil interface {
 type KubernetesUtil struct{}
 
 func (k *KubernetesUtil) InitCluster(initConfig []byte) error {
+	// TODO: audit policy should be user input
+	auditPolicy, err := resources.NewDefaultAuditPolicy().Marshal()
+	if err != nil {
+		return fmt.Errorf("failed to generate default audit policy: %w", err)
+	}
+	if err := os.WriteFile(auditPolicyPath, auditPolicy, 0o644); err != nil {
+		return fmt.Errorf("failed to write default audit policy: %w", err)
+	}
+
 	initConfigFile, err := os.CreateTemp("", "kubeadm-init.*.yaml")
 	if err != nil {
 		return fmt.Errorf("failed to create init config file %v: %w", initConfigFile.Name(), err)
@@ -102,6 +111,15 @@ func (k *KubernetesUtil) SetupCloudNodeManager(kubectl Client, cloudNodeManagerC
 
 // JoinCluster joins existing Kubernetes cluster using kubeadm join.
 func (k *KubernetesUtil) JoinCluster(joinConfig []byte) error {
+	// TODO: audit policy should be user input
+	auditPolicy, err := resources.NewDefaultAuditPolicy().Marshal()
+	if err != nil {
+		return fmt.Errorf("failed to generate default audit policy: %w", err)
+	}
+	if err := os.WriteFile(auditPolicyPath, auditPolicy, 0o644); err != nil {
+		return fmt.Errorf("failed to write default audit policy: %w", err)
+	}
+
 	joinConfigFile, err := os.CreateTemp("", "kubeadm-join.*.yaml")
 	if err != nil {
 		return fmt.Errorf("failed to create join config file %v: %w", joinConfigFile.Name(), err)

go.mod

@@ -208,6 +208,7 @@ require (
 	google.golang.org/appengine v1.6.7 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
+	k8s.io/apiserver v0.24.0
 	k8s.io/cluster-bootstrap v0.0.0 // indirect
 	k8s.io/component-base v0.24.0 // indirect
 	k8s.io/kube-openapi v0.0.0-20220328201542-3ee0da9b0b42 // indirect

go.sum

@@ -2301,6 +2301,7 @@ k8s.io/api v0.24.0/go.mod h1:5Jl90IUrJHUJYEMANRURMiVvJ0g7Ax7r3R1bqO8zx8I=
 k8s.io/apiextensions-apiserver v0.24.0/go.mod h1:iuVe4aEpe6827lvO6yWQVxiPSpPoSKVjkq+MIdg84cM=
 k8s.io/apimachinery v0.24.0 h1:ydFCyC/DjCvFCHK5OPMKBlxayQytB8pxy8YQInd5UyQ=
 k8s.io/apimachinery v0.24.0/go.mod h1:82Bi4sCzVBdpYjyI4jY6aHX+YCUchUIrZrXKedjd2UM=
+k8s.io/apiserver v0.24.0 h1:GR7kGsjOMfilRvlG3Stxv/3uz/ryvJ/aZXc5pqdsNV0=
 k8s.io/apiserver v0.24.0/go.mod h1:WFx2yiOMawnogNToVvUYT9nn1jaIkMKj41ZYCVycsBA=
 k8s.io/cli-runtime v0.24.0 h1:ot3Qf49T852uEyNApABO1UHHpFIckKK/NqpheZYN2gM=
 k8s.io/cli-runtime v0.24.0/go.mod h1:9XxoZDsEkRFUThnwqNviqzljtT/LdHtNWvcNFrAXl0A=