From 57874454f72c92685ecbb44d47255f0dd98f7e66 Mon Sep 17 00:00:00 2001 From: Markus Rudy Date: Thu, 31 Jul 2025 15:55:07 +0200 Subject: [PATCH] joinservice: read additional principals from ClusterConfig (#3900) * joinservice: read additional principals from ClusterConfig --- .../internal/initserver/initserver.go | 4 -- internal/constants/constants.go | 2 - .../join-service/templates/daemonset.yaml | 6 ++ .../join-service/templates/daemonset.yaml | 6 ++ .../join-service/templates/daemonset.yaml | 6 ++ .../join-service/templates/daemonset.yaml | 6 ++ .../join-service/templates/daemonset.yaml | 6 ++ .../join-service/templates/daemonset.yaml | 6 ++ joinservice/internal/server/BUILD.bazel | 1 + joinservice/internal/server/server.go | 56 +++++++++++++-- joinservice/internal/server/server_test.go | 70 ++++++++++++++++++- 11 files changed, 154 insertions(+), 15 deletions(-) diff --git a/bootstrapper/internal/initserver/initserver.go b/bootstrapper/internal/initserver/initserver.go index 4b75c6e3e..5f118d8c0 100644 --- a/bootstrapper/internal/initserver/initserver.go +++ b/bootstrapper/internal/initserver/initserver.go @@ -261,10 +261,6 @@ func (s *Server) Init(req *initproto.InitRequest, stream initproto.API_InitServe return errors.Join(err, s.sendLogsWithMessage(stream, status.Errorf(codes.Internal, "generating SSH host certificate: %s", err))) } - if err := s.fileHandler.Write(constants.SSHAdditionalPrincipalsPath, []byte(strings.Join(req.ApiserverCertSans, ",")), file.OptMkdirAll); err != nil { - return errors.Join(err, s.sendLogsWithMessage(stream, status.Errorf(codes.Internal, "writing list of public ssh principals: %s", err))) - } - if err := s.fileHandler.Write(constants.SSHHostCertificatePath, ssh.MarshalAuthorizedKey(hostCertificate), file.OptMkdirAll); err != nil { return errors.Join(err, s.sendLogsWithMessage(stream, status.Errorf(codes.Internal, "writing ssh host certificate: %s", err))) } diff --git a/internal/constants/constants.go b/internal/constants/constants.go index 2ad4a775f..56fca9ef7 100644 --- a/internal/constants/constants.go +++ b/internal/constants/constants.go @@ -50,8 +50,6 @@ const ( SSHHostKeyPath = "/var/run/state/ssh/ssh_host_ed25519_key" // SSHHostCertificatePath is the path to the SSH host certificate. SSHHostCertificatePath = "/var/run/state/ssh/ssh_host_cert.pub" - // SSHAdditionalPrincipalsPath stores additional principals (like the public IP of the load balancer) that get added to all host certificates. - SSHAdditionalPrincipalsPath = "/var/run/state/ssh/additional_principals.txt" // // Ports. diff --git a/internal/constellation/helm/charts/edgeless/constellation-services/charts/join-service/templates/daemonset.yaml b/internal/constellation/helm/charts/edgeless/constellation-services/charts/join-service/templates/daemonset.yaml index a7bfe1656..f14515244 100644 --- a/internal/constellation/helm/charts/edgeless/constellation-services/charts/join-service/templates/daemonset.yaml +++ b/internal/constellation/helm/charts/edgeless/constellation-services/charts/join-service/templates/daemonset.yaml @@ -50,6 +50,9 @@ spec: - mountPath: /etc/kubernetes name: kubeadm readOnly: true + - mountPath: /var/kubeadm-config + name: kubeadm-config + readOnly: true - mountPath: /var/secrets/google name: gcekey readOnly: true @@ -76,6 +79,9 @@ spec: - name: kubeadm hostPath: path: /etc/kubernetes + - name: kubeadm-config + configMap: + name: kubeadm-config - name: ssh hostPath: path: /var/run/state/ssh diff --git a/internal/constellation/helm/testdata/AWS/constellation-services/charts/join-service/templates/daemonset.yaml b/internal/constellation/helm/testdata/AWS/constellation-services/charts/join-service/templates/daemonset.yaml index 046ae08e6..538883439 100644 --- a/internal/constellation/helm/testdata/AWS/constellation-services/charts/join-service/templates/daemonset.yaml +++ b/internal/constellation/helm/testdata/AWS/constellation-services/charts/join-service/templates/daemonset.yaml @@ -50,6 +50,9 @@ spec: - mountPath: /etc/kubernetes name: kubeadm readOnly: true + - mountPath: /var/kubeadm-config + name: kubeadm-config + readOnly: true - mountPath: /var/secrets/google name: gcekey readOnly: true @@ -76,6 +79,9 @@ spec: - name: kubeadm hostPath: path: /etc/kubernetes + - name: kubeadm-config + configMap: + name: kubeadm-config - name: ssh hostPath: path: /var/run/state/ssh diff --git a/internal/constellation/helm/testdata/Azure/constellation-services/charts/join-service/templates/daemonset.yaml b/internal/constellation/helm/testdata/Azure/constellation-services/charts/join-service/templates/daemonset.yaml index 80e151886..b6fcd3f6b 100644 --- a/internal/constellation/helm/testdata/Azure/constellation-services/charts/join-service/templates/daemonset.yaml +++ b/internal/constellation/helm/testdata/Azure/constellation-services/charts/join-service/templates/daemonset.yaml @@ -50,6 +50,9 @@ spec: - mountPath: /etc/kubernetes name: kubeadm readOnly: true + - mountPath: /var/kubeadm-config + name: kubeadm-config + readOnly: true - mountPath: /var/secrets/google name: gcekey readOnly: true @@ -76,6 +79,9 @@ spec: - name: kubeadm hostPath: path: /etc/kubernetes + - name: kubeadm-config + configMap: + name: kubeadm-config - name: ssh hostPath: path: /var/run/state/ssh diff --git a/internal/constellation/helm/testdata/GCP/constellation-services/charts/join-service/templates/daemonset.yaml b/internal/constellation/helm/testdata/GCP/constellation-services/charts/join-service/templates/daemonset.yaml index c16d77b4f..bbe9747ba 100644 --- a/internal/constellation/helm/testdata/GCP/constellation-services/charts/join-service/templates/daemonset.yaml +++ b/internal/constellation/helm/testdata/GCP/constellation-services/charts/join-service/templates/daemonset.yaml @@ -50,6 +50,9 @@ spec: - mountPath: /etc/kubernetes name: kubeadm readOnly: true + - mountPath: /var/kubeadm-config + name: kubeadm-config + readOnly: true - mountPath: /var/secrets/google name: gcekey readOnly: true @@ -76,6 +79,9 @@ spec: - name: kubeadm hostPath: path: /etc/kubernetes + - name: kubeadm-config + configMap: + name: kubeadm-config - name: ssh hostPath: path: /var/run/state/ssh diff --git a/internal/constellation/helm/testdata/OpenStack/constellation-services/charts/join-service/templates/daemonset.yaml b/internal/constellation/helm/testdata/OpenStack/constellation-services/charts/join-service/templates/daemonset.yaml index c77b395b1..e680ff691 100644 --- a/internal/constellation/helm/testdata/OpenStack/constellation-services/charts/join-service/templates/daemonset.yaml +++ b/internal/constellation/helm/testdata/OpenStack/constellation-services/charts/join-service/templates/daemonset.yaml @@ -50,6 +50,9 @@ spec: - mountPath: /etc/kubernetes name: kubeadm readOnly: true + - mountPath: /var/kubeadm-config + name: kubeadm-config + readOnly: true - mountPath: /var/secrets/google name: gcekey readOnly: true @@ -76,6 +79,9 @@ spec: - name: kubeadm hostPath: path: /etc/kubernetes + - name: kubeadm-config + configMap: + name: kubeadm-config - name: ssh hostPath: path: /var/run/state/ssh diff --git a/internal/constellation/helm/testdata/QEMU/constellation-services/charts/join-service/templates/daemonset.yaml b/internal/constellation/helm/testdata/QEMU/constellation-services/charts/join-service/templates/daemonset.yaml index 7e8443eb4..1bd150448 100644 --- a/internal/constellation/helm/testdata/QEMU/constellation-services/charts/join-service/templates/daemonset.yaml +++ b/internal/constellation/helm/testdata/QEMU/constellation-services/charts/join-service/templates/daemonset.yaml @@ -50,6 +50,9 @@ spec: - mountPath: /etc/kubernetes name: kubeadm readOnly: true + - mountPath: /var/kubeadm-config + name: kubeadm-config + readOnly: true - mountPath: /var/secrets/google name: gcekey readOnly: true @@ -76,6 +79,9 @@ spec: - name: kubeadm hostPath: path: /etc/kubernetes + - name: kubeadm-config + configMap: + name: kubeadm-config - name: ssh hostPath: path: /var/run/state/ssh diff --git a/joinservice/internal/server/BUILD.bazel b/joinservice/internal/server/BUILD.bazel index c7835f7ef..369d49f58 100644 --- a/joinservice/internal/server/BUILD.bazel +++ b/joinservice/internal/server/BUILD.bazel @@ -15,6 +15,7 @@ go_library( "//internal/logger", "//internal/versions/components", "//joinservice/joinproto", + "@in_gopkg_yaml_v3//:yaml_v3", "@io_k8s_kubernetes//cmd/kubeadm/app/apis/kubeadm/v1beta3", "@org_golang_google_grpc//:grpc", "@org_golang_google_grpc//codes", diff --git a/joinservice/internal/server/server.go b/joinservice/internal/server/server.go index e3df8dc5d..1bfefb76d 100644 --- a/joinservice/internal/server/server.go +++ b/joinservice/internal/server/server.go @@ -13,7 +13,6 @@ import ( "fmt" "log/slog" "net" - "strings" "time" "github.com/edgelesssys/constellation/v2/internal/attestation" @@ -29,6 +28,7 @@ import ( "google.golang.org/grpc/codes" "google.golang.org/grpc/credentials" "google.golang.org/grpc/status" + "gopkg.in/yaml.v3" kubeadmv1 "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta3" ) @@ -119,13 +119,10 @@ func (s *Server) IssueJoinTicket(ctx context.Context, req *joinproto.IssueJoinTi return nil, status.Errorf(codes.Internal, "generating ssh emergency CA key: %s", err) } - principalList := req.HostCertificatePrincipals - additionalPrincipals, err := s.fileHandler.Read(constants.SSHAdditionalPrincipalsPath) - if err != nil { - log.With(slog.Any("error", err)).Error("Failed to read additional principals file") - return nil, status.Errorf(codes.Internal, "reading additional principals file: %s", err) + principalList := s.extendPrincipals(req.HostCertificatePrincipals) + if len(principalList) == 0 { + principalList = append(principalList, grpclog.PeerAddrFromContext(ctx)) } - principalList = append(principalList, strings.Split(string(additionalPrincipals), ",")...) publicKey, err := ssh.ParsePublicKey(req.HostPublicKey) if err != nil { @@ -270,3 +267,48 @@ type kubeClient interface { GetComponents(ctx context.Context, configMapName string) (components.Components, error) AddNodeToJoiningNodes(ctx context.Context, nodeName string, componentsHash string, isControlPlane bool) error } + +func (s *Server) extendPrincipals(principals []string) []string { + clusterConfigYAML, err := s.fileHandler.Read("/var/kubeadm-config/ClusterConfiguration") + if err != nil { + s.log.Error("Failed to read kubeadm ClusterConfiguration file", "error", err) + return principals + } + + var obj map[string]any + if err := yaml.Unmarshal(clusterConfigYAML, &obj); err != nil { + s.log.Error("Failed to unmarshal ClusterConfiguration file", "error", err) + return principals + } + apiServerAny, ok := obj["apiServer"] + if !ok { + s.log.Error("ClusterConfig has no apiServer field") + return principals + } + apiServerCfg, ok := apiServerAny.(map[string]any) + if !ok { + s.log.Error("Unexpected type of ClusterConfig.apiServer field", "type", fmt.Sprintf("%T", apiServerAny)) + return principals + } + certSANsAny, ok := apiServerCfg["certSANs"] + if !ok { + s.log.Error("ClusterConfig.apiServer has no certSANs field") + return principals + } + certSANsListAny, ok := certSANsAny.([]any) + if !ok { + s.log.Error("Unexpected type of ClusterConfig.apiServer.certSANs field", "type", fmt.Sprintf("%T", certSANsAny)) + return principals + } + // Don't append into the input slice. + principals = append([]string{}, principals...) + for i, sanAny := range certSANsListAny { + san, ok := sanAny.(string) + if !ok { + s.log.Error("Unexpected type of ClusterConfig.apiServer.certSANs field", "index", i, "type", fmt.Sprintf("%T", sanAny)) + } + principals = append(principals, san) + } + + return principals +} diff --git a/joinservice/internal/server/server_test.go b/joinservice/internal/server/server_test.go index 214acfefd..883660dc3 100644 --- a/joinservice/internal/server/server_test.go +++ b/joinservice/internal/server/server_test.go @@ -199,7 +199,6 @@ func TestIssueJoinTicket(t *testing.T) { ca: stubCA{cert: testCert, nodeName: "node"}, kubeClient: stubKubeClient{getComponentsVal: clusterComponents, getK8sComponentsRefFromNodeVersionCRDVal: "k8s-components-ref"}, missingAdditionalPrincipalsFile: true, - wantErr: true, }, "Host pubkey is missing": { kubeadm: stubTokenGetter{token: testJoinToken}, @@ -224,7 +223,7 @@ func TestIssueJoinTicket(t *testing.T) { fh := file.NewHandler(afero.NewMemMapFs()) if !tc.missingAdditionalPrincipalsFile { - require.NoError(fh.Write(constants.SSHAdditionalPrincipalsPath, []byte("*"), file.OptMkdirAll)) + require.NoError(fh.Write("/var/kubeadm-config/ClusterConfiguration", []byte(clusterConfig), file.OptMkdirAll)) } api := Server{ @@ -391,3 +390,70 @@ func (s *stubKubeClient) AddNodeToJoiningNodes(_ context.Context, nodeName strin s.componentsRef = componentsRef return s.addNodeToJoiningNodesErr } + +const clusterConfig = ` +apiServer: + certSANs: + - "*" + extraArgs: + - name: audit-log-maxage + value: "30" + - name: audit-log-maxbackup + value: "10" + - name: audit-log-maxsize + value: "100" + - name: audit-log-path + value: /var/log/kubernetes/audit/audit.log + - name: audit-policy-file + value: /etc/kubernetes/audit-policy.yaml + - name: kubelet-certificate-authority + value: /etc/kubernetes/pki/ca.crt + - name: profiling + value: "false" + - name: tls-cipher-suites + value: TLS_AES_128_GCM_SHA256,TLS_AES_256_GCM_SHA384,TLS_CHACHA20_POLY1305_SHA256,TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256,TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256,TLS_RSA_WITH_3DES_EDE_CBC_SHA,TLS_RSA_WITH_AES_128_CBC_SHA,TLS_RSA_WITH_AES_128_GCM_SHA256,TLS_RSA_WITH_AES_256_CBC_SHA,TLS_RSA_WITH_AES_256_GCM_SHA384 + extraVolumes: + - hostPath: /var/log/kubernetes/audit/ + mountPath: /var/log/kubernetes/audit/ + name: audit-log + pathType: DirectoryOrCreate + - hostPath: /etc/kubernetes/audit-policy.yaml + mountPath: /etc/kubernetes/audit-policy.yaml + name: audit + pathType: File + readOnly: true +apiVersion: kubeadm.k8s.io/v1beta4 +caCertificateValidityPeriod: 87600h0m0s +certificateValidityPeriod: 8760h0m0s +certificatesDir: /etc/kubernetes/pki +clusterName: mr-cilium-7d6460ea +controlPlaneEndpoint: 34.8.0.20:6443 +controllerManager: + extraArgs: + - name: cloud-provider + value: external + - name: configure-cloud-routes + value: "false" + - name: flex-volume-plugin-dir + value: /opt/libexec/kubernetes/kubelet-plugins/volume/exec/ + - name: profiling + value: "false" + - name: terminated-pod-gc-threshold + value: "1000" +dns: {} +encryptionAlgorithm: RSA-2048 +etcd: + local: + dataDir: /var/lib/etcd +imageRepository: registry.k8s.io +kind: ClusterConfiguration +kubernetesVersion: v1.30.14 +networking: + dnsDomain: cluster.local + serviceSubnet: 10.96.0.0/12 +proxy: {} +scheduler: + extraArgs: + - name: profiling + value: "false" +`