joinservice: read additional principals from ClusterConfig (#3900)

* joinservice: read additional principals from ClusterConfig

bootstrapper/internal/initserver/initserver.go

@@ -261,10 +261,6 @@ func (s *Server) Init(req *initproto.InitRequest, stream initproto.API_InitServe
 		return errors.Join(err, s.sendLogsWithMessage(stream, status.Errorf(codes.Internal, "generating SSH host certificate: %s", err)))
 	}
 
-	if err := s.fileHandler.Write(constants.SSHAdditionalPrincipalsPath, []byte(strings.Join(req.ApiserverCertSans, ",")), file.OptMkdirAll); err != nil {
-		return errors.Join(err, s.sendLogsWithMessage(stream, status.Errorf(codes.Internal, "writing list of public ssh principals: %s", err)))
-	}
-
 	if err := s.fileHandler.Write(constants.SSHHostCertificatePath, ssh.MarshalAuthorizedKey(hostCertificate), file.OptMkdirAll); err != nil {
 		return errors.Join(err, s.sendLogsWithMessage(stream, status.Errorf(codes.Internal, "writing ssh host certificate: %s", err)))
 	}

internal/constants/constants.go

@@ -50,8 +50,6 @@ const (
 	SSHHostKeyPath = "/var/run/state/ssh/ssh_host_ed25519_key"
 	// SSHHostCertificatePath is the path to the SSH host certificate.
 	SSHHostCertificatePath = "/var/run/state/ssh/ssh_host_cert.pub"
-	// SSHAdditionalPrincipalsPath stores additional principals (like the public IP of the load balancer) that get added to all host certificates.
-	SSHAdditionalPrincipalsPath = "/var/run/state/ssh/additional_principals.txt"
 
 	//
 	// Ports.

internal/constellation/helm/charts/edgeless/constellation-services/charts/join-service/templates/daemonset.yaml

@@ -50,6 +50,9 @@ spec:
             - mountPath: /etc/kubernetes
               name: kubeadm
               readOnly: true
+            - mountPath: /var/kubeadm-config
+              name: kubeadm-config
+              readOnly: true
             - mountPath: /var/secrets/google
               name: gcekey
               readOnly: true
@@ -76,6 +79,9 @@ spec:
         - name: kubeadm
           hostPath:
             path: /etc/kubernetes
+        - name: kubeadm-config
+          configMap:
+            name: kubeadm-config
         - name: ssh
           hostPath:
             path: /var/run/state/ssh

internal/constellation/helm/testdata/AWS/constellation-services/charts/join-service/templates/daemonset.yaml

@@ -50,6 +50,9 @@ spec:
             - mountPath: /etc/kubernetes
               name: kubeadm
               readOnly: true
+            - mountPath: /var/kubeadm-config
+              name: kubeadm-config
+              readOnly: true
             - mountPath: /var/secrets/google
               name: gcekey
               readOnly: true
@@ -76,6 +79,9 @@ spec:
         - name: kubeadm
           hostPath:
             path: /etc/kubernetes
+        - name: kubeadm-config
+          configMap:
+            name: kubeadm-config
         - name: ssh
           hostPath:
             path: /var/run/state/ssh

internal/constellation/helm/testdata/Azure/constellation-services/charts/join-service/templates/daemonset.yaml

@@ -50,6 +50,9 @@ spec:
             - mountPath: /etc/kubernetes
               name: kubeadm
               readOnly: true
+            - mountPath: /var/kubeadm-config
+              name: kubeadm-config
+              readOnly: true
             - mountPath: /var/secrets/google
               name: gcekey
               readOnly: true
@@ -76,6 +79,9 @@ spec:
         - name: kubeadm
           hostPath:
             path: /etc/kubernetes
+        - name: kubeadm-config
+          configMap:
+            name: kubeadm-config
         - name: ssh
           hostPath:
             path: /var/run/state/ssh

internal/constellation/helm/testdata/GCP/constellation-services/charts/join-service/templates/daemonset.yaml

@@ -50,6 +50,9 @@ spec:
             - mountPath: /etc/kubernetes
               name: kubeadm
               readOnly: true
+            - mountPath: /var/kubeadm-config
+              name: kubeadm-config
+              readOnly: true
             - mountPath: /var/secrets/google
               name: gcekey
               readOnly: true
@@ -76,6 +79,9 @@ spec:
         - name: kubeadm
           hostPath:
             path: /etc/kubernetes
+        - name: kubeadm-config
+          configMap:
+            name: kubeadm-config
         - name: ssh
           hostPath:
             path: /var/run/state/ssh

internal/constellation/helm/testdata/OpenStack/constellation-services/charts/join-service/templates/daemonset.yaml

@@ -50,6 +50,9 @@ spec:
             - mountPath: /etc/kubernetes
               name: kubeadm
               readOnly: true
+            - mountPath: /var/kubeadm-config
+              name: kubeadm-config
+              readOnly: true
             - mountPath: /var/secrets/google
               name: gcekey
               readOnly: true
@@ -76,6 +79,9 @@ spec:
         - name: kubeadm
           hostPath:
             path: /etc/kubernetes
+        - name: kubeadm-config
+          configMap:
+            name: kubeadm-config
         - name: ssh
           hostPath:
             path: /var/run/state/ssh

internal/constellation/helm/testdata/QEMU/constellation-services/charts/join-service/templates/daemonset.yaml

@@ -50,6 +50,9 @@ spec:
             - mountPath: /etc/kubernetes
               name: kubeadm
               readOnly: true
+            - mountPath: /var/kubeadm-config
+              name: kubeadm-config
+              readOnly: true
             - mountPath: /var/secrets/google
               name: gcekey
               readOnly: true
@@ -76,6 +79,9 @@ spec:
         - name: kubeadm
           hostPath:
             path: /etc/kubernetes
+        - name: kubeadm-config
+          configMap:
+            name: kubeadm-config
         - name: ssh
           hostPath:
             path: /var/run/state/ssh

joinservice/internal/server/BUILD.bazel

@@ -15,6 +15,7 @@ go_library(
         "//internal/logger",
         "//internal/versions/components",
         "//joinservice/joinproto",
+        "@in_gopkg_yaml_v3//:yaml_v3",
         "@io_k8s_kubernetes//cmd/kubeadm/app/apis/kubeadm/v1beta3",
         "@org_golang_google_grpc//:grpc",
         "@org_golang_google_grpc//codes",

joinservice/internal/server/server.go

@@ -13,7 +13,6 @@ import (
 	"fmt"
 	"log/slog"
 	"net"
-	"strings"
 	"time"
 
 	"github.com/edgelesssys/constellation/v2/internal/attestation"
@@ -29,6 +28,7 @@ import (
 	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/credentials"
 	"google.golang.org/grpc/status"
+	"gopkg.in/yaml.v3"
 
 	kubeadmv1 "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta3"
 )
@@ -119,13 +119,10 @@ func (s *Server) IssueJoinTicket(ctx context.Context, req *joinproto.IssueJoinTi
 		return nil, status.Errorf(codes.Internal, "generating ssh emergency CA key: %s", err)
 	}
 
-	principalList := req.HostCertificatePrincipals
+	principalList := s.extendPrincipals(req.HostCertificatePrincipals)
-	additionalPrincipals, err := s.fileHandler.Read(constants.SSHAdditionalPrincipalsPath)
+	if len(principalList) == 0 {
-	if err != nil {
+		principalList = append(principalList, grpclog.PeerAddrFromContext(ctx))
-		log.With(slog.Any("error", err)).Error("Failed to read additional principals file")
-		return nil, status.Errorf(codes.Internal, "reading additional principals file: %s", err)
 	}
-	principalList = append(principalList, strings.Split(string(additionalPrincipals), ",")...)
 
 	publicKey, err := ssh.ParsePublicKey(req.HostPublicKey)
 	if err != nil {
@@ -270,3 +267,48 @@ type kubeClient interface {
 	GetComponents(ctx context.Context, configMapName string) (components.Components, error)
 	AddNodeToJoiningNodes(ctx context.Context, nodeName string, componentsHash string, isControlPlane bool) error
 }
+
+func (s *Server) extendPrincipals(principals []string) []string {
+	clusterConfigYAML, err := s.fileHandler.Read("/var/kubeadm-config/ClusterConfiguration")
+	if err != nil {
+		s.log.Error("Failed to read kubeadm ClusterConfiguration file", "error", err)
+		return principals
+	}
+
+	var obj map[string]any
+	if err := yaml.Unmarshal(clusterConfigYAML, &obj); err != nil {
+		s.log.Error("Failed to unmarshal ClusterConfiguration file", "error", err)
+		return principals
+	}
+	apiServerAny, ok := obj["apiServer"]
+	if !ok {
+		s.log.Error("ClusterConfig has no apiServer field")
+		return principals
+	}
+	apiServerCfg, ok := apiServerAny.(map[string]any)
+	if !ok {
+		s.log.Error("Unexpected type of ClusterConfig.apiServer field", "type", fmt.Sprintf("%T", apiServerAny))
+		return principals
+	}
+	certSANsAny, ok := apiServerCfg["certSANs"]
+	if !ok {
+		s.log.Error("ClusterConfig.apiServer has no certSANs field")
+		return principals
+	}
+	certSANsListAny, ok := certSANsAny.([]any)
+	if !ok {
+		s.log.Error("Unexpected type of ClusterConfig.apiServer.certSANs field", "type", fmt.Sprintf("%T", certSANsAny))
+		return principals
+	}
+	// Don't append into the input slice.
+	principals = append([]string{}, principals...)
+	for i, sanAny := range certSANsListAny {
+		san, ok := sanAny.(string)
+		if !ok {
+			s.log.Error("Unexpected type of ClusterConfig.apiServer.certSANs field", "index", i, "type", fmt.Sprintf("%T", sanAny))
+		}
+		principals = append(principals, san)
+	}
+
+	return principals
+}

joinservice/internal/server/server_test.go

@@ -199,7 +199,6 @@ func TestIssueJoinTicket(t *testing.T) {
 			ca:                              stubCA{cert: testCert, nodeName: "node"},
 			kubeClient:                      stubKubeClient{getComponentsVal: clusterComponents, getK8sComponentsRefFromNodeVersionCRDVal: "k8s-components-ref"},
 			missingAdditionalPrincipalsFile: true,
-			wantErr:                         true,
 		},
 		"Host pubkey is missing": {
 			kubeadm: stubTokenGetter{token: testJoinToken},
@@ -224,7 +223,7 @@ func TestIssueJoinTicket(t *testing.T) {
 
 			fh := file.NewHandler(afero.NewMemMapFs())
 			if !tc.missingAdditionalPrincipalsFile {
-				require.NoError(fh.Write(constants.SSHAdditionalPrincipalsPath, []byte("*"), file.OptMkdirAll))
+				require.NoError(fh.Write("/var/kubeadm-config/ClusterConfiguration", []byte(clusterConfig), file.OptMkdirAll))
 			}
 
 			api := Server{
@@ -391,3 +390,70 @@ func (s *stubKubeClient) AddNodeToJoiningNodes(_ context.Context, nodeName strin
 	s.componentsRef = componentsRef
 	return s.addNodeToJoiningNodesErr
 }
+
+const clusterConfig = `
+apiServer:
+  certSANs:
+  - "*"
+  extraArgs:
+  - name: audit-log-maxage
+    value: "30"
+  - name: audit-log-maxbackup
+    value: "10"
+  - name: audit-log-maxsize
+    value: "100"
+  - name: audit-log-path
+    value: /var/log/kubernetes/audit/audit.log
+  - name: audit-policy-file
+    value: /etc/kubernetes/audit-policy.yaml
+  - name: kubelet-certificate-authority
+    value: /etc/kubernetes/pki/ca.crt
+  - name: profiling
+    value: "false"
+  - name: tls-cipher-suites
+    value: TLS_AES_128_GCM_SHA256,TLS_AES_256_GCM_SHA384,TLS_CHACHA20_POLY1305_SHA256,TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256,TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256,TLS_RSA_WITH_3DES_EDE_CBC_SHA,TLS_RSA_WITH_AES_128_CBC_SHA,TLS_RSA_WITH_AES_128_GCM_SHA256,TLS_RSA_WITH_AES_256_CBC_SHA,TLS_RSA_WITH_AES_256_GCM_SHA384
+  extraVolumes:
+  - hostPath: /var/log/kubernetes/audit/
+    mountPath: /var/log/kubernetes/audit/
+    name: audit-log
+    pathType: DirectoryOrCreate
+  - hostPath: /etc/kubernetes/audit-policy.yaml
+    mountPath: /etc/kubernetes/audit-policy.yaml
+    name: audit
+    pathType: File
+    readOnly: true
+apiVersion: kubeadm.k8s.io/v1beta4
+caCertificateValidityPeriod: 87600h0m0s
+certificateValidityPeriod: 8760h0m0s
+certificatesDir: /etc/kubernetes/pki
+clusterName: mr-cilium-7d6460ea
+controlPlaneEndpoint: 34.8.0.20:6443
+controllerManager:
+  extraArgs:
+  - name: cloud-provider
+    value: external
+  - name: configure-cloud-routes
+    value: "false"
+  - name: flex-volume-plugin-dir
+    value: /opt/libexec/kubernetes/kubelet-plugins/volume/exec/
+  - name: profiling
+    value: "false"
+  - name: terminated-pod-gc-threshold
+    value: "1000"
+dns: {}
+encryptionAlgorithm: RSA-2048
+etcd:
+  local:
+    dataDir: /var/lib/etcd
+imageRepository: registry.k8s.io
+kind: ClusterConfiguration
+kubernetesVersion: v1.30.14
+networking:
+  dnsDomain: cluster.local
+  serviceSubnet: 10.96.0.0/12
+proxy: {}
+scheduler:
+  extraArgs:
+  - name: profiling
+    value: "false"
+`