Constellation conformance mode (#161)

* add conformance mode

CHANGELOG.md

@@ -23,6 +23,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ### Added
 
 - Loadbalancer for control-plane recovery
+- K8s conformance mode
 
 ### Changed
 <!-- For changes in existing functionality.  -->

bootstrapper/cmd/bootstrapper/test.go

@@ -22,7 +22,7 @@ type clusterFake struct{}
 // InitCluster fakes bootstrapping a new cluster with the current node being the master, returning the arguments required to join the cluster.
 func (c *clusterFake) InitCluster(
 	context.Context, []string, string, string, []byte, []uint32, bool, []byte, bool,
-	resources.KMSConfig, map[string]string, []byte, *logger.Logger,
+	resources.KMSConfig, map[string]string, []byte, bool, *logger.Logger,
 ) ([]byte, error) {
 	return []byte{}, nil
 }

bootstrapper/initproto/init.pb.go

@@ -38,6 +38,7 @@ type InitRequest struct {
 	HelmDeployments        []byte        `protobuf:"bytes,11,opt,name=helm_deployments,json=helmDeployments,proto3" json:"helm_deployments,omitempty"`
 	EnforcedPcrs           []uint32      `protobuf:"varint,12,rep,packed,name=enforced_pcrs,json=enforcedPcrs,proto3" json:"enforced_pcrs,omitempty"`
 	EnforceIdkeydigest     bool          `protobuf:"varint,13,opt,name=enforce_idkeydigest,json=enforceIdkeydigest,proto3" json:"enforce_idkeydigest,omitempty"`
+	ConformanceMode        bool          `protobuf:"varint,14,opt,name=conformance_mode,json=conformanceMode,proto3" json:"conformance_mode,omitempty"`
 }
 
 func (x *InitRequest) Reset() {
@@ -163,6 +164,13 @@ func (x *InitRequest) GetEnforceIdkeydigest() bool {
 	return false
 }
 
+func (x *InitRequest) GetConformanceMode() bool {
+	if x != nil {
+		return x.ConformanceMode
+	}
+	return false
+}
+
 type InitResponse struct {
 	state         protoimpl.MessageState
 	sizeCache     protoimpl.SizeCache
@@ -285,7 +293,7 @@ var File_init_proto protoreflect.FileDescriptor
 
 var file_init_proto_rawDesc = []byte{
 	0x0a, 0x0a, 0x69, 0x6e, 0x69, 0x74, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x04, 0x69, 0x6e,
-	0x69, 0x74, 0x22, 0xb6, 0x04, 0x0a, 0x0b, 0x49, 0x6e, 0x69, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65,
+	0x69, 0x74, 0x22, 0xe1, 0x04, 0x0a, 0x0b, 0x49, 0x6e, 0x69, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65,
 	0x73, 0x74, 0x12, 0x36, 0x0a, 0x17, 0x61, 0x75, 0x74, 0x6f, 0x73, 0x63, 0x61, 0x6c, 0x69, 0x6e,
 	0x67, 0x5f, 0x6e, 0x6f, 0x64, 0x65, 0x5f, 0x67, 0x72, 0x6f, 0x75, 0x70, 0x73, 0x18, 0x01, 0x20,
 	0x03, 0x28, 0x09, 0x52, 0x15, 0x61, 0x75, 0x74, 0x6f, 0x73, 0x63, 0x61, 0x6c, 0x69, 0x6e, 0x67,
@@ -320,26 +328,29 @@ var file_init_proto_rawDesc = []byte{
 	0x66, 0x6f, 0x72, 0x63, 0x65, 0x64, 0x50, 0x63, 0x72, 0x73, 0x12, 0x2f, 0x0a, 0x13, 0x65, 0x6e,
 	0x66, 0x6f, 0x72, 0x63, 0x65, 0x5f, 0x69, 0x64, 0x6b, 0x65, 0x79, 0x64, 0x69, 0x67, 0x65, 0x73,
 	0x74, 0x18, 0x0d, 0x20, 0x01, 0x28, 0x08, 0x52, 0x12, 0x65, 0x6e, 0x66, 0x6f, 0x72, 0x63, 0x65,
-	0x49, 0x64, 0x6b, 0x65, 0x79, 0x64, 0x69, 0x67, 0x65, 0x73, 0x74, 0x22, 0x68, 0x0a, 0x0c, 0x49,
+	0x49, 0x64, 0x6b, 0x65, 0x79, 0x64, 0x69, 0x67, 0x65, 0x73, 0x74, 0x12, 0x29, 0x0a, 0x10, 0x63,
-	0x6e, 0x69, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x1e, 0x0a, 0x0a, 0x6b,
+	0x6f, 0x6e, 0x66, 0x6f, 0x72, 0x6d, 0x61, 0x6e, 0x63, 0x65, 0x5f, 0x6d, 0x6f, 0x64, 0x65, 0x18,
-	0x75, 0x62, 0x65, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52,
+	0x0e, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0f, 0x63, 0x6f, 0x6e, 0x66, 0x6f, 0x72, 0x6d, 0x61, 0x6e,
-	0x0a, 0x6b, 0x75, 0x62, 0x65, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x19, 0x0a, 0x08, 0x6f,
+	0x63, 0x65, 0x4d, 0x6f, 0x64, 0x65, 0x22, 0x68, 0x0a, 0x0c, 0x49, 0x6e, 0x69, 0x74, 0x52, 0x65,
-	0x77, 0x6e, 0x65, 0x72, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x07, 0x6f,
+	0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x1e, 0x0a, 0x0a, 0x6b, 0x75, 0x62, 0x65, 0x63, 0x6f,
-	0x77, 0x6e, 0x65, 0x72, 0x49, 0x64, 0x12, 0x1d, 0x0a, 0x0a, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65,
+	0x6e, 0x66, 0x69, 0x67, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0a, 0x6b, 0x75, 0x62, 0x65,
-	0x72, 0x5f, 0x69, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x09, 0x63, 0x6c, 0x75, 0x73,
+	0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x19, 0x0a, 0x08, 0x6f, 0x77, 0x6e, 0x65, 0x72, 0x5f,
-	0x74, 0x65, 0x72, 0x49, 0x64, 0x22, 0x47, 0x0a, 0x0a, 0x53, 0x53, 0x48, 0x55, 0x73, 0x65, 0x72,
+	0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x07, 0x6f, 0x77, 0x6e, 0x65, 0x72, 0x49,
-	0x4b, 0x65, 0x79, 0x12, 0x1a, 0x0a, 0x08, 0x75, 0x73, 0x65, 0x72, 0x6e, 0x61, 0x6d, 0x65, 0x18,
+	0x64, 0x12, 0x1d, 0x0a, 0x0a, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x69, 0x64, 0x18,
-	0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x75, 0x73, 0x65, 0x72, 0x6e, 0x61, 0x6d, 0x65, 0x12,
+	0x03, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x09, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x49, 0x64,
-	0x1d, 0x0a, 0x0a, 0x70, 0x75, 0x62, 0x6c, 0x69, 0x63, 0x5f, 0x6b, 0x65, 0x79, 0x18, 0x02, 0x20,
+	0x22, 0x47, 0x0a, 0x0a, 0x53, 0x53, 0x48, 0x55, 0x73, 0x65, 0x72, 0x4b, 0x65, 0x79, 0x12, 0x1a,
-	0x01, 0x28, 0x09, 0x52, 0x09, 0x70, 0x75, 0x62, 0x6c, 0x69, 0x63, 0x4b, 0x65, 0x79, 0x32, 0x34,
+	0x0a, 0x08, 0x75, 0x73, 0x65, 0x72, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09,
-	0x0a, 0x03, 0x41, 0x50, 0x49, 0x12, 0x2d, 0x0a, 0x04, 0x49, 0x6e, 0x69, 0x74, 0x12, 0x11, 0x2e,
+	0x52, 0x08, 0x75, 0x73, 0x65, 0x72, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x1d, 0x0a, 0x0a, 0x70, 0x75,
-	0x69, 0x6e, 0x69, 0x74, 0x2e, 0x49, 0x6e, 0x69, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74,
+	0x62, 0x6c, 0x69, 0x63, 0x5f, 0x6b, 0x65, 0x79, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09,
-	0x1a, 0x12, 0x2e, 0x69, 0x6e, 0x69, 0x74, 0x2e, 0x49, 0x6e, 0x69, 0x74, 0x52, 0x65, 0x73, 0x70,
+	0x70, 0x75, 0x62, 0x6c, 0x69, 0x63, 0x4b, 0x65, 0x79, 0x32, 0x34, 0x0a, 0x03, 0x41, 0x50, 0x49,
-	0x6f, 0x6e, 0x73, 0x65, 0x42, 0x3d, 0x5a, 0x3b, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63,
+	0x12, 0x2d, 0x0a, 0x04, 0x49, 0x6e, 0x69, 0x74, 0x12, 0x11, 0x2e, 0x69, 0x6e, 0x69, 0x74, 0x2e,
-	0x6f, 0x6d, 0x2f, 0x65, 0x64, 0x67, 0x65, 0x6c, 0x65, 0x73, 0x73, 0x73, 0x79, 0x73, 0x2f, 0x63,
+	0x49, 0x6e, 0x69, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x12, 0x2e, 0x69, 0x6e,
-	0x6f, 0x6e, 0x73, 0x74, 0x65, 0x6c, 0x6c, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x2f, 0x62, 0x6f, 0x6f,
+	0x69, 0x74, 0x2e, 0x49, 0x6e, 0x69, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x42,
-	0x74, 0x73, 0x74, 0x72, 0x61, 0x70, 0x70, 0x65, 0x72, 0x2f, 0x69, 0x6e, 0x69, 0x74, 0x70, 0x72,
+	0x3d, 0x5a, 0x3b, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x65, 0x64,
-	0x6f, 0x74, 0x6f, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
+	0x67, 0x65, 0x6c, 0x65, 0x73, 0x73, 0x73, 0x79, 0x73, 0x2f, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x65,
+	0x6c, 0x6c, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x2f, 0x62, 0x6f, 0x6f, 0x74, 0x73, 0x74, 0x72, 0x61,
+	0x70, 0x70, 0x65, 0x72, 0x2f, 0x69, 0x6e, 0x69, 0x74, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x06,
+	0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
 }
 
 var (

bootstrapper/initproto/init.proto

@@ -22,6 +22,7 @@ message InitRequest {
   bytes helm_deployments = 11;
   repeated uint32 enforced_pcrs = 12;
   bool enforce_idkeydigest = 13;
+  bool conformance_mode = 14;
 }
 
 message InitResponse {

bootstrapper/internal/initserver/initserver.go

@@ -143,6 +143,7 @@ func (s *Server) Init(ctx context.Context, req *initproto.InitRequest) (*initpro
 		},
 		sshProtoKeysToMap(req.SshUserKeys),
 		req.HelmDeployments,
+		req.ConformanceMode,
 		s.log,
 	)
 	if err != nil {
@@ -244,6 +245,7 @@ type ClusterInitializer interface {
 		kmsConfig resources.KMSConfig,
 		sshUserKeys map[string]string,
 		helmDeployments []byte,
+		conformanceMode bool,
 		log *logger.Logger,
 	) ([]byte, error)
 }

bootstrapper/internal/initserver/initserver_test.go

@@ -290,7 +290,7 @@ type stubClusterInitializer struct {
 
 func (i *stubClusterInitializer) InitCluster(
 	context.Context, []string, string, string, []byte, []uint32, bool, []byte, bool,
-	resources.KMSConfig, map[string]string, []byte, *logger.Logger,
+	resources.KMSConfig, map[string]string, []byte, bool, *logger.Logger,
 ) ([]byte, error) {
 	return i.initClusterKubeconfig, i.initClusterErr
 }

bootstrapper/internal/kubernetes/k8sapi/util.go

@@ -131,7 +131,7 @@ func (k *KubernetesUtil) InstallComponents(ctx context.Context, version versions
 }
 
 func (k *KubernetesUtil) InitCluster(
-	ctx context.Context, initConfig []byte, nodeName string, ips []net.IP, controlPlaneEndpoint string, log *logger.Logger,
+	ctx context.Context, initConfig []byte, nodeName string, ips []net.IP, controlPlaneEndpoint string, conformanceMode bool, log *logger.Logger,
 ) error {
 	// TODO: audit policy should be user input
 	auditPolicy, err := resources.NewDefaultAuditPolicy().Marshal()
@@ -188,7 +188,12 @@ func (k *KubernetesUtil) InitCluster(
 
 	// initialize the cluster
 	log.Infof("Initializing the cluster using kubeadm init")
-	cmd = exec.CommandContext(ctx, kubeadmPath, "init", "-v=5", "--skip-phases=preflight,certs,addon/kube-proxy", "--config", initConfigFile.Name())
+	skipPhases := "--skip-phases=preflight,certs"
+	if !conformanceMode {
+		skipPhases += ",addon/kube-proxy"
+	}
+
+	cmd = exec.CommandContext(ctx, kubeadmPath, "init", "-v=5", skipPhases, "--config", initConfigFile.Name())
 	out, err = cmd.CombinedOutput()
 	if err != nil {
 		var exitErr *exec.ExitError

bootstrapper/internal/kubernetes/k8sutil.go

@@ -19,7 +19,7 @@ import (
 
 type clusterUtil interface {
 	InstallComponents(ctx context.Context, version versions.ValidK8sVersion) error
-	InitCluster(ctx context.Context, initConfig []byte, nodeName string, ips []net.IP, controlPlaneEndpoint string, log *logger.Logger) error
+	InitCluster(ctx context.Context, initConfig []byte, nodeName string, ips []net.IP, controlPlaneEndpoint string, conformanceMode bool, log *logger.Logger) error
 	JoinCluster(ctx context.Context, joinConfig []byte, peerRole role.Role, controlPlaneEndpoint string, log *logger.Logger) error
 	SetupHelmDeployments(ctx context.Context, client k8sapi.Client, helmDeployments []byte, in k8sapi.SetupPodNetworkInput, log *logger.Logger) error
 	SetupAccessManager(kubectl k8sapi.Client, sshUsers kubernetes.Marshaler) error

bootstrapper/internal/kubernetes/kubernetes.go

@@ -77,7 +77,7 @@ func New(cloudProvider string, clusterUtil clusterUtil, configProvider configura
 // InitCluster initializes a new Kubernetes cluster and applies pod network provider.
 func (k *KubeWrapper) InitCluster(
 	ctx context.Context, autoscalingNodeGroups []string, cloudServiceAccountURI, versionString string, measurementSalt []byte,
-	enforcedPCRs []uint32, enforceIdKeyDigest bool, idKeyDigest []byte, azureCVM bool, kmsConfig resources.KMSConfig, sshUsers map[string]string, helmDeployments []byte, log *logger.Logger,
+	enforcedPCRs []uint32, enforceIdKeyDigest bool, idKeyDigest []byte, azureCVM bool, kmsConfig resources.KMSConfig, sshUsers map[string]string, helmDeployments []byte, conformanceMode bool, log *logger.Logger,
 ) ([]byte, error) {
 	k8sVersion, err := versions.NewValidK8sVersion(versionString)
 	if err != nil {
@@ -155,7 +155,7 @@ func (k *KubeWrapper) InitCluster(
 		return nil, fmt.Errorf("encoding kubeadm init configuration as YAML: %w", err)
 	}
 	log.Infof("Initializing Kubernetes cluster")
-	if err := k.clusterUtil.InitCluster(ctx, initConfigYAML, nodeName, validIPs, controlPlaneEndpoint, log); err != nil {
+	if err := k.clusterUtil.InitCluster(ctx, initConfigYAML, nodeName, validIPs, controlPlaneEndpoint, conformanceMode, log); err != nil {
 		return nil, fmt.Errorf("kubeadm init: %w", err)
 	}
 	kubeConfig, err := k.GetKubeconfig()

bootstrapper/internal/kubernetes/kubernetes_test.go

@@ -320,7 +320,7 @@ func TestInitCluster(t *testing.T) {
 
 			_, err := kube.InitCluster(
 				context.Background(), autoscalingNodeGroups, serviceAccountURI, string(tc.k8sVersion),
-				nil, nil, false, nil, true, resources.KMSConfig{MasterSecret: masterSecret}, nil, nil, logger.NewTest(t),
+				nil, nil, false, nil, true, resources.KMSConfig{MasterSecret: masterSecret}, nil, nil, false, logger.NewTest(t),
 			)
 
 			if tc.wantErr {
@@ -557,7 +557,7 @@ func (s *stubClusterUtil) InstallComponents(ctx context.Context, version version
 	return s.installComponentsErr
 }
 
-func (s *stubClusterUtil) InitCluster(ctx context.Context, initConfig []byte, nodeName string, ips []net.IP, controlPlaneEndpoint string, log *logger.Logger) error {
+func (s *stubClusterUtil) InitCluster(ctx context.Context, initConfig []byte, nodeName string, ips []net.IP, controlPlaneEndpoint string, conformanceMode bool, log *logger.Logger) error {
 	s.initConfigs = append(s.initConfigs, initConfig)
 	return s.initClusterErr
 }

cli/internal/cmd/helmloader.go

@@ -7,13 +7,13 @@ SPDX-License-Identifier: AGPL-3.0-only
 package cmd
 
 type helmLoader interface {
-	Load(csp string) ([]byte, error)
+	Load(csp string, conformanceMode bool) ([]byte, error)
 }
 
 type stubHelmLoader struct {
 	loadErr error
 }
 
-func (d *stubHelmLoader) Load(csp string) ([]byte, error) {
+func (d *stubHelmLoader) Load(csp string, conformanceMode bool) ([]byte, error) {
 	return nil, d.loadErr
 }

cli/internal/cmd/init.go

@@ -56,6 +56,7 @@ func NewInitCmd() *cobra.Command {
 	cmd.Flags().String("master-secret", "", "path to base64-encoded master secret")
 	cmd.Flags().String("endpoint", "", "endpoint of the bootstrapper, passed as HOST[:PORT]")
 	cmd.Flags().Bool("autoscale", false, "enable Kubernetes cluster-autoscaler")
+	cmd.Flags().Bool("conformance", false, "enable conformance mode")
 	return cmd
 }
 
@@ -138,7 +139,7 @@ func initialize(cmd *cobra.Command, newDialer func(validator *cloudcmd.Validator
 		autoscalingNodeGroups = append(autoscalingNodeGroups, workers.GroupID)
 	}
 
-	helmDeployments, err := helmLoader.Load(stat.CloudProvider)
+	helmDeployments, err := helmLoader.Load(stat.CloudProvider, flags.conformance)
 	if err != nil {
 		return fmt.Errorf("loading Helm charts: %w", err)
 	}
@@ -163,6 +164,7 @@ func initialize(cmd *cobra.Command, newDialer func(validator *cloudcmd.Validator
 		HelmDeployments:        helmDeployments,
 		EnforcedPcrs:           getEnforcedMeasurements(provider, config),
 		EnforceIdkeydigest:     getEnforceIdKeyDigest(provider, config),
+		ConformanceMode:        flags.conformance,
 	}
 	resp, err := initCall(cmd.Context(), newDialer(validator), flags.endpoint, req)
 	if err != nil {
@@ -289,6 +291,10 @@ func evalFlagArgs(cmd *cobra.Command, fileHandler file.Handler) (initFlags, erro
 	if err != nil {
 		return initFlags{}, fmt.Errorf("parsing autoscale flag: %w", err)
 	}
+	conformance, err := cmd.Flags().GetBool("conformance")
+	if err != nil {
+		return initFlags{}, fmt.Errorf("parsing autoscale flag: %w", err)
+	}
 	configPath, err := cmd.Flags().GetString("config")
 	if err != nil {
 		return initFlags{}, fmt.Errorf("parsing config path flag: %w", err)
@@ -298,6 +304,7 @@ func evalFlagArgs(cmd *cobra.Command, fileHandler file.Handler) (initFlags, erro
 		configPath:       configPath,
 		endpoint:         endpoint,
 		autoscale:        autoscale,
+		conformance:      conformance,
 		masterSecretPath: masterSecretPath,
 	}, nil
 }
@@ -308,6 +315,7 @@ type initFlags struct {
 	masterSecretPath string
 	endpoint         string
 	autoscale        bool
+	conformance      bool
 }
 
 // masterSecret holds the master key and salt for deriving keys.

cli/internal/helm/loader.go

@@ -31,8 +31,8 @@ var HelmFS embed.FS
 
 type ChartLoader struct{}
 
-func (i *ChartLoader) Load(csp string) ([]byte, error) {
+func (i *ChartLoader) Load(csp string, conformanceMode bool) ([]byte, error) {
-	ciliumDeployment, err := i.loadCilium(csp)
+	ciliumDeployment, err := i.loadCilium(csp, conformanceMode)
 	if err != nil {
 		return nil, err
 	}
@@ -44,7 +44,7 @@ func (i *ChartLoader) Load(csp string) ([]byte, error) {
 	return depl, nil
 }
 
-func (i *ChartLoader) loadCilium(csp string) (helm.Deployment, error) {
+func (i *ChartLoader) loadCilium(csp string, conformanceMode bool) (helm.Deployment, error) {
 	chart, err := loadChartsDir(HelmFS, "charts/cilium")
 	if err != nil {
 		return helm.Deployment{}, err
@@ -60,6 +60,16 @@ func (i *ChartLoader) loadCilium(csp string) (helm.Deployment, error) {
 	default:
 		return helm.Deployment{}, fmt.Errorf("unknown csp: %s", csp)
 	}
+	if conformanceMode {
+		ciliumVals["kubeProxyReplacementHealthzBindAddr"] = ""
+		ciliumVals["kubeProxyReplacement"] = "partial"
+		ciliumVals["sessionAffinity"] = true
+		ciliumVals["cni"] = map[string]interface{}{
+			"chainingMode": "portmap",
+		}
+
+	}
+
 	return helm.Deployment{Chart: chart, Values: ciliumVals}, nil
 }