package kubernetes import ( "context" "errors" "testing" "time" "github.com/edgelesssys/constellation/coordinator/kubernetes/k8sapi" "github.com/edgelesssys/constellation/coordinator/kubernetes/k8sapi/resources" "github.com/edgelesssys/constellation/coordinator/role" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "go.uber.org/goleak" kubeadm "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta3" "sigs.k8s.io/yaml" ) func TestMain(m *testing.M) { goleak.VerifyTestMain(m) } type stubClusterUtil struct { installComponentsErr error initClusterErr error setupPodNetworkErr error setupAutoscalingError error setupCloudControllerManagerError error setupCloudNodeManagerError error setupKMSError error joinClusterErr error startKubeletErr error restartKubeletErr error createJoinTokenResponse *kubeadm.BootstrapTokenDiscovery createJoinTokenErr error initConfigs [][]byte joinConfigs [][]byte } func (s *stubClusterUtil) InstallComponents(ctx context.Context, version string) error { return s.installComponentsErr } func (s *stubClusterUtil) InitCluster(initConfig []byte) error { s.initConfigs = append(s.initConfigs, initConfig) return s.initClusterErr } func (s *stubClusterUtil) SetupPodNetwork(kubectl k8sapi.Client, podNetworkConfiguration resources.Marshaler) error { return s.setupPodNetworkErr } func (s *stubClusterUtil) SetupAutoscaling(kubectl k8sapi.Client, clusterAutoscalerConfiguration resources.Marshaler, secrets resources.Marshaler) error { return s.setupAutoscalingError } func (s *stubClusterUtil) SetupCloudControllerManager(kubectl k8sapi.Client, cloudControllerManagerConfiguration resources.Marshaler, configMaps resources.Marshaler, secrets resources.Marshaler) error { return s.setupCloudControllerManagerError } func (s *stubClusterUtil) SetupKMS(kubectl k8sapi.Client, kmsDeployment resources.Marshaler) error { return s.setupKMSError } func (s *stubClusterUtil) SetupCloudNodeManager(kubectl k8sapi.Client, cloudNodeManagerConfiguration resources.Marshaler) error { return s.setupCloudNodeManagerError } func (s *stubClusterUtil) JoinCluster(joinConfig []byte) error { s.joinConfigs = append(s.joinConfigs, joinConfig) return s.joinClusterErr } func (s *stubClusterUtil) StartKubelet() error { return s.startKubeletErr } func (s *stubClusterUtil) RestartKubelet() error { return s.restartKubeletErr } func (s *stubClusterUtil) GetControlPlaneJoinCertificateKey() (string, error) { return "", nil } func (s *stubClusterUtil) CreateJoinToken(ttl time.Duration) (*kubeadm.BootstrapTokenDiscovery, error) { return s.createJoinTokenResponse, s.createJoinTokenErr } type stubConfigProvider struct { InitConfig k8sapi.KubeadmInitYAML JoinConfig k8sapi.KubeadmJoinYAML } func (s *stubConfigProvider) InitConfiguration(_ bool) k8sapi.KubeadmInitYAML { return s.InitConfig } func (s *stubConfigProvider) JoinConfiguration(_ bool) k8sapi.KubeadmJoinYAML { s.JoinConfig = k8sapi.KubeadmJoinYAML{ JoinConfiguration: kubeadm.JoinConfiguration{ Discovery: kubeadm.Discovery{ BootstrapToken: &kubeadm.BootstrapTokenDiscovery{}, }, }, } return s.JoinConfig } type stubKubeCTL struct { ApplyErr error resources []resources.Marshaler kubeconfigs [][]byte } func (s *stubKubeCTL) Apply(resources resources.Marshaler, forceConflicts bool) error { s.resources = append(s.resources, resources) return s.ApplyErr } func (s *stubKubeCTL) SetKubeconfig(kubeconfig []byte) { s.kubeconfigs = append(s.kubeconfigs, kubeconfig) } type stubKubeconfigReader struct { Kubeconfig []byte ReadErr error } func (s *stubKubeconfigReader) ReadKubeconfig() ([]byte, error) { return s.Kubeconfig, s.ReadErr } func TestInitCluster(t *testing.T) { someErr := errors.New("failed") coordinatorVPNIP := "192.0.2.0" coordinatorProviderID := "somecloudprovider://instance-id" instanceName := "instance-id" supportsClusterAutoscaler := false cloudprovider := "some-cloudprovider" cloudControllerManagerImage := "some-image:latest" cloudControllerManagerPath := "/some_path" autoscalingNodeGroups := []string{"0,10,autoscaling_group_0"} testCases := map[string]struct { clusterUtil stubClusterUtil kubeCTL stubKubeCTL kubeconfigReader stubKubeconfigReader initConfig k8sapi.KubeadmInitYAML joinConfig k8sapi.KubeadmJoinYAML wantErr bool }{ "kubeadm init works": { clusterUtil: stubClusterUtil{}, kubeconfigReader: stubKubeconfigReader{ Kubeconfig: []byte("someKubeconfig"), }, wantErr: false, }, "kubeadm init errors": { clusterUtil: stubClusterUtil{initClusterErr: someErr}, kubeconfigReader: stubKubeconfigReader{ Kubeconfig: []byte("someKubeconfig"), }, wantErr: true, }, "pod network setup errors": { clusterUtil: stubClusterUtil{setupPodNetworkErr: someErr}, kubeconfigReader: stubKubeconfigReader{ Kubeconfig: []byte("someKubeconfig"), }, wantErr: true, }, } for name, tc := range testCases { t.Run(name, func(t *testing.T) { assert := assert.New(t) require := require.New(t) kube := KubeWrapper{ clusterUtil: &tc.clusterUtil, configProvider: &stubConfigProvider{InitConfig: k8sapi.KubeadmInitYAML{}}, client: &tc.kubeCTL, kubeconfigReader: &tc.kubeconfigReader, } err := kube.InitCluster( InitClusterInput{ APIServerAdvertiseIP: coordinatorVPNIP, NodeName: instanceName, ProviderID: coordinatorProviderID, SupportClusterAutoscaler: supportsClusterAutoscaler, AutoscalingCloudprovider: cloudprovider, AutoscalingNodeGroups: autoscalingNodeGroups, CloudControllerManagerName: cloudprovider, CloudControllerManagerImage: cloudControllerManagerImage, CloudControllerManagerPath: cloudControllerManagerPath, }, ) if tc.wantErr { assert.Error(err) return } require.NoError(err) var kubeadmConfig k8sapi.KubeadmInitYAML require.NoError(resources.UnmarshalK8SResources(tc.clusterUtil.initConfigs[0], &kubeadmConfig)) assert.Equal(kubeadmConfig.InitConfiguration.LocalAPIEndpoint.AdvertiseAddress, "192.0.2.0") assert.Equal(kubeadmConfig.ClusterConfiguration.Networking.PodSubnet, "10.244.0.0/16") }) } } func TestJoinCluster(t *testing.T) { someErr := errors.New("failed") joinCommand := &kubeadm.BootstrapTokenDiscovery{ APIServerEndpoint: "192.0.2.0:6443", Token: "kube-fake-token", CACertHashes: []string{"sha256:a60ebe9b0879090edd83b40a4df4bebb20506bac1e51d518ff8f4505a721930f"}, } nodeVPNIP := "192.0.2.0" coordinatorProviderID := "somecloudprovider://instance-id" instanceName := "instance-id" client := fakeK8SClient{} testCases := map[string]struct { clusterUtil stubClusterUtil wantErr bool }{ "kubeadm join works": { clusterUtil: stubClusterUtil{}, wantErr: false, }, "kubeadm join errors": { clusterUtil: stubClusterUtil{joinClusterErr: someErr}, wantErr: true, }, } for name, tc := range testCases { t.Run(name, func(t *testing.T) { assert := assert.New(t) require := require.New(t) kube := New(&tc.clusterUtil, &stubConfigProvider{}, &client) err := kube.JoinCluster(joinCommand, instanceName, nodeVPNIP, nodeVPNIP, coordinatorProviderID, "", true, role.Node) if tc.wantErr { assert.Error(err) return } require.NoError(err) var joinConfig kubeadm.JoinConfiguration require.NoError(yaml.Unmarshal(tc.clusterUtil.joinConfigs[0], &joinConfig)) assert.Equal("192.0.2.0:6443", joinConfig.Discovery.BootstrapToken.APIServerEndpoint) assert.Equal("kube-fake-token", joinConfig.Discovery.BootstrapToken.Token) assert.Equal([]string{"sha256:a60ebe9b0879090edd83b40a4df4bebb20506bac1e51d518ff8f4505a721930f"}, joinConfig.Discovery.BootstrapToken.CACertHashes) assert.Equal(map[string]string{"node-ip": "192.0.2.0"}, joinConfig.NodeRegistration.KubeletExtraArgs) }) } } func TestGetKubeconfig(t *testing.T) { testCases := map[string]struct { Kubewrapper KubeWrapper wantErr bool }{ "check single replacement": { Kubewrapper: KubeWrapper{kubeconfigReader: &stubKubeconfigReader{ Kubeconfig: []byte("127.0.0.1:16443"), }}, }, "check multiple replacement": { Kubewrapper: KubeWrapper{kubeconfigReader: &stubKubeconfigReader{ Kubeconfig: []byte("127.0.0.1:16443...127.0.0.1:16443"), }}, }, } for name, tc := range testCases { t.Run(name, func(t *testing.T) { assert := assert.New(t) require := require.New(t) data, err := tc.Kubewrapper.GetKubeconfig() require.NoError(err) assert.NotContains(string(data), "127.0.0.1:16443") assert.Contains(string(data), "10.118.0.1:6443") }) } }