constellation/bootstrapper/internal/kubernetes/k8sapi/kubectl/client/client_test.go

package client

import (
	"bytes"
	"context"
	"errors"
	"io"
	"net/http"
	"testing"

	"github.com/edgelesssys/constellation/bootstrapper/internal/kubernetes/k8sapi/resources"
	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
	"go.uber.org/goleak"
	"google.golang.org/protobuf/proto"
	appsv1 "k8s.io/api/apps/v1"
	corev1 "k8s.io/api/core/v1"
	k8s "k8s.io/api/core/v1"
	apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
	apiextensionsclientv1 "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset/typed/apiextensions/v1"
	"k8s.io/apimachinery/pkg/api/meta"
	"k8s.io/apimachinery/pkg/api/meta/testrestmapper"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/runtime"
	"k8s.io/apimachinery/pkg/runtime/schema"
	"k8s.io/apimachinery/pkg/runtime/serializer/json"
	"k8s.io/apimachinery/pkg/watch"
	"k8s.io/cli-runtime/pkg/resource"
	"k8s.io/client-go/kubernetes/fake"
	"k8s.io/client-go/kubernetes/scheme"
	restfake "k8s.io/client-go/rest/fake"
	"k8s.io/client-go/restmapper"
)

func TestMain(m *testing.M) {
	goleak.VerifyTestMain(m)
}

var (
	corev1GV        = schema.GroupVersion{Version: "v1"}
	nginxDeployment = &appsv1.Deployment{
		TypeMeta: metav1.TypeMeta{
			APIVersion: "apps/v1",
			Kind:       "Deployment",
		},
		ObjectMeta: metav1.ObjectMeta{
			Labels: map[string]string{
				"app": "nginx",
			},
			Name: "my-nginx",
		},
		Spec: appsv1.DeploymentSpec{
			Replicas: proto.Int32(3),
			Selector: &metav1.LabelSelector{
				MatchLabels: map[string]string{
					"app": "nginx",
				},
			},
			Template: k8s.PodTemplateSpec{
				ObjectMeta: metav1.ObjectMeta{
					Labels: map[string]string{
						"app": "nginx",
					},
				},
				Spec: k8s.PodSpec{
					Containers: []k8s.Container{
						{
							Name:  "nginx",
							Image: "nginx:1.14.2",
							Ports: []k8s.ContainerPort{
								{
									ContainerPort: 80,
								},
							},
						},
					},
				},
			},
		},
	}
	tolerationsDeployment = appsv1.Deployment{
		ObjectMeta: metav1.ObjectMeta{
			Namespace: "test-ns",
			Name:      "test-deployment",
		},
	}
	selectorsDeployment = appsv1.Deployment{
		ObjectMeta: metav1.ObjectMeta{
			Namespace: "test-ns",
			Name:      "test-deployment",
		},
		Spec: appsv1.DeploymentSpec{
			Template: k8s.PodTemplateSpec{
				Spec: k8s.PodSpec{
					NodeSelector: map[string]string{},
				},
			},
		},
	}
	nginxDeplJSON, _ = marshalJSON(nginxDeployment)
	nginxDeplYAML, _ = marshalYAML(nginxDeployment)
)

type unmarshableResource struct{}

func (*unmarshableResource) Marshal() ([]byte, error) {
	return nil, errors.New("someErr")
}

func stringBody(body string) io.ReadCloser {
	return io.NopCloser(bytes.NewReader([]byte(body)))
}

func fakeClientWith(t *testing.T, testName string, data map[string]string) resource.FakeClientFunc {
	return func(version schema.GroupVersion) (resource.RESTClient, error) {
		return &restfake.RESTClient{
			GroupVersion:         corev1GV,
			NegotiatedSerializer: scheme.Codecs.WithoutConversion(),
			Client: restfake.CreateHTTPClient(func(req *http.Request) (*http.Response, error) {
				p := req.URL.Path
				q := req.URL.RawQuery
				if len(q) != 0 {
					p = p + "?" + q
				}
				body, ok := data[p]
				if !ok {
					t.Fatalf("%s: unexpected request: %s (%s)\n%#v", testName, p, req.URL, req)
				}
				header := http.Header{}
				header.Set("Content-Type", runtime.ContentTypeJSON)
				return &http.Response{
					StatusCode: http.StatusOK,
					Header:     header,
					Body:       stringBody(body),
				}, nil
			}),
		}, nil
	}
}

func newClientWithFakes(t *testing.T, data map[string]string, objects ...runtime.Object) Client {
	clientset := fake.NewSimpleClientset(objects...)
	builder := resource.NewFakeBuilder(
		fakeClientWith(t, "", data),
		func() (meta.RESTMapper, error) {
			return testrestmapper.TestOnlyStaticRESTMapper(scheme.Scheme), nil
		},
		func() (restmapper.CategoryExpander, error) {
			return resource.FakeCategoryExpander, nil
		}).
		Unstructured()
	client := Client{
		clientset: clientset,
		builder:   builder,
	}
	return client
}

func failingClient() resource.FakeClientFunc {
	return func(version schema.GroupVersion) (resource.RESTClient, error) {
		return &restfake.RESTClient{
			GroupVersion:         corev1GV,
			NegotiatedSerializer: scheme.Codecs.WithoutConversion(),
			Resp:                 &http.Response{StatusCode: 501},
		}, nil
	}
}

func newFailingClient(objects ...runtime.Object) Client {
	clientset := fake.NewSimpleClientset(objects...)
	builder := resource.NewFakeBuilder(
		failingClient(),
		func() (meta.RESTMapper, error) {
			return testrestmapper.TestOnlyStaticRESTMapper(scheme.Scheme), nil
		},
		func() (restmapper.CategoryExpander, error) {
			return resource.FakeCategoryExpander, nil
		}).
		WithScheme(scheme.Scheme, scheme.Scheme.PrioritizedVersionsAllGroups()...)
	client := Client{
		clientset: clientset,
		builder:   builder,
	}
	return client
}

func marshalJSON(obj runtime.Object) ([]byte, error) {
	serializer := json.NewSerializer(json.DefaultMetaFactory, nil, nil, false)
	var buf bytes.Buffer
	if err := serializer.Encode(obj, &buf); err != nil {
		return nil, err
	}
	return buf.Bytes(), nil
}

func marshalYAML(obj runtime.Object) ([]byte, error) {
	serializer := json.NewYAMLSerializer(json.DefaultMetaFactory, nil, nil)
	var buf bytes.Buffer
	if err := serializer.Encode(obj, &buf); err != nil {
		return nil, err
	}
	return buf.Bytes(), nil
}

func TestApplyOneObject(t *testing.T) {
	testCases := map[string]struct {
		httpResponseData map[string]string
		wantObj          runtime.Object
		resourcesYAML    string
		failingClient    bool
		wantErr          bool
	}{
		"apply works": {
			httpResponseData: map[string]string{
				"/deployments/my-nginx?fieldManager=constellation-bootstrapper&force=true": string(nginxDeplJSON),
			},
			wantObj:       nginxDeployment,
			resourcesYAML: string(nginxDeplYAML),
			wantErr:       false,
		},
		"apply fails": {
			httpResponseData: map[string]string{},
			wantObj:          nginxDeployment,
			resourcesYAML:    string(nginxDeplYAML),
			failingClient:    true,
			wantErr:          true,
		},
	}

	for name, tc := range testCases {
		t.Run(name, func(t *testing.T) {
			assert := assert.New(t)
			require := require.New(t)

			var client Client
			if tc.failingClient {
				client = newFailingClient(tc.wantObj)
			} else {
				client = newClientWithFakes(t, tc.httpResponseData, tc.wantObj)
			}

			reader := bytes.NewReader([]byte(tc.resourcesYAML))
			res := client.builder.
				ContinueOnError().
				Stream(reader, "yaml").
				Flatten().
				Do()
			assert.NoError(res.Err())
			infos, err := res.Infos()
			assert.NoError(err)
			require.Len(infos, 1)

			err = client.ApplyOneObject(infos[0], true)

			if tc.wantErr {
				assert.Error(err)
				return
			}
			require.NoError(err)
		})
	}
}

func TestGetObjects(t *testing.T) {
	testCases := map[string]struct {
		wantResources    resources.Marshaler
		httpResponseData map[string]string
		resourcesYAML    string
		wantErr          bool
	}{
		"GetObjects works on cluster-autoscaler deployment": {
			wantResources: resources.NewDefaultAutoscalerDeployment(nil, nil, nil, ""),
			resourcesYAML: string(nginxDeplYAML),
			wantErr:       false,
		},
		"GetObjects works on cloud-controller-manager deployment": {
			wantResources: resources.NewDefaultCloudControllerManagerDeployment("someProvider", "someImage", "somePath", "someCIDR", nil, nil, nil, nil),
			resourcesYAML: string(nginxDeplYAML),
			wantErr:       false,
		},
		"GetObjects Marshal failure detected": {
			wantResources: &unmarshableResource{},
			resourcesYAML: string(nginxDeplYAML),
			wantErr:       true,
		},
	}

	for name, tc := range testCases {
		t.Run(name, func(t *testing.T) {
			assert := assert.New(t)
			require := require.New(t)

			client := newClientWithFakes(t, tc.httpResponseData)
			infos, err := client.GetObjects(tc.wantResources)

			if tc.wantErr {
				assert.Error(err)
				return
			}
			require.NoError(err)
			assert.NotNil(infos)
		})
	}
}

func TestAddTolerationsToDeployment(t *testing.T) {
	testCases := map[string]struct {
		namespace   string
		name        string
		tolerations []corev1.Toleration
		wantErr     bool
	}{
		"Success": {
			namespace: "test-ns",
			name:      "test-deployment",
		},
		"Specifying non-existent deployment fails": {
			namespace: "test-ns",
			name:      "wrong-name",
			wantErr:   true,
		},
		"Wrong namespace": {
			name:    "test-deployment",
			wantErr: true,
		},
	}

	for name, tc := range testCases {
		t.Run(name, func(t *testing.T) {
			assert := assert.New(t)
			require := require.New(t)

			client := newClientWithFakes(t, map[string]string{}, &tolerationsDeployment)
			err := client.AddTolerationsToDeployment(context.Background(), tc.tolerations, tc.name, tc.namespace)
			if tc.wantErr {
				assert.Error(err)
				return
			}
			require.NoError(err)
		})
	}
}

func TestAddNodeSelectorsToDeployment(t *testing.T) {
	testCases := map[string]struct {
		namespace string
		name      string
		selectors map[string]string
		wantErr   bool
	}{
		"Success": {
			namespace: "test-ns",
			name:      "test-deployment",
			selectors: map[string]string{"some-key": "some-value"},
		},
		"Specifying non-existent deployment fails": {
			namespace: "test-ns",
			name:      "wrong-name",
			wantErr:   true,
		},
		"Wrong namespace": {
			name:    "test-deployment",
			wantErr: true,
		},
	}

	for name, tc := range testCases {
		t.Run(name, func(t *testing.T) {
			assert := assert.New(t)
			require := require.New(t)

			client := newClientWithFakes(t, map[string]string{}, &selectorsDeployment)
			err := client.AddNodeSelectorsToDeployment(context.Background(), tc.selectors, tc.name, tc.namespace)
			if tc.wantErr {
				assert.Error(err)
				return
			}
			require.NoError(err)
		})
	}
}

func TestWaitForCRD(t *testing.T) {
	testCases := map[string]struct {
		crd      string
		events   []watch.Event
		watchErr error
		wantErr  bool
	}{
		"Success": {
			crd: "test-crd",
			events: []watch.Event{
				{
					Type: watch.Added,
					Object: &apiextensionsv1.CustomResourceDefinition{
						Status: apiextensionsv1.CustomResourceDefinitionStatus{
							Conditions: []apiextensionsv1.CustomResourceDefinitionCondition{
								{
									Type:   apiextensionsv1.Established,
									Status: apiextensionsv1.ConditionTrue,
								},
							},
						},
					},
				},
			},
		},
		"watch error": {
			crd:      "test-crd",
			watchErr: errors.New("watch error"),
			wantErr:  true,
		},
		"crd deleted": {
			crd:     "test-crd",
			events:  []watch.Event{{Type: watch.Deleted}},
			wantErr: true,
		},
		"other error": {
			crd:     "test-crd",
			events:  []watch.Event{{Type: watch.Error}},
			wantErr: true,
		},
	}

	for name, tc := range testCases {
		t.Run(name, func(t *testing.T) {
			assert := assert.New(t)
			require := require.New(t)

			client := Client{
				apiextensionClient: &stubCRDWatcher{events: tc.events, watchErr: tc.watchErr},
			}
			err := client.WaitForCRD(context.Background(), tc.crd)
			if tc.wantErr {
				assert.Error(err)
				return
			}
			require.NoError(err)
		})
	}
}

type stubCRDWatcher struct {
	events   []watch.Event
	watchErr error

	apiextensionsclientv1.ApiextensionsV1Interface
}

func (w *stubCRDWatcher) CustomResourceDefinitions() apiextensionsclientv1.CustomResourceDefinitionInterface {
	return &stubCustomResourceDefinitions{
		events:   w.events,
		watchErr: w.watchErr,
	}
}

type stubCustomResourceDefinitions struct {
	events   []watch.Event
	watchErr error

	apiextensionsclientv1.CustomResourceDefinitionInterface
}

func (c *stubCustomResourceDefinitions) Watch(ctx context.Context, opts metav1.ListOptions) (watch.Interface, error) {
	eventChan := make(chan watch.Event, len(c.events))
	for _, event := range c.events {
		eventChan <- event
	}
	return &stubCRDWatch{events: eventChan}, c.watchErr
}

type stubCRDWatch struct {
	events chan watch.Event
}

func (w *stubCRDWatch) Stop() {
	close(w.events)
}

func (w *stubCRDWatch) ResultChan() <-chan watch.Event {
	return w.events
}