constellation/coordinator/kubernetes/kubernetes_test.go
Daniel Weiße 042f668d20 AB#2190 Verification service (#232)
* Add verification service

* Update verify command to use new Constellation verification service

* Deploy verification service on cluster init

* Update pcr-reader to use verification service

* Add verification service build workflow

Signed-off-by: Daniel Weiße <dw@edgeless.systems>

Signed-off-by: Daniel Weiße <dw@edgeless.systems>
2022-06-28 17:03:28 +02:00

649 lines
21 KiB
Go

package kubernetes
import (
"context"
"errors"
"regexp"
"testing"
"time"
"github.com/edgelesssys/constellation/coordinator/cloudprovider/cloudtypes"
"github.com/edgelesssys/constellation/coordinator/kubernetes/k8sapi"
"github.com/edgelesssys/constellation/coordinator/kubernetes/k8sapi/resources"
"github.com/edgelesssys/constellation/coordinator/role"
attestationtypes "github.com/edgelesssys/constellation/internal/attestation/types"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"go.uber.org/goleak"
kubeadm "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta3"
)
func TestMain(m *testing.M) {
goleak.VerifyTestMain(m)
}
func TestInitCluster(t *testing.T) {
someErr := errors.New("failed")
coordinatorVPNIP := "192.0.2.0"
serviceAccountUri := "some-service-account-uri"
masterSecret := []byte("some-master-secret")
autoscalingNodeGroups := []string{"0,10,autoscaling_group_0"}
nodeName := "node-name"
providerID := "provider-id"
privateIP := "192.0.2.1"
publicIP := "192.0.2.2"
loadbalancerIP := "192.0.2.3"
aliasIPRange := "192.0.2.0/24"
testCases := map[string]struct {
clusterUtil stubClusterUtil
kubeCTL stubKubeCTL
providerMetadata ProviderMetadata
CloudControllerManager CloudControllerManager
CloudNodeManager CloudNodeManager
ClusterAutoscaler ClusterAutoscaler
kubeconfigReader configReader
wantConfig k8sapi.KubeadmInitYAML
wantErr bool
}{
"kubeadm init works without metadata": {
clusterUtil: stubClusterUtil{},
kubeconfigReader: &stubKubeconfigReader{
Kubeconfig: []byte("someKubeconfig"),
},
providerMetadata: &stubProviderMetadata{SupportedResp: false},
CloudControllerManager: &stubCloudControllerManager{},
CloudNodeManager: &stubCloudNodeManager{SupportedResp: false},
ClusterAutoscaler: &stubClusterAutoscaler{},
wantConfig: k8sapi.KubeadmInitYAML{
InitConfiguration: kubeadm.InitConfiguration{
NodeRegistration: kubeadm.NodeRegistrationOptions{
KubeletExtraArgs: map[string]string{
"node-ip": "",
"provider-id": "",
},
Name: coordinatorVPNIP,
},
},
ClusterConfiguration: kubeadm.ClusterConfiguration{},
},
},
"kubeadm init works with metadata and loadbalancer": {
clusterUtil: stubClusterUtil{},
kubeconfigReader: &stubKubeconfigReader{
Kubeconfig: []byte("someKubeconfig"),
},
providerMetadata: &stubProviderMetadata{
SupportedResp: true,
SelfResp: cloudtypes.Instance{
Name: nodeName,
ProviderID: providerID,
PrivateIPs: []string{privateIP},
PublicIPs: []string{publicIP},
AliasIPRanges: []string{aliasIPRange},
},
GetLoadBalancerIPResp: loadbalancerIP,
SupportsLoadBalancerResp: true,
},
CloudControllerManager: &stubCloudControllerManager{},
CloudNodeManager: &stubCloudNodeManager{SupportedResp: false},
ClusterAutoscaler: &stubClusterAutoscaler{},
wantConfig: k8sapi.KubeadmInitYAML{
InitConfiguration: kubeadm.InitConfiguration{
NodeRegistration: kubeadm.NodeRegistrationOptions{
KubeletExtraArgs: map[string]string{
"node-ip": privateIP,
"provider-id": providerID,
},
Name: nodeName,
},
},
ClusterConfiguration: kubeadm.ClusterConfiguration{
ControlPlaneEndpoint: loadbalancerIP,
APIServer: kubeadm.APIServer{
CertSANs: []string{publicIP, privateIP},
},
},
},
wantErr: false,
},
"kubeadm init fails when retrieving metadata self": {
clusterUtil: stubClusterUtil{},
kubeconfigReader: &stubKubeconfigReader{
Kubeconfig: []byte("someKubeconfig"),
},
providerMetadata: &stubProviderMetadata{
SelfErr: someErr,
SupportedResp: true,
},
CloudControllerManager: &stubCloudControllerManager{},
CloudNodeManager: &stubCloudNodeManager{},
ClusterAutoscaler: &stubClusterAutoscaler{},
wantErr: true,
},
"kubeadm init fails when retrieving metadata subnetwork cidr": {
clusterUtil: stubClusterUtil{},
kubeconfigReader: &stubKubeconfigReader{
Kubeconfig: []byte("someKubeconfig"),
},
providerMetadata: &stubProviderMetadata{
GetSubnetworkCIDRErr: someErr,
SupportedResp: true,
},
CloudControllerManager: &stubCloudControllerManager{},
CloudNodeManager: &stubCloudNodeManager{},
ClusterAutoscaler: &stubClusterAutoscaler{},
wantErr: true,
},
"kubeadm init fails when retrieving metadata loadbalancer ip": {
clusterUtil: stubClusterUtil{},
kubeconfigReader: &stubKubeconfigReader{
Kubeconfig: []byte("someKubeconfig"),
},
providerMetadata: &stubProviderMetadata{
GetLoadBalancerIPErr: someErr,
SupportsLoadBalancerResp: true,
SupportedResp: true,
},
CloudControllerManager: &stubCloudControllerManager{},
CloudNodeManager: &stubCloudNodeManager{},
ClusterAutoscaler: &stubClusterAutoscaler{},
wantErr: true,
},
"kubeadm init fails when applying the init config": {
clusterUtil: stubClusterUtil{initClusterErr: someErr},
kubeconfigReader: &stubKubeconfigReader{
Kubeconfig: []byte("someKubeconfig"),
},
providerMetadata: &stubProviderMetadata{},
CloudControllerManager: &stubCloudControllerManager{},
CloudNodeManager: &stubCloudNodeManager{},
ClusterAutoscaler: &stubClusterAutoscaler{},
wantErr: true,
},
"kubeadm init fails when setting up the pod network": {
clusterUtil: stubClusterUtil{setupPodNetworkErr: someErr},
kubeconfigReader: &stubKubeconfigReader{
Kubeconfig: []byte("someKubeconfig"),
},
providerMetadata: &stubProviderMetadata{},
CloudControllerManager: &stubCloudControllerManager{},
CloudNodeManager: &stubCloudNodeManager{},
ClusterAutoscaler: &stubClusterAutoscaler{},
wantErr: true,
},
"kubeadm init fails when setting up the activation service": {
clusterUtil: stubClusterUtil{setupActivationServiceError: someErr},
kubeconfigReader: &stubKubeconfigReader{
Kubeconfig: []byte("someKubeconfig"),
},
providerMetadata: &stubProviderMetadata{},
CloudControllerManager: &stubCloudControllerManager{SupportedResp: true},
CloudNodeManager: &stubCloudNodeManager{},
ClusterAutoscaler: &stubClusterAutoscaler{},
wantErr: true,
},
"kubeadm init fails when setting the cloud contoller manager": {
clusterUtil: stubClusterUtil{setupCloudControllerManagerError: someErr},
kubeconfigReader: &stubKubeconfigReader{
Kubeconfig: []byte("someKubeconfig"),
},
providerMetadata: &stubProviderMetadata{},
CloudControllerManager: &stubCloudControllerManager{SupportedResp: true},
CloudNodeManager: &stubCloudNodeManager{},
ClusterAutoscaler: &stubClusterAutoscaler{},
wantErr: true,
},
"kubeadm init fails when setting the cloud node manager": {
clusterUtil: stubClusterUtil{setupCloudNodeManagerError: someErr},
kubeconfigReader: &stubKubeconfigReader{
Kubeconfig: []byte("someKubeconfig"),
},
providerMetadata: &stubProviderMetadata{},
CloudControllerManager: &stubCloudControllerManager{},
CloudNodeManager: &stubCloudNodeManager{SupportedResp: true},
ClusterAutoscaler: &stubClusterAutoscaler{},
wantErr: true,
},
"kubeadm init fails when setting the cluster autoscaler": {
clusterUtil: stubClusterUtil{setupAutoscalingError: someErr},
kubeconfigReader: &stubKubeconfigReader{
Kubeconfig: []byte("someKubeconfig"),
},
providerMetadata: &stubProviderMetadata{},
CloudControllerManager: &stubCloudControllerManager{},
CloudNodeManager: &stubCloudNodeManager{},
ClusterAutoscaler: &stubClusterAutoscaler{SupportedResp: true},
wantErr: true,
},
"kubeadm init fails when reading kubeconfig": {
clusterUtil: stubClusterUtil{},
kubeconfigReader: &stubKubeconfigReader{
ReadErr: someErr,
},
providerMetadata: &stubProviderMetadata{},
CloudControllerManager: &stubCloudControllerManager{},
CloudNodeManager: &stubCloudNodeManager{},
ClusterAutoscaler: &stubClusterAutoscaler{},
wantErr: true,
},
"kubeadm init fails when setting up the kms": {
clusterUtil: stubClusterUtil{setupKMSError: someErr},
kubeconfigReader: &stubKubeconfigReader{
Kubeconfig: []byte("someKubeconfig"),
},
providerMetadata: &stubProviderMetadata{SupportedResp: false},
CloudControllerManager: &stubCloudControllerManager{},
CloudNodeManager: &stubCloudNodeManager{SupportedResp: false},
ClusterAutoscaler: &stubClusterAutoscaler{},
wantErr: true,
},
"kubeadm init fails when setting up verification service": {
clusterUtil: stubClusterUtil{setupVerificationServiceErr: someErr},
kubeconfigReader: &stubKubeconfigReader{
Kubeconfig: []byte("someKubeconfig"),
},
providerMetadata: &stubProviderMetadata{SupportedResp: false},
CloudControllerManager: &stubCloudControllerManager{},
CloudNodeManager: &stubCloudNodeManager{SupportedResp: false},
ClusterAutoscaler: &stubClusterAutoscaler{},
wantErr: true,
},
}
for name, tc := range testCases {
t.Run(name, func(t *testing.T) {
assert := assert.New(t)
require := require.New(t)
kube := KubeWrapper{
clusterUtil: &tc.clusterUtil,
providerMetadata: tc.providerMetadata,
cloudControllerManager: tc.CloudControllerManager,
cloudNodeManager: tc.CloudNodeManager,
clusterAutoscaler: tc.ClusterAutoscaler,
configProvider: &stubConfigProvider{InitConfig: k8sapi.KubeadmInitYAML{}},
client: &tc.kubeCTL,
kubeconfigReader: tc.kubeconfigReader,
}
err := kube.InitCluster(context.Background(), autoscalingNodeGroups, serviceAccountUri, coordinatorVPNIP, attestationtypes.ID{}, masterSecret, nil)
if tc.wantErr {
assert.Error(err)
return
}
require.NoError(err)
var kubeadmConfig k8sapi.KubeadmInitYAML
require.NoError(resources.UnmarshalK8SResources(tc.clusterUtil.initConfigs[0], &kubeadmConfig))
require.Equal(tc.wantConfig.ClusterConfiguration, kubeadmConfig.ClusterConfiguration)
require.Equal(tc.wantConfig.InitConfiguration, kubeadmConfig.InitConfiguration)
})
}
}
func TestJoinCluster(t *testing.T) {
someErr := errors.New("failed")
joinCommand := &kubeadm.BootstrapTokenDiscovery{
APIServerEndpoint: "192.0.2.0:6443",
Token: "kube-fake-token",
CACertHashes: []string{"sha256:a60ebe9b0879090edd83b40a4df4bebb20506bac1e51d518ff8f4505a721930f"},
}
nodeVPNIP := "192.0.2.0"
certKey := "cert-key"
testCases := map[string]struct {
clusterUtil stubClusterUtil
providerMetadata ProviderMetadata
CloudControllerManager CloudControllerManager
wantConfig kubeadm.JoinConfiguration
role role.Role
wantErr bool
}{
"kubeadm join worker works without metadata": {
clusterUtil: stubClusterUtil{},
providerMetadata: &stubProviderMetadata{},
CloudControllerManager: &stubCloudControllerManager{},
role: role.Node,
wantConfig: kubeadm.JoinConfiguration{
Discovery: kubeadm.Discovery{
BootstrapToken: joinCommand,
},
NodeRegistration: kubeadm.NodeRegistrationOptions{
Name: nodeVPNIP,
KubeletExtraArgs: map[string]string{"node-ip": "192.0.2.0"},
},
},
},
"kubeadm join worker works with metadata": {
clusterUtil: stubClusterUtil{},
providerMetadata: &stubProviderMetadata{
SupportedResp: true,
SelfResp: cloudtypes.Instance{
ProviderID: "provider-id",
Name: "metadata-name",
PrivateIPs: []string{"192.0.2.1"},
},
},
CloudControllerManager: &stubCloudControllerManager{},
role: role.Node,
wantConfig: kubeadm.JoinConfiguration{
Discovery: kubeadm.Discovery{
BootstrapToken: joinCommand,
},
NodeRegistration: kubeadm.NodeRegistrationOptions{
Name: "metadata-name",
KubeletExtraArgs: map[string]string{"node-ip": "192.0.2.1"},
},
},
},
"kubeadm join worker works with metadata and cloud controller manager": {
clusterUtil: stubClusterUtil{},
providerMetadata: &stubProviderMetadata{
SupportedResp: true,
SelfResp: cloudtypes.Instance{
ProviderID: "provider-id",
Name: "metadata-name",
PrivateIPs: []string{"192.0.2.1"},
},
},
CloudControllerManager: &stubCloudControllerManager{
SupportedResp: true,
},
role: role.Node,
wantConfig: kubeadm.JoinConfiguration{
Discovery: kubeadm.Discovery{
BootstrapToken: joinCommand,
},
NodeRegistration: kubeadm.NodeRegistrationOptions{
Name: "metadata-name",
KubeletExtraArgs: map[string]string{"node-ip": "192.0.2.1"},
},
},
},
"kubeadm join control-plane node works with metadata": {
clusterUtil: stubClusterUtil{},
providerMetadata: &stubProviderMetadata{
SupportedResp: true,
SelfResp: cloudtypes.Instance{
ProviderID: "provider-id",
Name: "metadata-name",
PrivateIPs: []string{"192.0.2.1"},
},
},
CloudControllerManager: &stubCloudControllerManager{},
role: role.Coordinator,
wantConfig: kubeadm.JoinConfiguration{
Discovery: kubeadm.Discovery{
BootstrapToken: joinCommand,
},
NodeRegistration: kubeadm.NodeRegistrationOptions{
Name: "metadata-name",
KubeletExtraArgs: map[string]string{"node-ip": "192.0.2.1"},
},
ControlPlane: &kubeadm.JoinControlPlane{
LocalAPIEndpoint: kubeadm.APIEndpoint{
AdvertiseAddress: "192.0.2.1",
BindPort: 6443,
},
CertificateKey: certKey,
},
},
},
"kubeadm join worker fails when retrieving self metadata": {
clusterUtil: stubClusterUtil{},
providerMetadata: &stubProviderMetadata{
SupportedResp: true,
SelfErr: someErr,
},
CloudControllerManager: &stubCloudControllerManager{},
role: role.Node,
wantErr: true,
},
"kubeadm join worker fails when applying the join config": {
clusterUtil: stubClusterUtil{joinClusterErr: someErr},
providerMetadata: &stubProviderMetadata{},
CloudControllerManager: &stubCloudControllerManager{},
role: role.Node,
wantErr: true,
},
"kubeadm join worker works fails when setting the metadata for the cloud controller manager": {
clusterUtil: stubClusterUtil{},
providerMetadata: &stubProviderMetadata{
SupportedResp: true,
SelfResp: cloudtypes.Instance{
ProviderID: "provider-id",
Name: "metadata-name",
PrivateIPs: []string{"192.0.2.1"},
},
SetVPNIPErr: someErr,
},
CloudControllerManager: &stubCloudControllerManager{
SupportedResp: true,
},
role: role.Node,
wantErr: true,
},
}
for name, tc := range testCases {
t.Run(name, func(t *testing.T) {
assert := assert.New(t)
require := require.New(t)
kube := KubeWrapper{
clusterUtil: &tc.clusterUtil,
providerMetadata: tc.providerMetadata,
cloudControllerManager: tc.CloudControllerManager,
configProvider: &stubConfigProvider{},
}
err := kube.JoinCluster(context.Background(), joinCommand, nodeVPNIP, certKey, tc.role)
if tc.wantErr {
assert.Error(err)
return
}
require.NoError(err)
var joinYaml k8sapi.KubeadmJoinYAML
joinYaml, err = joinYaml.Unmarshal(tc.clusterUtil.joinConfigs[0])
require.NoError(err)
assert.Equal(tc.wantConfig, joinYaml.JoinConfiguration)
})
}
}
func TestGetKubeconfig(t *testing.T) {
testCases := map[string]struct {
Kubewrapper KubeWrapper
wantErr bool
}{
"check single replacement": {
Kubewrapper: KubeWrapper{kubeconfigReader: &stubKubeconfigReader{
Kubeconfig: []byte("127.0.0.1:16443"),
}},
},
"check multiple replacement": {
Kubewrapper: KubeWrapper{kubeconfigReader: &stubKubeconfigReader{
Kubeconfig: []byte("127.0.0.1:16443...127.0.0.1:16443"),
}},
},
}
for name, tc := range testCases {
t.Run(name, func(t *testing.T) {
assert := assert.New(t)
require := require.New(t)
data, err := tc.Kubewrapper.GetKubeconfig()
require.NoError(err)
assert.NotContains(string(data), "127.0.0.1:16443")
assert.Contains(string(data), "10.118.0.1:6443")
})
}
}
func TestK8sCompliantHostname(t *testing.T) {
compliantHostname := regexp.MustCompile(`^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$`)
testCases := map[string]struct {
hostname string
wantHostname string
}{
"azure scale set names work": {
hostname: "constellation-scale-set-coordinators-name_0",
wantHostname: "constellation-scale-set-coordinators-name-0",
},
"compliant hostname is not modified": {
hostname: "abcd-123",
wantHostname: "abcd-123",
},
"uppercase hostnames are lowercased": {
hostname: "ABCD",
wantHostname: "abcd",
},
}
for name, tc := range testCases {
t.Run(name, func(t *testing.T) {
assert := assert.New(t)
hostname := k8sCompliantHostname(tc.hostname)
assert.Equal(tc.wantHostname, hostname)
assert.Regexp(compliantHostname, hostname)
})
}
}
type stubClusterUtil struct {
installComponentsErr error
initClusterErr error
setupPodNetworkErr error
setupAutoscalingError error
setupActivationServiceError error
setupCloudControllerManagerError error
setupCloudNodeManagerError error
setupKMSError error
setupAccessManagerError error
setupVerificationServiceErr error
joinClusterErr error
startKubeletErr error
restartKubeletErr error
createJoinTokenResponse *kubeadm.BootstrapTokenDiscovery
createJoinTokenErr error
initConfigs [][]byte
joinConfigs [][]byte
}
func (s *stubClusterUtil) InstallComponents(ctx context.Context, version string) error {
return s.installComponentsErr
}
func (s *stubClusterUtil) InitCluster(ctx context.Context, initConfig []byte) error {
s.initConfigs = append(s.initConfigs, initConfig)
return s.initClusterErr
}
func (s *stubClusterUtil) SetupPodNetwork(context.Context, k8sapi.SetupPodNetworkInput) error {
return s.setupPodNetworkErr
}
func (s *stubClusterUtil) SetupAutoscaling(kubectl k8sapi.Client, clusterAutoscalerConfiguration resources.Marshaler, secrets resources.Marshaler) error {
return s.setupAutoscalingError
}
func (s *stubClusterUtil) SetupActivationService(kubectl k8sapi.Client, activationServiceConfiguration resources.Marshaler) error {
return s.setupActivationServiceError
}
func (s *stubClusterUtil) SetupCloudControllerManager(kubectl k8sapi.Client, cloudControllerManagerConfiguration resources.Marshaler, configMaps resources.Marshaler, secrets resources.Marshaler) error {
return s.setupCloudControllerManagerError
}
func (s *stubClusterUtil) SetupKMS(kubectl k8sapi.Client, kmsDeployment resources.Marshaler) error {
return s.setupKMSError
}
func (s *stubClusterUtil) SetupAccessManager(kubectl k8sapi.Client, accessManagerConfiguration resources.Marshaler) error {
return s.setupAccessManagerError
}
func (s *stubClusterUtil) SetupCloudNodeManager(kubectl k8sapi.Client, cloudNodeManagerConfiguration resources.Marshaler) error {
return s.setupCloudNodeManagerError
}
func (s *stubClusterUtil) SetupVerificationService(kubectl k8sapi.Client, verificationServiceConfiguration resources.Marshaler) error {
return s.setupVerificationServiceErr
}
func (s *stubClusterUtil) JoinCluster(ctx context.Context, joinConfig []byte) error {
s.joinConfigs = append(s.joinConfigs, joinConfig)
return s.joinClusterErr
}
func (s *stubClusterUtil) StartKubelet() error {
return s.startKubeletErr
}
func (s *stubClusterUtil) RestartKubelet() error {
return s.restartKubeletErr
}
func (s *stubClusterUtil) GetControlPlaneJoinCertificateKey(context.Context) (string, error) {
return "", nil
}
func (s *stubClusterUtil) CreateJoinToken(ctx context.Context, ttl time.Duration) (*kubeadm.BootstrapTokenDiscovery, error) {
return s.createJoinTokenResponse, s.createJoinTokenErr
}
func (s *stubClusterUtil) FixCilium(nodeName string) {
}
type stubConfigProvider struct {
InitConfig k8sapi.KubeadmInitYAML
JoinConfig k8sapi.KubeadmJoinYAML
}
func (s *stubConfigProvider) InitConfiguration(_ bool) k8sapi.KubeadmInitYAML {
return s.InitConfig
}
func (s *stubConfigProvider) JoinConfiguration(_ bool) k8sapi.KubeadmJoinYAML {
s.JoinConfig = k8sapi.KubeadmJoinYAML{
JoinConfiguration: kubeadm.JoinConfiguration{
Discovery: kubeadm.Discovery{
BootstrapToken: &kubeadm.BootstrapTokenDiscovery{},
},
},
}
return s.JoinConfig
}
type stubKubeCTL struct {
ApplyErr error
resources []resources.Marshaler
kubeconfigs [][]byte
}
func (s *stubKubeCTL) Apply(resources resources.Marshaler, forceConflicts bool) error {
s.resources = append(s.resources, resources)
return s.ApplyErr
}
func (s *stubKubeCTL) SetKubeconfig(kubeconfig []byte) {
s.kubeconfigs = append(s.kubeconfigs, kubeconfig)
}
type stubKubeconfigReader struct {
Kubeconfig []byte
ReadErr error
}
func (s *stubKubeconfigReader) ReadKubeconfig() ([]byte, error) {
return s.Kubeconfig, s.ReadErr
}