Git-Mirrors/constellation
1
0
Fork 0
mirror of https://github.com/edgelesssys/constellation.git synced 2024-09-20 00:06:21 +00:00
Code Issues Packages Projects Releases Wiki Activity

Print kubeadm init/join output on success

Browse Source
This commit is contained in:
Malte Poll 2022-07-07 11:43:35 +02:00 committed by Paul Meyer
parent 66b573ea5d
commit 5d54ce689b
10 changed files with 34 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
bootstrapper/cmd/bootstrapper/test.go
View File

@ -7,6 +7,7 @@ import (
"github.com/edgelesssys/constellation/bootstrapper/role"
attestationtypes "github.com/edgelesssys/constellation/internal/attestation/types"
"github.com/edgelesssys/constellation/internal/cloud/metadata"
"go.uber.org/zap"
kubeadm "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta3"
)
@ -14,13 +15,13 @@ import (
type clusterFake struct{}
// InitCluster fakes bootstrapping a new cluster with the current node being the master, returning the arguments required to join the cluster.
func (c *clusterFake) InitCluster(context.Context, []string, string, string, attestationtypes.ID, kubernetes.KMSConfig, map[string]string,
func (c *clusterFake) InitCluster(context.Context, []string, string, string, attestationtypes.ID, kubernetes.KMSConfig, map[string]string, *zap.Logger,
) ([]byte, error) {
return []byte{}, nil
}
// JoinCluster will fake joining the current node to an existing cluster.
func (c *clusterFake) JoinCluster(context.Context, *kubeadm.BootstrapTokenDiscovery, string, role.Role) error {
func (c *clusterFake) JoinCluster(context.Context, *kubeadm.BootstrapTokenDiscovery, string, role.Role, *zap.Logger) error {
return nil
}

2
bootstrapper/internal/initserver/initserver.go
View File

@ -124,6 +124,7 @@ func (s *Server) Init(ctx context.Context, req *initproto.InitRequest) (*initpro
UseExistingKEK: req.UseExistingKek,
},
sshProtoKeysToMap(req.SshUserKeys),
s.logger,
)
if err != nil {
return nil, status.Errorf(codes.Internal, "initializing cluster: %s", err)
@ -192,6 +193,7 @@ type ClusterInitializer interface {
id attestationtypes.ID,
kmsConfig kubernetes.KMSConfig,
sshUserKeys map[string]string,
logger *zap.Logger,
) ([]byte, error)
}

2
bootstrapper/internal/initserver/initserver_test.go
View File

@ -217,7 +217,7 @@ type stubClusterInitializer struct {
initClusterErr error
}
func (i *stubClusterInitializer) InitCluster(context.Context, []string, string, string, attestationtypes.ID, kubernetes.KMSConfig, map[string]string,
func (i *stubClusterInitializer) InitCluster(context.Context, []string, string, string, attestationtypes.ID, kubernetes.KMSConfig, map[string]string, *zap.Logger,
) ([]byte, error) {
return i.initClusterKubeconfig, i.initClusterErr
}

3
bootstrapper/internal/joinclient/client.go
View File

@ -245,7 +245,7 @@ func (c *JoinClient) startNodeAndJoin(ticket *joinproto.IssueJoinTicketResponse)
Token: ticket.Token,
CACertHashes: []string{ticket.DiscoveryTokenCaCertHash},
}
if err := c.joiner.JoinCluster(ctx, btd, ticket.CertificateKey, c.role); err != nil {
if err := c.joiner.JoinCluster(ctx, btd, ticket.CertificateKey, c.role, c.log); err != nil {
return fmt.Errorf("joining Kubernetes cluster: %w", err)
}
@ -337,6 +337,7 @@ type ClusterJoiner interface {
args *kubeadm.BootstrapTokenDiscovery,
certKey string,
peerRole role.Role,
logger *zap.Logger,
) error
}

2
bootstrapper/internal/joinclient/client_test.go
View File

@ -385,7 +385,7 @@ type stubClusterJoiner struct {
joinClusterErr error
}
func (j *stubClusterJoiner) JoinCluster(context.Context, *kubeadm.BootstrapTokenDiscovery, string, role.Role) error {
func (j *stubClusterJoiner) JoinCluster(context.Context, *kubeadm.BootstrapTokenDiscovery, string, role.Role, *zap.Logger) error {
j.joinClusterCalled = true
return j.joinClusterErr
}

16
bootstrapper/internal/kubernetes/k8sapi/util.go
View File

@ -12,6 +12,7 @@ import (
"time"
"github.com/edgelesssys/constellation/bootstrapper/internal/kubernetes/k8sapi/resources"
"go.uber.org/zap"
kubeadm "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta3"
)
@ -76,7 +77,7 @@ func (k *KubernetesUtil) InstallComponents(ctx context.Context, version string)
return enableSystemdUnit(ctx, kubeletServiceEtcPath)
}
func (k *KubernetesUtil) InitCluster(ctx context.Context, initConfig []byte) error {
func (k *KubernetesUtil) InitCluster(ctx context.Context, initConfig []byte, logger *zap.Logger) error {
// TODO: audit policy should be user input
auditPolicy, err := resources.NewDefaultAuditPolicy().Marshal()
if err != nil {
@ -97,14 +98,15 @@ func (k *KubernetesUtil) InitCluster(ctx context.Context, initConfig []byte) err
}
cmd := exec.CommandContext(ctx, kubeadmPath, "init", "-v=5", "--config", initConfigFile.Name())
_, err = cmd.Output()
out, err := cmd.CombinedOutput()
if err != nil {
var exitErr *exec.ExitError
if errors.As(err, &exitErr) {
return fmt.Errorf("kubeadm init failed (code %v) with: %s", exitErr.ExitCode(), exitErr.Stderr)
return fmt.Errorf("kubeadm init failed (code %v) with: %s", exitErr.ExitCode(), out)
}
return fmt.Errorf("kubeadm init: %w", err)
}
logger.Info("kubeadm init succeeded", zap.String("output", string(out)))
return nil
}
@ -280,7 +282,7 @@ func (k *KubernetesUtil) SetupVerificationService(kubectl Client, verificationSe
}
// JoinCluster joins existing Kubernetes cluster using kubeadm join.
func (k *KubernetesUtil) JoinCluster(ctx context.Context, joinConfig []byte) error {
func (k *KubernetesUtil) JoinCluster(ctx context.Context, joinConfig []byte, logger *zap.Logger) error {
// TODO: audit policy should be user input
auditPolicy, err := resources.NewDefaultAuditPolicy().Marshal()
if err != nil {
@ -302,13 +304,15 @@ func (k *KubernetesUtil) JoinCluster(ctx context.Context, joinConfig []byte) err
// run `kubeadm join` to join a worker node to an existing Kubernetes cluster
cmd := exec.CommandContext(ctx, kubeadmPath, "join", "-v=5", "--config", joinConfigFile.Name())
if _, err := cmd.Output(); err != nil {
out, err := cmd.CombinedOutput()
if err != nil {
var exitErr *exec.ExitError
if errors.As(err, &exitErr) {
return fmt.Errorf("kubeadm join failed (code %v) with: %s (full err: %s)", exitErr.ExitCode(), exitErr.Stderr, err)
return fmt.Errorf("kubeadm join failed (code %v) with: %s (full err: %s)", exitErr.ExitCode(), out, err)
}
return fmt.Errorf("kubeadm join: %w", err)
}
logger.Info("kubeadm join succeeded", zap.String("output", string(out)))
return nil
}

5
bootstrapper/internal/kubernetes/k8sutil.go
View File

@ -6,13 +6,14 @@ import (
"github.com/edgelesssys/constellation/bootstrapper/internal/kubernetes/k8sapi"
"github.com/edgelesssys/constellation/bootstrapper/internal/kubernetes/k8sapi/resources"
"go.uber.org/zap"
kubeadm "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta3"
)
type clusterUtil interface {
InstallComponents(ctx context.Context, version string) error
InitCluster(ctx context.Context, initConfig []byte) error
JoinCluster(ctx context.Context, joinConfig []byte) error
InitCluster(ctx context.Context, initConfig []byte, logger *zap.Logger) error
JoinCluster(ctx context.Context, joinConfig []byte, logger *zap.Logger) error
SetupPodNetwork(context.Context, k8sapi.SetupPodNetworkInput) error
SetupAccessManager(kubectl k8sapi.Client, sshUsers resources.Marshaler) error
SetupAutoscaling(kubectl k8sapi.Client, clusterAutoscalerConfiguration resources.Marshaler, secrets resources.Marshaler) error

9
bootstrapper/internal/kubernetes/kubernetes.go
View File

@ -16,6 +16,7 @@ import (
attestationtypes "github.com/edgelesssys/constellation/internal/attestation/types"
"github.com/edgelesssys/constellation/internal/cloud/metadata"
"github.com/spf13/afero"
"go.uber.org/zap"
kubeadm "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta3"
)
@ -75,7 +76,7 @@ type KMSConfig struct {
// InitCluster initializes a new Kubernetes cluster and applies pod network provider.
func (k *KubeWrapper) InitCluster(
ctx context.Context, autoscalingNodeGroups []string, cloudServiceAccountURI, k8sVersion string,
id attestationtypes.ID, kmsConfig KMSConfig, sshUsers map[string]string,
id attestationtypes.ID, kmsConfig KMSConfig, sshUsers map[string]string, logger *zap.Logger,
) ([]byte, error) {
// TODO: k8s version should be user input
if err := k.clusterUtil.InstallComponents(ctx, k8sVersion); err != nil {
@ -141,7 +142,7 @@ func (k *KubeWrapper) InitCluster(
if err != nil {
return nil, fmt.Errorf("encoding kubeadm init configuration as YAML: %w", err)
}
if err := k.clusterUtil.InitCluster(ctx, initConfigYAML); err != nil {
if err := k.clusterUtil.InitCluster(ctx, initConfigYAML, logger); err != nil {
return nil, fmt.Errorf("kubeadm init: %w", err)
}
kubeConfig, err := k.GetKubeconfig()
@ -206,7 +207,7 @@ func (k *KubeWrapper) InitCluster(
}
// JoinCluster joins existing Kubernetes cluster.
func (k *KubeWrapper) JoinCluster(ctx context.Context, args *kubeadm.BootstrapTokenDiscovery, certKey string, peerRole role.Role) error {
func (k *KubeWrapper) JoinCluster(ctx context.Context, args *kubeadm.BootstrapTokenDiscovery, certKey string, peerRole role.Role, logger *zap.Logger) error {
// TODO: k8s version should be user input
if err := k.clusterUtil.InstallComponents(ctx, "1.23.6"); err != nil {
return err
@ -248,7 +249,7 @@ func (k *KubeWrapper) JoinCluster(ctx context.Context, args *kubeadm.BootstrapTo
if err != nil {
return fmt.Errorf("encoding kubeadm join configuration as YAML: %w", err)
}
if err := k.clusterUtil.JoinCluster(ctx, joinConfigYAML); err != nil {
if err := k.clusterUtil.JoinCluster(ctx, joinConfigYAML, logger); err != nil {
return fmt.Errorf("joining cluster: %v; %w ", string(joinConfigYAML), err)
}

10
bootstrapper/internal/kubernetes/kubernetes_test.go
View File

@ -15,6 +15,8 @@ import (
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"go.uber.org/goleak"
"go.uber.org/zap"
"go.uber.org/zap/zaptest"
kubeadm "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta3"
)
@ -268,7 +270,7 @@ func TestInitCluster(t *testing.T) {
kubeconfigReader: tc.kubeconfigReader,
getIPAddr: func() (string, error) { return privateIP, nil },
}
_, err := kube.InitCluster(context.Background(), autoscalingNodeGroups, serviceAccountUri, k8sVersion, attestationtypes.ID{}, KMSConfig{MasterSecret: masterSecret}, nil)
_, err := kube.InitCluster(context.Background(), autoscalingNodeGroups, serviceAccountUri, k8sVersion, attestationtypes.ID{}, KMSConfig{MasterSecret: masterSecret}, nil, zaptest.NewLogger(t))
if tc.wantErr {
assert.Error(err)
@ -425,7 +427,7 @@ func TestJoinCluster(t *testing.T) {
getIPAddr: func() (string, error) { return privateIP, nil },
}
err := kube.JoinCluster(context.Background(), joinCommand, certKey, tc.role)
err := kube.JoinCluster(context.Background(), joinCommand, certKey, tc.role, zaptest.NewLogger(t))
if tc.wantErr {
assert.Error(err)
return
@ -499,7 +501,7 @@ func (s *stubClusterUtil) InstallComponents(ctx context.Context, version string)
return s.installComponentsErr
}
func (s *stubClusterUtil) InitCluster(ctx context.Context, initConfig []byte) error {
func (s *stubClusterUtil) InitCluster(ctx context.Context, initConfig []byte, logger *zap.Logger) error {
s.initConfigs = append(s.initConfigs, initConfig)
return s.initClusterErr
}
@ -540,7 +542,7 @@ func (s *stubClusterUtil) SetupVerificationService(kubectl k8sapi.Client, verifi
return s.setupVerificationServiceErr
}
func (s *stubClusterUtil) JoinCluster(ctx context.Context, joinConfig []byte) error {
func (s *stubClusterUtil) JoinCluster(ctx context.Context, joinConfig []byte, logger *zap.Logger) error {
s.joinConfigs = append(s.joinConfigs, joinConfig)
return s.joinClusterErr
}

2
cli/internal/azure/client/compute.go
View File

@ -32,7 +32,7 @@ func (c *Client) CreateInstances(ctx context.Context, input CreateInstancesInput
// Create control plane scale set
createControlPlaneInput := CreateScaleSetInput{
Name: "constellation-scale-set-controlpalens-" + c.uid,
Name: "constellation-scale-set-controlplanes-" + c.uid,
NamePrefix: c.name + "-control-plane-" + c.uid + "-",
Count: input.CountControlPlanes,
InstanceType: input.InstanceType,