init: create kubeconfig file with unique user/cluster name (#1133)

* Generate kubeconfig with unique name

* Move create name flag to config

* Add name validation to config

* Move name flag in e2e tests to config generation

* Remove name flag from create

* Update ascii cinema flow

---------

Signed-off-by: Daniel Weiße <dw@edgeless.systems>
This commit is contained in:
Daniel Weiße 2023-02-10 13:27:22 +01:00 committed by GitHub
parent fd860ddb91
commit c29107f5be
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
29 changed files with 359 additions and 436 deletions

View file

@ -28,6 +28,7 @@ import (
"github.com/edgelesssys/constellation/v2/internal/role"
"github.com/edgelesssys/constellation/v2/internal/versions/components"
corev1 "k8s.io/api/core/v1"
"k8s.io/apiserver/pkg/authentication/user"
kubeconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
"github.com/edgelesssys/constellation/v2/internal/crypto"
@ -87,25 +88,26 @@ func (k *KubernetesUtil) InstallComponents(ctx context.Context, kubernetesCompon
}
// InitCluster instruments kubeadm to initialize the K8s cluster.
// On success an admin kubeconfig file is returned.
func (k *KubernetesUtil) InitCluster(
ctx context.Context, initConfig []byte, nodeName string, ips []net.IP, controlPlaneEndpoint string, conformanceMode bool, log *logger.Logger,
) error {
ctx context.Context, initConfig []byte, nodeName, clusterName string, ips []net.IP, controlPlaneEndpoint string, conformanceMode bool, log *logger.Logger,
) ([]byte, error) {
// TODO: audit policy should be user input
auditPolicy, err := resources.NewDefaultAuditPolicy().Marshal()
if err != nil {
return fmt.Errorf("generating default audit policy: %w", err)
return nil, fmt.Errorf("generating default audit policy: %w", err)
}
if err := os.WriteFile(auditPolicyPath, auditPolicy, 0o644); err != nil {
return fmt.Errorf("writing default audit policy: %w", err)
return nil, fmt.Errorf("writing default audit policy: %w", err)
}
initConfigFile, err := os.CreateTemp("", "kubeadm-init.*.yaml")
if err != nil {
return fmt.Errorf("creating init config file %v: %w", initConfigFile.Name(), err)
return nil, fmt.Errorf("creating init config file %v: %w", initConfigFile.Name(), err)
}
if _, err := initConfigFile.Write(initConfig); err != nil {
return fmt.Errorf("writing kubeadm init yaml config %v: %w", initConfigFile.Name(), err)
return nil, fmt.Errorf("writing kubeadm init yaml config %v: %w", initConfigFile.Name(), err)
}
// preflight
@ -115,9 +117,9 @@ func (k *KubernetesUtil) InitCluster(
if err != nil {
var exitErr *exec.ExitError
if errors.As(err, &exitErr) {
return fmt.Errorf("kubeadm init phase preflight failed (code %v) with: %s", exitErr.ExitCode(), out)
return nil, fmt.Errorf("kubeadm init phase preflight failed (code %v) with: %s", exitErr.ExitCode(), out)
}
return fmt.Errorf("kubeadm init: %w", err)
return nil, fmt.Errorf("kubeadm init: %w", err)
}
// create CA certs
@ -127,20 +129,20 @@ func (k *KubernetesUtil) InitCluster(
if err != nil {
var exitErr *exec.ExitError
if errors.As(err, &exitErr) {
return fmt.Errorf("kubeadm init phase certs all failed (code %v) with: %s", exitErr.ExitCode(), out)
return nil, fmt.Errorf("kubeadm init phase certs all failed (code %v) with: %s", exitErr.ExitCode(), out)
}
return fmt.Errorf("kubeadm init: %w", err)
return nil, fmt.Errorf("kubeadm init: %w", err)
}
// create kubelet key and CA signed certificate for the node
log.Infof("Creating signed kubelet certificate")
if err := k.createSignedKubeletCert(nodeName, ips); err != nil {
return err
return nil, fmt.Errorf("creating signed kubelete certificate: %w", err)
}
log.Infof("Preparing node for Konnectivity")
if err := k.prepareControlPlaneForKonnectivity(ctx, controlPlaneEndpoint); err != nil {
return fmt.Errorf("setup konnectivity: %w", err)
return nil, fmt.Errorf("setup konnectivity: %w", err)
}
// initialize the cluster
@ -155,12 +157,29 @@ func (k *KubernetesUtil) InitCluster(
if err != nil {
var exitErr *exec.ExitError
if errors.As(err, &exitErr) {
return fmt.Errorf("kubeadm init failed (code %v) with: %s", exitErr.ExitCode(), out)
return nil, fmt.Errorf("kubeadm init failed (code %v) with: %s", exitErr.ExitCode(), out)
}
return fmt.Errorf("kubeadm init: %w", err)
return nil, fmt.Errorf("kubeadm init: %w", err)
}
log.With(zap.String("output", string(out))).Infof("kubeadm init succeeded")
return nil
userName := clusterName + "-admin"
log.With(zap.String("userName", userName)).Infof("Creating admin kubeconfig file")
cmd = exec.CommandContext(
ctx, constants.KubeadmPath, "kubeconfig", "user",
"--client-name", userName, "--config", initConfigFile.Name(), "--org", user.SystemPrivilegedGroup,
)
out, err = cmd.Output()
if err != nil {
var exitErr *exec.ExitError
if errors.As(err, &exitErr) {
return nil, fmt.Errorf("kubeadm kubeconfig user failed (code %v) with: %s", exitErr.ExitCode(), out)
}
return nil, fmt.Errorf("kubeadm kubeconfig user: %w", err)
}
log.Infof("kubeadm kubeconfig user succeeded")
return out, nil
}
func (k *KubernetesUtil) prepareControlPlaneForKonnectivity(ctx context.Context, loadBalancerEndpoint string) error {

View file

@ -271,6 +271,12 @@ func (k *KubeadmInitYAML) SetNodeName(nodeName string) {
k.InitConfiguration.NodeRegistration.Name = nodeName
}
// SetClusterName sets the name of the Kubernetes cluster.
// This name is reflected in the kubeconfig file and in the name of the default admin user.
func (k *KubeadmInitYAML) SetClusterName(clusterName string) {
k.ClusterConfiguration.ClusterName = clusterName
}
// SetCertSANs sets the SANs for the certificate.
func (k *KubeadmInitYAML) SetCertSANs(certSANs []string) {
for _, certSAN := range certSANs {

View file

@ -19,7 +19,7 @@ import (
type clusterUtil interface {
InstallComponents(ctx context.Context, kubernetesComponents components.Components) error
InitCluster(ctx context.Context, initConfig []byte, nodeName string, ips []net.IP, controlPlaneEndpoint string, conformanceMode bool, log *logger.Logger) error
InitCluster(ctx context.Context, initConfig []byte, nodeName, clusterName string, ips []net.IP, controlPlaneEndpoint string, conformanceMode bool, log *logger.Logger) ([]byte, error)
JoinCluster(ctx context.Context, joinConfig []byte, peerRole role.Role, controlPlaneEndpoint string, log *logger.Logger) error
FixCilium(log *logger.Logger)
StartKubelet() error

View file

@ -1,29 +0,0 @@
/*
Copyright (c) Edgeless Systems GmbH
SPDX-License-Identifier: AGPL-3.0-only
*/
package kubernetes
import (
"fmt"
"github.com/spf13/afero"
)
const kubeconfigPath = "/etc/kubernetes/admin.conf"
// KubeconfigReader implements ConfigReader.
type KubeconfigReader struct {
fs afero.Afero
}
// ReadKubeconfig reads the Kubeconfig from disk.
func (r KubeconfigReader) ReadKubeconfig() ([]byte, error) {
kubeconfig, err := r.fs.ReadFile(kubeconfigPath)
if err != nil {
return nil, fmt.Errorf("reading kubernetes config: %w", err)
}
return kubeconfig, nil
}

View file

@ -1,40 +0,0 @@
/*
Copyright (c) Edgeless Systems GmbH
SPDX-License-Identifier: AGPL-3.0-only
*/
package kubernetes
import (
"testing"
"github.com/spf13/afero"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestReadKubeconfig(t *testing.T) {
require := require.New(t)
assert := assert.New(t)
fs := afero.Afero{
Fs: afero.NewMemMapFs(),
}
require.NoError(fs.WriteFile(kubeconfigPath, []byte("someConfig"), 0o644))
reader := KubeconfigReader{fs}
config, err := reader.ReadKubeconfig()
require.NoError(err)
assert.Equal([]byte("someConfig"), config)
}
func TestReadKubeconfigFails(t *testing.T) {
assert := assert.New(t)
fs := afero.Afero{
Fs: afero.NewMemMapFs(),
}
reader := KubeconfigReader{fs}
_, err := reader.ReadKubeconfig()
assert.Error(err)
}

View file

@ -30,7 +30,6 @@ import (
"github.com/edgelesssys/constellation/v2/internal/logger"
"github.com/edgelesssys/constellation/v2/internal/role"
"github.com/edgelesssys/constellation/v2/internal/versions/components"
"github.com/spf13/afero"
"go.uber.org/zap"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@ -39,11 +38,6 @@ import (
var validHostnameRegex = regexp.MustCompile(`^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$`)
// configReader provides kubeconfig as []byte.
type configReader interface {
ReadKubeconfig() ([]byte, error)
}
// configurationProvider provides kubeadm init and join configuration.
type configurationProvider interface {
InitConfiguration(externalCloudProvider bool, k8sVersion string) k8sapi.KubeadmInitYAML
@ -62,7 +56,6 @@ type KubeWrapper struct {
kubeAPIWaiter kubeAPIWaiter
configProvider configurationProvider
client k8sapi.Client
kubeconfigReader configReader
providerMetadata ProviderMetadata
initialMeasurements measurements.M
getIPAddr func() (string, error)
@ -79,7 +72,6 @@ func New(cloudProvider string, clusterUtil clusterUtil, configProvider configura
kubeAPIWaiter: kubeAPIWaiter,
configProvider: configProvider,
client: client,
kubeconfigReader: &KubeconfigReader{fs: afero.Afero{Fs: afero.NewOsFs()}},
providerMetadata: providerMetadata,
initialMeasurements: measurements,
getIPAddr: getIPAddr,
@ -88,8 +80,8 @@ func New(cloudProvider string, clusterUtil clusterUtil, configProvider configura
// InitCluster initializes a new Kubernetes cluster and applies pod network provider.
func (k *KubeWrapper) InitCluster(
ctx context.Context, cloudServiceAccountURI, versionString string, measurementSalt []byte, enforcedPCRs []uint32,
enforceIDKeyDigest bool, azureCVM bool,
ctx context.Context, cloudServiceAccountURI, versionString, clusterName string,
measurementSalt []byte, enforcedPCRs []uint32, enforceIDKeyDigest bool, azureCVM bool,
helmReleasesRaw []byte, conformanceMode bool, kubernetesComponents components.Components, log *logger.Logger,
) ([]byte, error) {
log.With(zap.String("version", versionString)).Infof("Installing Kubernetes components")
@ -139,6 +131,7 @@ func (k *KubeWrapper) InitCluster(
cloudprovider.FromString(k.cloudProvider) == cloudprovider.GCP
initConfig := k.configProvider.InitConfiguration(ccmSupported, versionString)
initConfig.SetNodeIP(nodeIP)
initConfig.SetClusterName(clusterName)
initConfig.SetCertSANs([]string{nodeIP})
initConfig.SetNodeName(nodeName)
initConfig.SetProviderID(instance.ProviderID)
@ -148,13 +141,11 @@ func (k *KubeWrapper) InitCluster(
return nil, fmt.Errorf("encoding kubeadm init configuration as YAML: %w", err)
}
log.Infof("Initializing Kubernetes cluster")
if err := k.clusterUtil.InitCluster(ctx, initConfigYAML, nodeName, validIPs, controlPlaneEndpoint, conformanceMode, log); err != nil {
kubeConfig, err := k.clusterUtil.InitCluster(ctx, initConfigYAML, nodeName, clusterName, validIPs, controlPlaneEndpoint, conformanceMode, log)
if err != nil {
return nil, fmt.Errorf("kubeadm init: %w", err)
}
kubeConfig, err := k.GetKubeconfig()
if err != nil {
return nil, fmt.Errorf("reading kubeconfig after cluster initialization: %w", err)
}
err = k.client.Initialize(kubeConfig)
if err != nil {
return nil, fmt.Errorf("initializing kubectl client: %w", err)
@ -250,7 +241,7 @@ func (k *KubeWrapper) InitCluster(
k.clusterUtil.FixCilium(log)
return k.GetKubeconfig()
return kubeConfig, nil
}
// JoinCluster joins existing Kubernetes cluster.
@ -311,11 +302,6 @@ func (k *KubeWrapper) JoinCluster(ctx context.Context, args *kubeadm.BootstrapTo
return nil
}
// GetKubeconfig returns the current nodes kubeconfig of stored on disk.
func (k *KubeWrapper) GetKubeconfig() ([]byte, error) {
return k.kubeconfigReader.ReadKubeconfig()
}
// setupK8sComponentsConfigMap applies a ConfigMap (cf. server-side apply) to store the installed k8s components.
// It returns the name of the ConfigMap.
func (k *KubeWrapper) setupK8sComponentsConfigMap(ctx context.Context, components components.Components, clusterVersion string) (string, error) {

View file

@ -50,16 +50,12 @@ func TestInitCluster(t *testing.T) {
kubectl stubKubectl
kubeAPIWaiter stubKubeAPIWaiter
providerMetadata ProviderMetadata
kubeconfigReader configReader
wantConfig k8sapi.KubeadmInitYAML
wantErr bool
k8sVersion versions.ValidK8sVersion
}{
"kubeadm init works with metadata and loadbalancer": {
clusterUtil: stubClusterUtil{},
kubeconfigReader: &stubKubeconfigReader{
kubeconfig: []byte("someKubeconfig"),
},
clusterUtil: stubClusterUtil{kubeconfig: []byte("someKubeconfig")},
kubeAPIWaiter: stubKubeAPIWaiter{},
providerMetadata: &stubProviderMetadata{
selfResp: metadata.InstanceMetadata{
@ -81,6 +77,7 @@ func TestInitCluster(t *testing.T) {
},
},
ClusterConfiguration: kubeadm.ClusterConfiguration{
ClusterName: "kubernetes",
ControlPlaneEndpoint: loadbalancerIP,
APIServer: kubeadm.APIServer{
CertSANs: []string{privateIP},
@ -91,10 +88,7 @@ func TestInitCluster(t *testing.T) {
k8sVersion: versions.Default,
},
"kubeadm init fails when annotating itself": {
clusterUtil: stubClusterUtil{},
kubeconfigReader: &stubKubeconfigReader{
kubeconfig: []byte("someKubeconfig"),
},
clusterUtil: stubClusterUtil{kubeconfig: []byte("someKubeconfig")},
kubeAPIWaiter: stubKubeAPIWaiter{},
providerMetadata: &stubProviderMetadata{
selfResp: metadata.InstanceMetadata{
@ -110,10 +104,7 @@ func TestInitCluster(t *testing.T) {
k8sVersion: versions.Default,
},
"kubeadm init fails when retrieving metadata self": {
clusterUtil: stubClusterUtil{},
kubeconfigReader: &stubKubeconfigReader{
kubeconfig: []byte("someKubeconfig"),
},
clusterUtil: stubClusterUtil{kubeconfig: []byte("someKubeconfig")},
kubeAPIWaiter: stubKubeAPIWaiter{},
providerMetadata: &stubProviderMetadata{
selfErr: someErr,
@ -122,10 +113,7 @@ func TestInitCluster(t *testing.T) {
k8sVersion: versions.Default,
},
"kubeadm init fails when retrieving metadata loadbalancer ip": {
clusterUtil: stubClusterUtil{},
kubeconfigReader: &stubKubeconfigReader{
kubeconfig: []byte("someKubeconfig"),
},
clusterUtil: stubClusterUtil{kubeconfig: []byte("someKubeconfig")},
providerMetadata: &stubProviderMetadata{
getLoadBalancerEndpointErr: someErr,
},
@ -133,9 +121,9 @@ func TestInitCluster(t *testing.T) {
k8sVersion: versions.Default,
},
"kubeadm init fails when applying the init config": {
clusterUtil: stubClusterUtil{initClusterErr: someErr},
kubeconfigReader: &stubKubeconfigReader{
kubeconfig: []byte("someKubeconfig"),
clusterUtil: stubClusterUtil{
initClusterErr: someErr,
kubeconfig: []byte("someKubeconfig"),
},
kubeAPIWaiter: stubKubeAPIWaiter{},
providerMetadata: &stubProviderMetadata{},
@ -143,95 +131,67 @@ func TestInitCluster(t *testing.T) {
k8sVersion: versions.Default,
},
"kubeadm init fails when deploying cilium": {
clusterUtil: stubClusterUtil{},
helmClient: stubHelmClient{ciliumError: someErr},
kubeconfigReader: &stubKubeconfigReader{
kubeconfig: []byte("someKubeconfig"),
},
clusterUtil: stubClusterUtil{kubeconfig: []byte("someKubeconfig")},
helmClient: stubHelmClient{ciliumError: someErr},
providerMetadata: &stubProviderMetadata{},
wantErr: true,
k8sVersion: versions.Default,
},
"kubeadm init fails when setting up constellation-services chart": {
clusterUtil: stubClusterUtil{},
helmClient: stubHelmClient{servicesError: someErr},
kubeconfigReader: &stubKubeconfigReader{
kubeconfig: []byte("someKubeconfig"),
},
clusterUtil: stubClusterUtil{kubeconfig: []byte("someKubeconfig")},
helmClient: stubHelmClient{servicesError: someErr},
kubeAPIWaiter: stubKubeAPIWaiter{},
providerMetadata: &stubProviderMetadata{},
wantErr: true,
k8sVersion: versions.Default,
},
"kubeadm init fails when setting the cloud node manager": {
clusterUtil: stubClusterUtil{},
helmClient: stubHelmClient{servicesError: someErr},
kubeconfigReader: &stubKubeconfigReader{
kubeconfig: []byte("someKubeconfig"),
},
clusterUtil: stubClusterUtil{kubeconfig: []byte("someKubeconfig")},
helmClient: stubHelmClient{servicesError: someErr},
kubeAPIWaiter: stubKubeAPIWaiter{},
providerMetadata: &stubProviderMetadata{},
wantErr: true,
k8sVersion: versions.Default,
},
"kubeadm init fails when setting the cluster autoscaler": {
clusterUtil: stubClusterUtil{},
helmClient: stubHelmClient{servicesError: someErr},
kubeconfigReader: &stubKubeconfigReader{
kubeconfig: []byte("someKubeconfig"),
},
clusterUtil: stubClusterUtil{kubeconfig: []byte("someKubeconfig")},
helmClient: stubHelmClient{servicesError: someErr},
kubeAPIWaiter: stubKubeAPIWaiter{},
providerMetadata: &stubProviderMetadata{},
wantErr: true,
k8sVersion: versions.Default,
},
"kubeadm init fails when reading kubeconfig": {
clusterUtil: stubClusterUtil{},
kubeconfigReader: &stubKubeconfigReader{
readErr: someErr,
},
clusterUtil: stubClusterUtil{kubeconfig: []byte("someKubeconfig")},
kubeAPIWaiter: stubKubeAPIWaiter{},
providerMetadata: &stubProviderMetadata{},
wantErr: true,
k8sVersion: versions.Default,
},
"kubeadm init fails when setting up konnectivity": {
clusterUtil: stubClusterUtil{},
helmClient: stubHelmClient{servicesError: someErr},
kubeconfigReader: &stubKubeconfigReader{
kubeconfig: []byte("someKubeconfig"),
},
clusterUtil: stubClusterUtil{kubeconfig: []byte("someKubeconfig")},
helmClient: stubHelmClient{servicesError: someErr},
kubeAPIWaiter: stubKubeAPIWaiter{},
providerMetadata: &stubProviderMetadata{},
wantErr: true,
k8sVersion: versions.Default,
},
"kubeadm init fails when setting up verification service": {
clusterUtil: stubClusterUtil{},
helmClient: stubHelmClient{servicesError: someErr},
kubeconfigReader: &stubKubeconfigReader{
kubeconfig: []byte("someKubeconfig"),
},
clusterUtil: stubClusterUtil{kubeconfig: []byte("someKubeconfig")},
kubeAPIWaiter: stubKubeAPIWaiter{},
providerMetadata: &stubProviderMetadata{},
wantErr: true,
k8sVersion: versions.Default,
},
"kubeadm init fails when waiting for kubeAPI server": {
clusterUtil: stubClusterUtil{},
kubeconfigReader: &stubKubeconfigReader{
kubeconfig: []byte("someKubeconfig"),
},
clusterUtil: stubClusterUtil{kubeconfig: []byte("someKubeconfig")},
kubeAPIWaiter: stubKubeAPIWaiter{waitErr: someErr},
providerMetadata: &stubProviderMetadata{},
k8sVersion: versions.Default,
wantErr: true,
},
"unsupported k8sVersion fails cluster creation": {
clusterUtil: stubClusterUtil{},
kubeconfigReader: &stubKubeconfigReader{
kubeconfig: []byte("someKubeconfig"),
},
clusterUtil: stubClusterUtil{kubeconfig: []byte("someKubeconfig")},
kubeAPIWaiter: stubKubeAPIWaiter{},
providerMetadata: &stubProviderMetadata{},
k8sVersion: "1.19",
@ -251,12 +211,11 @@ func TestInitCluster(t *testing.T) {
kubeAPIWaiter: &tc.kubeAPIWaiter,
configProvider: &stubConfigProvider{initConfig: k8sapi.KubeadmInitYAML{}},
client: &tc.kubectl,
kubeconfigReader: tc.kubeconfigReader,
getIPAddr: func() (string, error) { return privateIP, nil },
}
_, err := kube.InitCluster(
context.Background(), serviceAccountURI, string(tc.k8sVersion),
context.Background(), serviceAccountURI, string(tc.k8sVersion), "kubernetes",
nil, nil, false, true, []byte("{}"), false, nil, logger.NewTest(t),
)
@ -503,6 +462,8 @@ type stubClusterUtil struct {
joinClusterErr error
startKubeletErr error
kubeconfig []byte
initConfigs [][]byte
joinConfigs [][]byte
}
@ -515,9 +476,9 @@ func (s *stubClusterUtil) InstallComponents(ctx context.Context, kubernetesCompo
return s.installComponentsErr
}
func (s *stubClusterUtil) InitCluster(ctx context.Context, initConfig []byte, nodeName string, ips []net.IP, controlPlaneEndpoint string, conformanceMode bool, log *logger.Logger) error {
func (s *stubClusterUtil) InitCluster(ctx context.Context, initConfig []byte, nodeName, clusterName string, ips []net.IP, controlPlaneEndpoint string, conformanceMode bool, log *logger.Logger) ([]byte, error) {
s.initConfigs = append(s.initConfigs, initConfig)
return s.initClusterErr
return s.kubeconfig, s.initClusterErr
}
func (s *stubClusterUtil) SetupAutoscaling(kubectl k8sapi.Client, clusterAutoscalerConfiguration kubernetes.Marshaler, secrets kubernetes.Marshaler) error {
@ -611,15 +572,6 @@ func (s *stubKubectl) ListAllNamespaces(ctx context.Context) (*corev1.NamespaceL
return s.listAllNamespacesResp, s.listAllNamespacesErr
}
type stubKubeconfigReader struct {
kubeconfig []byte
readErr error
}
func (s *stubKubeconfigReader) ReadKubeconfig() ([]byte, error) {
return s.kubeconfig, s.readErr
}
type stubHelmClient struct {
ciliumError error
certManagerError error