Git-Mirrors/constellation
1
0
Fork 0
mirror of https://github.com/edgelesssys/constellation.git synced 2025-08-03 04:26:20 -04:00
Code Issues Projects Releases Packages Wiki Activity

kubernetes: verify Kubernetes components

Browse source
This commit is contained in:
Leonard Cohnen 2022-11-14 19:09:49 +01:00 • committed by 3u13r
parent 2c9ddbc6e7
commit 1e98b686b6
17 changed files with 439 additions and 229 deletions
Download patch file Download diff file Expand all files Collapse all files

17
bootstrapper/internal/kubernetes/k8sapi/constants.go
View file

@ -1,17 +0,0 @@
/*
Copyright (c) Edgeless Systems GmbH
SPDX-License-Identifier: AGPL-3.0-only
*/
package k8sapi
const (
// Paths and permissions necessary for Kubernetes installation.
cniPluginsDir = "/opt/cni/bin"
binDir = "/run/state/bin"
kubeadmPath = "/run/state/bin/kubeadm"
kubeletPath = "/run/state/bin/kubelet"
kubeletServicePath = "/usr/lib/systemd/system/kubelet.service"
executablePerm = 0o544
)

42
bootstrapper/internal/kubernetes/k8sapi/install.go
View file

@ -10,6 +10,7 @@ import (
"archive/tar"
"compress/gzip"
"context"
"crypto/sha256"
"errors"
"fmt"
"io"
@ -20,6 +21,7 @@ import (
"time"
"github.com/edgelesssys/constellation/v2/internal/retry"
"github.com/edgelesssys/constellation/v2/internal/versions"
"github.com/spf13/afero"
"k8s.io/utils/clock"
)
@ -50,28 +52,38 @@ func newOSInstaller() *osInstaller {
}
// Install downloads a resource from a URL, applies any given text transformations and extracts the resulting file if required.
// The resulting file(s) are copied to all destinations.
func (i *osInstaller) Install(
ctx context.Context, sourceURL string, destinations []string, perm fs.FileMode, extract bool,
) error {
tempPath, err := i.retryDownloadToTempDir(ctx, sourceURL)
// The resulting file(s) are copied to the destination. It also verifies the sha256 hash of the downloaded file.
func (i *osInstaller) Install(ctx context.Context, kubernetesComponent versions.ComponentVersion) error {
tempPath, err := i.retryDownloadToTempDir(ctx, kubernetesComponent.URL)
if err != nil {
return err
}
file, err := i.fs.OpenFile(tempPath, os.O_RDONLY, 0)
if err != nil {
return fmt.Errorf("opening file %q: %w", tempPath, err)
}
sha := sha256.New()
if _, err := io.Copy(sha, file); err != nil {
return fmt.Errorf("reading file %q: %w", tempPath, err)
}
calculatedHash := fmt.Sprintf("sha256:%x", sha.Sum(nil))
if calculatedHash != kubernetesComponent.Hash {
return fmt.Errorf("hash of file %q %s does not match expected hash %s", tempPath, calculatedHash, kubernetesComponent.Hash)
}
defer func() {
_ = i.fs.Remove(tempPath)
}()
for _, destination := range destinations {
var err error
if extract {
err = i.extractArchive(tempPath, destination, perm)
} else {
err = i.copy(tempPath, destination, perm)
}
if err != nil {
return fmt.Errorf("installing from %q: copying to destination %q: %w", sourceURL, destination, err)
}
if kubernetesComponent.Extract {
err = i.extractArchive(tempPath, kubernetesComponent.InstallPath, executablePerm)
} else {
err = i.copy(tempPath, kubernetesComponent.InstallPath, executablePerm)
}
if err != nil {
return fmt.Errorf("installing from %q: copying to destination %q: %w", kubernetesComponent.URL, kubernetesComponent.InstallPath, err)
}
return nil
}

48
bootstrapper/internal/kubernetes/k8sapi/install_test.go
View file

@ -22,6 +22,7 @@ import (
"testing"
"time"
"github.com/edgelesssys/constellation/v2/internal/versions"
"github.com/spf13/afero"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
@ -31,29 +32,52 @@ import (
)
func TestInstall(t *testing.T) {
serverURL := "http://server/path"
testCases := map[string]struct {
server httpBufconnServer
component versions.ComponentVersion
hash string
destination string
extract bool
readonly bool
wantErr bool
wantFiles map[string][]byte
}{
"download works": {
server: newHTTPBufconnServerWithBody([]byte("file-contents")),
destination: "/destination",
wantFiles: map[string][]byte{"/destination": []byte("file-contents")},
server: newHTTPBufconnServerWithBody([]byte("file-contents")),
component: versions.ComponentVersion{
URL: serverURL,
Hash: "sha256:f03779b36bece74893fd6533a67549675e21573eb0e288d87158738f9c24594e",
InstallPath: "/destination",
},
wantFiles: map[string][]byte{"/destination": []byte("file-contents")},
},
"download with extract works": {
server: newHTTPBufconnServerWithBody(createTarGz([]byte("file-contents"), "/destination")),
destination: "/prefix",
extract: true,
wantFiles: map[string][]byte{"/prefix/destination": []byte("file-contents")},
server: newHTTPBufconnServerWithBody(createTarGz([]byte("file-contents"), "/destination")),
component: versions.ComponentVersion{
URL: serverURL,
Hash: "sha256:a52a1664ca0a6ec9790384e3d058852ab8b3a8f389a9113d150fdc6ab308d949",
InstallPath: "/prefix",
Extract: true,
},
wantFiles: map[string][]byte{"/prefix/destination": []byte("file-contents")},
},
"hash validation fails": {
server: newHTTPBufconnServerWithBody([]byte("file-contents")),
component: versions.ComponentVersion{
URL: serverURL,
Hash: "sha256:abc",
InstallPath: "/destination",
},
wantErr: true,
},
"download fails": {
server: newHTTPBufconnServer(func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(500) }),
destination: "/destination",
wantErr: true,
server: newHTTPBufconnServer(func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(500) }),
component: versions.ComponentVersion{
URL: serverURL,
Hash: "sha256:abc",
InstallPath: "/destination",
},
wantErr: true,
},
}
@ -81,7 +105,7 @@ func TestInstall(t *testing.T) {
retriable: func(err error) bool { return false },
}
err := inst.Install(context.Background(), "http://server/path", []string{tc.destination}, fs.ModePerm, tc.extract)
err := inst.Install(context.Background(), tc.component)
if tc.wantErr {
assert.Error(err)
return

55
bootstrapper/internal/kubernetes/k8sapi/k8sutil.go
View file

@ -13,7 +13,6 @@ import (
"encoding/pem"
"errors"
"fmt"
"io/fs"
"net"
"net/http"
"os"
@ -42,6 +41,10 @@ import (
const (
// kubeletStartTimeout is the maximum time given to the kubelet service to (re)start.
kubeletStartTimeout = 10 * time.Minute
// crdTimeout is the maximum time given to the CRDs to be created.
crdTimeout = 30 * time.Second
executablePerm = 0o544
kubeletServicePath = "/usr/lib/systemd/system/kubelet.service"
)
// Client provides the functions to talk to the k8s API.
@ -57,7 +60,7 @@ type Client interface {
type installer interface {
Install(
ctx context.Context, sourceURL string, destinations []string, perm fs.FileMode, extract bool,
ctx context.Context, kubernetesComponent versions.ComponentVersion,
) error
}
@ -75,34 +78,26 @@ func NewKubernetesUtil() *KubernetesUtil {
}
}
// InstallComponentsFromCLI installs the kubernetes components passed from the CLI.
func (k *KubernetesUtil) InstallComponentsFromCLI(ctx context.Context, kubernetesComponents versions.ComponentVersions) error {
for _, component := range kubernetesComponents {
if err := k.inst.Install(ctx, component); err != nil {
return fmt.Errorf("installing kubernetes component from URL %s: %w", component.URL, err)
}
}
return enableSystemdUnit(ctx, kubeletServicePath)
}
// InstallComponents installs kubernetes components in the version specified.
// TODO(AB#2543,3u13r): Remove this function once the JoinService is extended.
func (k *KubernetesUtil) InstallComponents(ctx context.Context, version versions.ValidK8sVersion) error {
versionConf := versions.VersionConfigs[version]
if err := k.inst.Install(
ctx, versionConf.CNIPlugins.URL, []string{cniPluginsDir}, executablePerm, true,
); err != nil {
return fmt.Errorf("installing cni plugins: %w", err)
}
if err := k.inst.Install(
ctx, versionConf.Crictl.URL, []string{binDir}, executablePerm, true,
); err != nil {
return fmt.Errorf("installing crictl: %w", err)
}
if err := k.inst.Install(
ctx, versionConf.Kubelet.URL, []string{kubeletPath}, executablePerm, false,
); err != nil {
return fmt.Errorf("installing kubelet: %w", err)
}
if err := k.inst.Install(
ctx, versionConf.Kubeadm.URL, []string{kubeadmPath}, executablePerm, false,
); err != nil {
return fmt.Errorf("installing kubeadm: %w", err)
}
if err := k.inst.Install(
ctx, versionConf.Kubectl.URL, []string{constants.KubectlPath}, executablePerm, false,
); err != nil {
return fmt.Errorf("installing kubectl: %w", err)
for _, component := range versionConf.KubernetesComponents {
if err := k.inst.Install(ctx, component); err != nil {
return fmt.Errorf("installing kubernetes component from URL %s: %w", component.URL, err)
}
}
return enableSystemdUnit(ctx, kubeletServicePath)
@ -132,7 +127,7 @@ func (k *KubernetesUtil) InitCluster(
// preflight
log.Infof("Running kubeadm preflight checks")
cmd := exec.CommandContext(ctx, kubeadmPath, "init", "phase", "preflight", "-v=5", "--config", initConfigFile.Name())
cmd := exec.CommandContext(ctx, constants.KubeadmPath, "init", "phase", "preflight", "-v=5", "--config", initConfigFile.Name())
out, err := cmd.CombinedOutput()
if err != nil {
var exitErr *exec.ExitError
@ -144,7 +139,7 @@ func (k *KubernetesUtil) InitCluster(
// create CA certs
log.Infof("Creating Kubernetes control-plane certificates and keys")
cmd = exec.CommandContext(ctx, kubeadmPath, "init", "phase", "certs", "all", "-v=5", "--config", initConfigFile.Name())
cmd = exec.CommandContext(ctx, constants.KubeadmPath, "init", "phase", "certs", "all", "-v=5", "--config", initConfigFile.Name())
out, err = cmd.CombinedOutput()
if err != nil {
var exitErr *exec.ExitError
@ -172,7 +167,7 @@ func (k *KubernetesUtil) InitCluster(
skipPhases += ",addon/kube-proxy"
}
cmd = exec.CommandContext(ctx, kubeadmPath, "init", "-v=5", skipPhases, "--config", initConfigFile.Name())
cmd = exec.CommandContext(ctx, constants.KubeadmPath, "init", "-v=5", skipPhases, "--config", initConfigFile.Name())
out, err = cmd.CombinedOutput()
if err != nil {
var exitErr *exec.ExitError
@ -362,7 +357,7 @@ func (k *KubernetesUtil) JoinCluster(ctx context.Context, joinConfig []byte, pee
}
// run `kubeadm join` to join a worker node to an existing Kubernetes cluster
cmd := exec.CommandContext(ctx, kubeadmPath, "join", "-v=5", "--config", joinConfigFile.Name())
cmd := exec.CommandContext(ctx, constants.KubeadmPath, "join", "-v=5", "--config", joinConfigFile.Name())
out, err := cmd.CombinedOutput()
if err != nil {
var exitErr *exec.ExitError

1
bootstrapper/internal/kubernetes/k8sutil.go
View file

@ -20,6 +20,7 @@ import (
type clusterUtil interface {
InstallComponents(ctx context.Context, version versions.ValidK8sVersion) error
InstallComponentsFromCLI(ctx context.Context, kubernetesComponents versions.ComponentVersions) error
InitCluster(ctx context.Context, initConfig []byte, nodeName string, ips []net.IP, controlPlaneEndpoint string, conformanceMode bool, log *logger.Logger) error
JoinCluster(ctx context.Context, joinConfig []byte, peerRole role.Role, controlPlaneEndpoint string, log *logger.Logger) error
SetupKonnectivity(kubectl k8sapi.Client, konnectivityAgentsDaemonSet kubernetes.Marshaler) error

4
bootstrapper/internal/kubernetes/kubernetes.go
View file

@ -87,14 +87,14 @@ func New(cloudProvider string, clusterUtil clusterUtil, configProvider configura
func (k *KubeWrapper) InitCluster(
ctx context.Context, cloudServiceAccountURI, versionString string, measurementSalt []byte, enforcedPCRs []uint32,
enforceIDKeyDigest bool, idKeyDigest []byte, azureCVM bool,
helmReleasesRaw []byte, conformanceMode bool, log *logger.Logger,
helmReleasesRaw []byte, conformanceMode bool, kubernetesComponents versions.ComponentVersions, log *logger.Logger,
) ([]byte, error) {
k8sVersion, err := versions.NewValidK8sVersion(versionString)
if err != nil {
return nil, err
}
log.With(zap.String("version", string(k8sVersion))).Infof("Installing Kubernetes components")
if err := k.clusterUtil.InstallComponents(ctx, k8sVersion); err != nil {
if err := k.clusterUtil.InstallComponentsFromCLI(ctx, kubernetesComponents); err != nil {
return nil, err
}

27
bootstrapper/internal/kubernetes/kubernetes_test.go
View file

@ -237,7 +237,7 @@ func TestInitCluster(t *testing.T) {
_, err := kube.InitCluster(
context.Background(), serviceAccountURI, string(tc.k8sVersion),
nil, nil, false, nil, true, []byte("{}"), false, logger.NewTest(t),
nil, nil, false, nil, true, []byte("{}"), false, nil, logger.NewTest(t),
)
if tc.wantErr {
@ -415,16 +415,17 @@ func TestK8sCompliantHostname(t *testing.T) {
}
type stubClusterUtil struct {
installComponentsErr error
initClusterErr error
setupAutoscalingError error
setupKonnectivityError error
setupGCPGuestAgentErr error
setupOLMErr error
setupNMOErr error
setupNodeOperatorErr error
joinClusterErr error
startKubeletErr error
installComponentsErr error
installComponentsFromCLIErr error
initClusterErr error
setupAutoscalingError error
setupKonnectivityError error
setupGCPGuestAgentErr error
setupOLMErr error
setupNMOErr error
setupNodeOperatorErr error
joinClusterErr error
startKubeletErr error
initConfigs [][]byte
joinConfigs [][]byte
@ -438,6 +439,10 @@ func (s *stubClusterUtil) InstallComponents(ctx context.Context, version version
return s.installComponentsErr
}
func (s *stubClusterUtil) InstallComponentsFromCLI(ctx context.Context, kubernetesComponents versions.ComponentVersions) error {
return s.installComponentsFromCLIErr
}
// TODO: Upon changing this function, please refactor it to reduce the number of arguments to <= 5.
//
//revive:disable-next-line