kubernetes: verify Kubernetes components

This commit is contained in:
Leonard Cohnen 2022-11-14 19:09:49 +01:00 committed by 3u13r
parent 2c9ddbc6e7
commit 1e98b686b6
17 changed files with 439 additions and 229 deletions

View File

@ -25,6 +25,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
- Automatic CSI driver deployment for Azure and GCP during Constellation init
- Improve reproducibility by pinning the Kubernetes components.
### Changed
<!-- For changes in existing functionality. -->
- Constellation operators are now deployed using Helm.

View File

@ -12,6 +12,7 @@ import (
"github.com/edgelesssys/constellation/v2/internal/cloud/metadata"
"github.com/edgelesssys/constellation/v2/internal/logger"
"github.com/edgelesssys/constellation/v2/internal/role"
"github.com/edgelesssys/constellation/v2/internal/versions"
kubeadm "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta3"
)
@ -21,7 +22,7 @@ type clusterFake struct{}
// InitCluster fakes bootstrapping a new cluster with the current node being the master, returning the arguments required to join the cluster.
func (c *clusterFake) InitCluster(
context.Context, string, string, []byte, []uint32, bool, []byte, bool,
[]byte, bool, *logger.Logger,
[]byte, bool, versions.ComponentVersions, *logger.Logger,
) ([]byte, error) {
return []byte{}, nil
}

View File

@ -39,6 +39,7 @@ type InitRequest struct {
EnforcedPcrs []uint32 `protobuf:"varint,12,rep,packed,name=enforced_pcrs,json=enforcedPcrs,proto3" json:"enforced_pcrs,omitempty"`
EnforceIdkeydigest bool `protobuf:"varint,13,opt,name=enforce_idkeydigest,json=enforceIdkeydigest,proto3" json:"enforce_idkeydigest,omitempty"`
ConformanceMode bool `protobuf:"varint,14,opt,name=conformance_mode,json=conformanceMode,proto3" json:"conformance_mode,omitempty"`
KubernetesComponents []*KubernetesComponent `protobuf:"bytes,15,rep,name=kubernetes_components,json=kubernetesComponents,proto3" json:"kubernetes_components,omitempty"`
}
func (x *InitRequest) Reset() {
@ -157,6 +158,13 @@ func (x *InitRequest) GetConformanceMode() bool {
return false
}
func (x *InitRequest) GetKubernetesComponents() []*KubernetesComponent {
if x != nil {
return x.KubernetesComponents
}
return nil
}
type InitResponse struct {
state protoimpl.MessageState
sizeCache protoimpl.SizeCache
@ -220,11 +228,82 @@ func (x *InitResponse) GetClusterId() []byte {
return nil
}
type KubernetesComponent struct {
state protoimpl.MessageState
sizeCache protoimpl.SizeCache
unknownFields protoimpl.UnknownFields
Url string `protobuf:"bytes,1,opt,name=url,proto3" json:"url,omitempty"`
Hash string `protobuf:"bytes,2,opt,name=hash,proto3" json:"hash,omitempty"`
InstallPath string `protobuf:"bytes,3,opt,name=install_path,json=installPath,proto3" json:"install_path,omitempty"`
Extract bool `protobuf:"varint,4,opt,name=extract,proto3" json:"extract,omitempty"`
}
func (x *KubernetesComponent) Reset() {
*x = KubernetesComponent{}
if protoimpl.UnsafeEnabled {
mi := &file_init_proto_msgTypes[2]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
}
func (x *KubernetesComponent) String() string {
return protoimpl.X.MessageStringOf(x)
}
func (*KubernetesComponent) ProtoMessage() {}
func (x *KubernetesComponent) ProtoReflect() protoreflect.Message {
mi := &file_init_proto_msgTypes[2]
if protoimpl.UnsafeEnabled && x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
ms.StoreMessageInfo(mi)
}
return ms
}
return mi.MessageOf(x)
}
// Deprecated: Use KubernetesComponent.ProtoReflect.Descriptor instead.
func (*KubernetesComponent) Descriptor() ([]byte, []int) {
return file_init_proto_rawDescGZIP(), []int{2}
}
func (x *KubernetesComponent) GetUrl() string {
if x != nil {
return x.Url
}
return ""
}
func (x *KubernetesComponent) GetHash() string {
if x != nil {
return x.Hash
}
return ""
}
func (x *KubernetesComponent) GetInstallPath() string {
if x != nil {
return x.InstallPath
}
return ""
}
func (x *KubernetesComponent) GetExtract() bool {
if x != nil {
return x.Extract
}
return false
}
var File_init_proto protoreflect.FileDescriptor
var file_init_proto_rawDesc = []byte{
0x0a, 0x0a, 0x69, 0x6e, 0x69, 0x74, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x04, 0x69, 0x6e,
0x69, 0x74, 0x22, 0xf3, 0x03, 0x0a, 0x0b, 0x49, 0x6e, 0x69, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65,
0x69, 0x74, 0x22, 0xc3, 0x04, 0x0a, 0x0b, 0x49, 0x6e, 0x69, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65,
0x73, 0x74, 0x12, 0x23, 0x0a, 0x0d, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x73, 0x65, 0x63,
0x72, 0x65, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0c, 0x6d, 0x61, 0x73, 0x74, 0x65,
0x72, 0x53, 0x65, 0x63, 0x72, 0x65, 0x74, 0x12, 0x17, 0x0a, 0x07, 0x6b, 0x6d, 0x73, 0x5f, 0x75,
@ -255,22 +334,34 @@ var file_init_proto_rawDesc = []byte{
0x63, 0x65, 0x49, 0x64, 0x6b, 0x65, 0x79, 0x64, 0x69, 0x67, 0x65, 0x73, 0x74, 0x12, 0x29, 0x0a,
0x10, 0x63, 0x6f, 0x6e, 0x66, 0x6f, 0x72, 0x6d, 0x61, 0x6e, 0x63, 0x65, 0x5f, 0x6d, 0x6f, 0x64,
0x65, 0x18, 0x0e, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0f, 0x63, 0x6f, 0x6e, 0x66, 0x6f, 0x72, 0x6d,
0x61, 0x6e, 0x63, 0x65, 0x4d, 0x6f, 0x64, 0x65, 0x22, 0x68, 0x0a, 0x0c, 0x49, 0x6e, 0x69, 0x74,
0x61, 0x6e, 0x63, 0x65, 0x4d, 0x6f, 0x64, 0x65, 0x12, 0x4e, 0x0a, 0x15, 0x6b, 0x75, 0x62, 0x65,
0x72, 0x6e, 0x65, 0x74, 0x65, 0x73, 0x5f, 0x63, 0x6f, 0x6d, 0x70, 0x6f, 0x6e, 0x65, 0x6e, 0x74,
0x73, 0x18, 0x0f, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x69, 0x6e, 0x69, 0x74, 0x2e, 0x4b,
0x75, 0x62, 0x65, 0x72, 0x6e, 0x65, 0x74, 0x65, 0x73, 0x43, 0x6f, 0x6d, 0x70, 0x6f, 0x6e, 0x65,
0x6e, 0x74, 0x52, 0x14, 0x6b, 0x75, 0x62, 0x65, 0x72, 0x6e, 0x65, 0x74, 0x65, 0x73, 0x43, 0x6f,
0x6d, 0x70, 0x6f, 0x6e, 0x65, 0x6e, 0x74, 0x73, 0x22, 0x68, 0x0a, 0x0c, 0x49, 0x6e, 0x69, 0x74,
0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x1e, 0x0a, 0x0a, 0x6b, 0x75, 0x62, 0x65,
0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0a, 0x6b, 0x75,
0x62, 0x65, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x19, 0x0a, 0x08, 0x6f, 0x77, 0x6e, 0x65,
0x72, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x07, 0x6f, 0x77, 0x6e, 0x65,
0x72, 0x49, 0x64, 0x12, 0x1d, 0x0a, 0x0a, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x69,
0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x09, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72,
0x49, 0x64, 0x32, 0x34, 0x0a, 0x03, 0x41, 0x50, 0x49, 0x12, 0x2d, 0x0a, 0x04, 0x49, 0x6e, 0x69,
0x74, 0x12, 0x11, 0x2e, 0x69, 0x6e, 0x69, 0x74, 0x2e, 0x49, 0x6e, 0x69, 0x74, 0x52, 0x65, 0x71,
0x75, 0x65, 0x73, 0x74, 0x1a, 0x12, 0x2e, 0x69, 0x6e, 0x69, 0x74, 0x2e, 0x49, 0x6e, 0x69, 0x74,
0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x42, 0x40, 0x5a, 0x3e, 0x67, 0x69, 0x74, 0x68,
0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x65, 0x64, 0x67, 0x65, 0x6c, 0x65, 0x73, 0x73, 0x73,
0x79, 0x73, 0x2f, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x65, 0x6c, 0x6c, 0x61, 0x74, 0x69, 0x6f, 0x6e,
0x2f, 0x76, 0x32, 0x2f, 0x62, 0x6f, 0x6f, 0x74, 0x73, 0x74, 0x72, 0x61, 0x70, 0x70, 0x65, 0x72,
0x2f, 0x69, 0x6e, 0x69, 0x74, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74,
0x6f, 0x33,
0x49, 0x64, 0x22, 0x78, 0x0a, 0x13, 0x4b, 0x75, 0x62, 0x65, 0x72, 0x6e, 0x65, 0x74, 0x65, 0x73,
0x43, 0x6f, 0x6d, 0x70, 0x6f, 0x6e, 0x65, 0x6e, 0x74, 0x12, 0x10, 0x0a, 0x03, 0x75, 0x72, 0x6c,
0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x75, 0x72, 0x6c, 0x12, 0x12, 0x0a, 0x04, 0x68,
0x61, 0x73, 0x68, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x68, 0x61, 0x73, 0x68, 0x12,
0x21, 0x0a, 0x0c, 0x69, 0x6e, 0x73, 0x74, 0x61, 0x6c, 0x6c, 0x5f, 0x70, 0x61, 0x74, 0x68, 0x18,
0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x69, 0x6e, 0x73, 0x74, 0x61, 0x6c, 0x6c, 0x50, 0x61,
0x74, 0x68, 0x12, 0x18, 0x0a, 0x07, 0x65, 0x78, 0x74, 0x72, 0x61, 0x63, 0x74, 0x18, 0x04, 0x20,
0x01, 0x28, 0x08, 0x52, 0x07, 0x65, 0x78, 0x74, 0x72, 0x61, 0x63, 0x74, 0x32, 0x34, 0x0a, 0x03,
0x41, 0x50, 0x49, 0x12, 0x2d, 0x0a, 0x04, 0x49, 0x6e, 0x69, 0x74, 0x12, 0x11, 0x2e, 0x69, 0x6e,
0x69, 0x74, 0x2e, 0x49, 0x6e, 0x69, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x12,
0x2e, 0x69, 0x6e, 0x69, 0x74, 0x2e, 0x49, 0x6e, 0x69, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e,
0x73, 0x65, 0x42, 0x40, 0x5a, 0x3e, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d,
0x2f, 0x65, 0x64, 0x67, 0x65, 0x6c, 0x65, 0x73, 0x73, 0x73, 0x79, 0x73, 0x2f, 0x63, 0x6f, 0x6e,
0x73, 0x74, 0x65, 0x6c, 0x6c, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x2f, 0x76, 0x32, 0x2f, 0x62, 0x6f,
0x6f, 0x74, 0x73, 0x74, 0x72, 0x61, 0x70, 0x70, 0x65, 0x72, 0x2f, 0x69, 0x6e, 0x69, 0x74, 0x70,
0x72, 0x6f, 0x74, 0x6f, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
}
var (
@ -285,19 +376,21 @@ func file_init_proto_rawDescGZIP() []byte {
return file_init_proto_rawDescData
}
var file_init_proto_msgTypes = make([]protoimpl.MessageInfo, 2)
var file_init_proto_msgTypes = make([]protoimpl.MessageInfo, 3)
var file_init_proto_goTypes = []interface{}{
(*InitRequest)(nil), // 0: init.InitRequest
(*InitResponse)(nil), // 1: init.InitResponse
(*KubernetesComponent)(nil), // 2: init.KubernetesComponent
}
var file_init_proto_depIdxs = []int32{
0, // 0: init.API.Init:input_type -> init.InitRequest
1, // 1: init.API.Init:output_type -> init.InitResponse
1, // [1:2] is the sub-list for method output_type
0, // [0:1] is the sub-list for method input_type
0, // [0:0] is the sub-list for extension type_name
0, // [0:0] is the sub-list for extension extendee
0, // [0:0] is the sub-list for field type_name
2, // 0: init.InitRequest.kubernetes_components:type_name -> init.KubernetesComponent
0, // 1: init.API.Init:input_type -> init.InitRequest
1, // 2: init.API.Init:output_type -> init.InitResponse
2, // [2:3] is the sub-list for method output_type
1, // [1:2] is the sub-list for method input_type
1, // [1:1] is the sub-list for extension type_name
1, // [1:1] is the sub-list for extension extendee
0, // [0:1] is the sub-list for field type_name
}
func init() { file_init_proto_init() }
@ -330,6 +423,18 @@ func file_init_proto_init() {
return nil
}
}
file_init_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} {
switch v := v.(*KubernetesComponent); i {
case 0:
return &v.state
case 1:
return &v.sizeCache
case 2:
return &v.unknownFields
default:
return nil
}
}
}
type x struct{}
out := protoimpl.TypeBuilder{
@ -337,7 +442,7 @@ func file_init_proto_init() {
GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
RawDescriptor: file_init_proto_rawDesc,
NumEnums: 0,
NumMessages: 2,
NumMessages: 3,
NumExtensions: 0,
NumServices: 1,
},

View File

@ -23,6 +23,7 @@ message InitRequest {
repeated uint32 enforced_pcrs = 12;
bool enforce_idkeydigest = 13;
bool conformance_mode = 14;
repeated KubernetesComponent kubernetes_components = 15;
}
message InitResponse {
@ -30,3 +31,10 @@ message InitResponse {
bytes owner_id = 2;
bytes cluster_id = 3;
}
message KubernetesComponent {
string url = 1;
string hash = 2;
string install_path = 3;
bool extract = 4;
}

View File

@ -25,6 +25,7 @@ import (
"github.com/edgelesssys/constellation/v2/internal/logger"
"github.com/edgelesssys/constellation/v2/internal/nodestate"
"github.com/edgelesssys/constellation/v2/internal/role"
"github.com/edgelesssys/constellation/v2/internal/versions"
"go.uber.org/zap"
"google.golang.org/grpc"
"google.golang.org/grpc/codes"
@ -133,6 +134,7 @@ func (s *Server) Init(ctx context.Context, req *initproto.InitRequest) (*initpro
s.issuerWrapper.VMType() == vmtype.AzureCVM,
req.HelmDeployments,
req.ConformanceMode,
versions.NewComponentVersionsFromProto(req.KubernetesComponents),
s.log,
)
if err != nil {
@ -233,6 +235,7 @@ type ClusterInitializer interface {
azureCVM bool,
helmDeployments []byte,
conformanceMode bool,
kubernetesComponents versions.ComponentVersions,
log *logger.Logger,
) ([]byte, error)
}

View File

@ -19,6 +19,7 @@ import (
"github.com/edgelesssys/constellation/v2/internal/crypto/testvector"
"github.com/edgelesssys/constellation/v2/internal/file"
"github.com/edgelesssys/constellation/v2/internal/logger"
"github.com/edgelesssys/constellation/v2/internal/versions"
"github.com/spf13/afero"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
@ -250,7 +251,7 @@ type stubClusterInitializer struct {
func (i *stubClusterInitializer) InitCluster(
context.Context, string, string, []byte, []uint32, bool, []byte, bool,
[]byte, bool, *logger.Logger,
[]byte, bool, versions.ComponentVersions, *logger.Logger,
) ([]byte, error) {
return i.initClusterKubeconfig, i.initClusterErr
}

View File

@ -1,17 +0,0 @@
/*
Copyright (c) Edgeless Systems GmbH
SPDX-License-Identifier: AGPL-3.0-only
*/
package k8sapi
const (
// Paths and permissions necessary for Kubernetes installation.
cniPluginsDir = "/opt/cni/bin"
binDir = "/run/state/bin"
kubeadmPath = "/run/state/bin/kubeadm"
kubeletPath = "/run/state/bin/kubelet"
kubeletServicePath = "/usr/lib/systemd/system/kubelet.service"
executablePerm = 0o544
)

View File

@ -10,6 +10,7 @@ import (
"archive/tar"
"compress/gzip"
"context"
"crypto/sha256"
"errors"
"fmt"
"io"
@ -20,6 +21,7 @@ import (
"time"
"github.com/edgelesssys/constellation/v2/internal/retry"
"github.com/edgelesssys/constellation/v2/internal/versions"
"github.com/spf13/afero"
"k8s.io/utils/clock"
)
@ -50,28 +52,38 @@ func newOSInstaller() *osInstaller {
}
// Install downloads a resource from a URL, applies any given text transformations and extracts the resulting file if required.
// The resulting file(s) are copied to all destinations.
func (i *osInstaller) Install(
ctx context.Context, sourceURL string, destinations []string, perm fs.FileMode, extract bool,
) error {
tempPath, err := i.retryDownloadToTempDir(ctx, sourceURL)
// The resulting file(s) are copied to the destination. It also verifies the sha256 hash of the downloaded file.
func (i *osInstaller) Install(ctx context.Context, kubernetesComponent versions.ComponentVersion) error {
tempPath, err := i.retryDownloadToTempDir(ctx, kubernetesComponent.URL)
if err != nil {
return err
}
file, err := i.fs.OpenFile(tempPath, os.O_RDONLY, 0)
if err != nil {
return fmt.Errorf("opening file %q: %w", tempPath, err)
}
sha := sha256.New()
if _, err := io.Copy(sha, file); err != nil {
return fmt.Errorf("reading file %q: %w", tempPath, err)
}
calculatedHash := fmt.Sprintf("sha256:%x", sha.Sum(nil))
if calculatedHash != kubernetesComponent.Hash {
return fmt.Errorf("hash of file %q %s does not match expected hash %s", tempPath, calculatedHash, kubernetesComponent.Hash)
}
defer func() {
_ = i.fs.Remove(tempPath)
}()
for _, destination := range destinations {
var err error
if extract {
err = i.extractArchive(tempPath, destination, perm)
if kubernetesComponent.Extract {
err = i.extractArchive(tempPath, kubernetesComponent.InstallPath, executablePerm)
} else {
err = i.copy(tempPath, destination, perm)
err = i.copy(tempPath, kubernetesComponent.InstallPath, executablePerm)
}
if err != nil {
return fmt.Errorf("installing from %q: copying to destination %q: %w", sourceURL, destination, err)
}
return fmt.Errorf("installing from %q: copying to destination %q: %w", kubernetesComponent.URL, kubernetesComponent.InstallPath, err)
}
return nil
}

View File

@ -22,6 +22,7 @@ import (
"testing"
"time"
"github.com/edgelesssys/constellation/v2/internal/versions"
"github.com/spf13/afero"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
@ -31,28 +32,51 @@ import (
)
func TestInstall(t *testing.T) {
serverURL := "http://server/path"
testCases := map[string]struct {
server httpBufconnServer
component versions.ComponentVersion
hash string
destination string
extract bool
readonly bool
wantErr bool
wantFiles map[string][]byte
}{
"download works": {
server: newHTTPBufconnServerWithBody([]byte("file-contents")),
destination: "/destination",
component: versions.ComponentVersion{
URL: serverURL,
Hash: "sha256:f03779b36bece74893fd6533a67549675e21573eb0e288d87158738f9c24594e",
InstallPath: "/destination",
},
wantFiles: map[string][]byte{"/destination": []byte("file-contents")},
},
"download with extract works": {
server: newHTTPBufconnServerWithBody(createTarGz([]byte("file-contents"), "/destination")),
destination: "/prefix",
extract: true,
component: versions.ComponentVersion{
URL: serverURL,
Hash: "sha256:a52a1664ca0a6ec9790384e3d058852ab8b3a8f389a9113d150fdc6ab308d949",
InstallPath: "/prefix",
Extract: true,
},
wantFiles: map[string][]byte{"/prefix/destination": []byte("file-contents")},
},
"hash validation fails": {
server: newHTTPBufconnServerWithBody([]byte("file-contents")),
component: versions.ComponentVersion{
URL: serverURL,
Hash: "sha256:abc",
InstallPath: "/destination",
},
wantErr: true,
},
"download fails": {
server: newHTTPBufconnServer(func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(500) }),
destination: "/destination",
component: versions.ComponentVersion{
URL: serverURL,
Hash: "sha256:abc",
InstallPath: "/destination",
},
wantErr: true,
},
}
@ -81,7 +105,7 @@ func TestInstall(t *testing.T) {
retriable: func(err error) bool { return false },
}
err := inst.Install(context.Background(), "http://server/path", []string{tc.destination}, fs.ModePerm, tc.extract)
err := inst.Install(context.Background(), tc.component)
if tc.wantErr {
assert.Error(err)
return

View File

@ -13,7 +13,6 @@ import (
"encoding/pem"
"errors"
"fmt"
"io/fs"
"net"
"net/http"
"os"
@ -42,6 +41,10 @@ import (
const (
// kubeletStartTimeout is the maximum time given to the kubelet service to (re)start.
kubeletStartTimeout = 10 * time.Minute
// crdTimeout is the maximum time given to the CRDs to be created.
crdTimeout = 30 * time.Second
executablePerm = 0o544
kubeletServicePath = "/usr/lib/systemd/system/kubelet.service"
)
// Client provides the functions to talk to the k8s API.
@ -57,7 +60,7 @@ type Client interface {
type installer interface {
Install(
ctx context.Context, sourceURL string, destinations []string, perm fs.FileMode, extract bool,
ctx context.Context, kubernetesComponent versions.ComponentVersion,
) error
}
@ -75,34 +78,26 @@ func NewKubernetesUtil() *KubernetesUtil {
}
}
// InstallComponentsFromCLI installs the kubernetes components passed from the CLI.
func (k *KubernetesUtil) InstallComponentsFromCLI(ctx context.Context, kubernetesComponents versions.ComponentVersions) error {
for _, component := range kubernetesComponents {
if err := k.inst.Install(ctx, component); err != nil {
return fmt.Errorf("installing kubernetes component from URL %s: %w", component.URL, err)
}
}
return enableSystemdUnit(ctx, kubeletServicePath)
}
// InstallComponents installs kubernetes components in the version specified.
// TODO(AB#2543,3u13r): Remove this function once the JoinService is extended.
func (k *KubernetesUtil) InstallComponents(ctx context.Context, version versions.ValidK8sVersion) error {
versionConf := versions.VersionConfigs[version]
if err := k.inst.Install(
ctx, versionConf.CNIPlugins.URL, []string{cniPluginsDir}, executablePerm, true,
); err != nil {
return fmt.Errorf("installing cni plugins: %w", err)
for _, component := range versionConf.KubernetesComponents {
if err := k.inst.Install(ctx, component); err != nil {
return fmt.Errorf("installing kubernetes component from URL %s: %w", component.URL, err)
}
if err := k.inst.Install(
ctx, versionConf.Crictl.URL, []string{binDir}, executablePerm, true,
); err != nil {
return fmt.Errorf("installing crictl: %w", err)
}
if err := k.inst.Install(
ctx, versionConf.Kubelet.URL, []string{kubeletPath}, executablePerm, false,
); err != nil {
return fmt.Errorf("installing kubelet: %w", err)
}
if err := k.inst.Install(
ctx, versionConf.Kubeadm.URL, []string{kubeadmPath}, executablePerm, false,
); err != nil {
return fmt.Errorf("installing kubeadm: %w", err)
}
if err := k.inst.Install(
ctx, versionConf.Kubectl.URL, []string{constants.KubectlPath}, executablePerm, false,
); err != nil {
return fmt.Errorf("installing kubectl: %w", err)
}
return enableSystemdUnit(ctx, kubeletServicePath)
@ -132,7 +127,7 @@ func (k *KubernetesUtil) InitCluster(
// preflight
log.Infof("Running kubeadm preflight checks")
cmd := exec.CommandContext(ctx, kubeadmPath, "init", "phase", "preflight", "-v=5", "--config", initConfigFile.Name())
cmd := exec.CommandContext(ctx, constants.KubeadmPath, "init", "phase", "preflight", "-v=5", "--config", initConfigFile.Name())
out, err := cmd.CombinedOutput()
if err != nil {
var exitErr *exec.ExitError
@ -144,7 +139,7 @@ func (k *KubernetesUtil) InitCluster(
// create CA certs
log.Infof("Creating Kubernetes control-plane certificates and keys")
cmd = exec.CommandContext(ctx, kubeadmPath, "init", "phase", "certs", "all", "-v=5", "--config", initConfigFile.Name())
cmd = exec.CommandContext(ctx, constants.KubeadmPath, "init", "phase", "certs", "all", "-v=5", "--config", initConfigFile.Name())
out, err = cmd.CombinedOutput()
if err != nil {
var exitErr *exec.ExitError
@ -172,7 +167,7 @@ func (k *KubernetesUtil) InitCluster(
skipPhases += ",addon/kube-proxy"
}
cmd = exec.CommandContext(ctx, kubeadmPath, "init", "-v=5", skipPhases, "--config", initConfigFile.Name())
cmd = exec.CommandContext(ctx, constants.KubeadmPath, "init", "-v=5", skipPhases, "--config", initConfigFile.Name())
out, err = cmd.CombinedOutput()
if err != nil {
var exitErr *exec.ExitError
@ -362,7 +357,7 @@ func (k *KubernetesUtil) JoinCluster(ctx context.Context, joinConfig []byte, pee
}
// run `kubeadm join` to join a worker node to an existing Kubernetes cluster
cmd := exec.CommandContext(ctx, kubeadmPath, "join", "-v=5", "--config", joinConfigFile.Name())
cmd := exec.CommandContext(ctx, constants.KubeadmPath, "join", "-v=5", "--config", joinConfigFile.Name())
out, err := cmd.CombinedOutput()
if err != nil {
var exitErr *exec.ExitError

View File

@ -20,6 +20,7 @@ import (
type clusterUtil interface {
InstallComponents(ctx context.Context, version versions.ValidK8sVersion) error
InstallComponentsFromCLI(ctx context.Context, kubernetesComponents versions.ComponentVersions) error
InitCluster(ctx context.Context, initConfig []byte, nodeName string, ips []net.IP, controlPlaneEndpoint string, conformanceMode bool, log *logger.Logger) error
JoinCluster(ctx context.Context, joinConfig []byte, peerRole role.Role, controlPlaneEndpoint string, log *logger.Logger) error
SetupKonnectivity(kubectl k8sapi.Client, konnectivityAgentsDaemonSet kubernetes.Marshaler) error

View File

@ -87,14 +87,14 @@ func New(cloudProvider string, clusterUtil clusterUtil, configProvider configura
func (k *KubeWrapper) InitCluster(
ctx context.Context, cloudServiceAccountURI, versionString string, measurementSalt []byte, enforcedPCRs []uint32,
enforceIDKeyDigest bool, idKeyDigest []byte, azureCVM bool,
helmReleasesRaw []byte, conformanceMode bool, log *logger.Logger,
helmReleasesRaw []byte, conformanceMode bool, kubernetesComponents versions.ComponentVersions, log *logger.Logger,
) ([]byte, error) {
k8sVersion, err := versions.NewValidK8sVersion(versionString)
if err != nil {
return nil, err
}
log.With(zap.String("version", string(k8sVersion))).Infof("Installing Kubernetes components")
if err := k.clusterUtil.InstallComponents(ctx, k8sVersion); err != nil {
if err := k.clusterUtil.InstallComponentsFromCLI(ctx, kubernetesComponents); err != nil {
return nil, err
}

View File

@ -237,7 +237,7 @@ func TestInitCluster(t *testing.T) {
_, err := kube.InitCluster(
context.Background(), serviceAccountURI, string(tc.k8sVersion),
nil, nil, false, nil, true, []byte("{}"), false, logger.NewTest(t),
nil, nil, false, nil, true, []byte("{}"), false, nil, logger.NewTest(t),
)
if tc.wantErr {
@ -416,6 +416,7 @@ func TestK8sCompliantHostname(t *testing.T) {
type stubClusterUtil struct {
installComponentsErr error
installComponentsFromCLIErr error
initClusterErr error
setupAutoscalingError error
setupKonnectivityError error
@ -438,6 +439,10 @@ func (s *stubClusterUtil) InstallComponents(ctx context.Context, version version
return s.installComponentsErr
}
func (s *stubClusterUtil) InstallComponentsFromCLI(ctx context.Context, kubernetesComponents versions.ComponentVersions) error {
return s.installComponentsFromCLIErr
}
// TODO: Upon changing this function, please refactor it to reduce the number of arguments to <= 5.
//
//revive:disable-next-line

View File

@ -132,6 +132,7 @@ func initialize(cmd *cobra.Command, newDialer func(validator *cloudcmd.Validator
UseExistingKek: false,
CloudServiceAccountUri: serviceAccURI,
KubernetesVersion: conf.KubernetesVersion,
KubernetesComponents: versions.VersionConfigs[k8sVersion].KubernetesComponents.ToProto(),
HelmDeployments: helmDeployments,
EnforcedPcrs: conf.GetEnforcedPCRs(),
EnforceIdkeydigest: conf.EnforcesIDKeyDigest(),

View File

@ -56,12 +56,6 @@ const (
DebugdPort = 4000
// KonnectivityPort port for konnectivity k8s service.
KonnectivityPort = 8132
// NodePortFrom start of range to use for K8s node port
// https://kubernetes.io/docs/concepts/services-networking/service/#type-nodeport
NodePortFrom = 30000
// NodePortTo end of range to use for K8s node port
// https://kubernetes.io/docs/concepts/services-networking/service/#type-nodeport
NodePortTo = 32767
//
// Filenames.
@ -84,6 +78,15 @@ const (
// KubectlPath path to kubectl binary.
KubectlPath = "/run/state/bin/kubectl"
// CniPluginsDir path directory for CNI plugins.
CniPluginsDir = "/opt/cni/bin"
// BinDir install path for CNI config.
BinDir = "/run/state/bin"
// KubeadmPath install path for kubeadm.
KubeadmPath = "/run/state/bin/kubeadm"
// KubeletPath install path for kubelet.
KubeletPath = "/run/state/bin/kubelet"
//
// Filenames for Constellation's micro services.
//

View File

@ -102,11 +102,15 @@ func main() {
if !ok {
return true
}
if ident.Name == "ArtifactVersion" {
if ident.Name == "ComponentVersions" {
for _, elt := range x.Elts {
// component is one list element
component := elt.(*ast.CompositeLit)
var url *ast.KeyValueExpr
var hash *ast.KeyValueExpr
// Find the URL field
for _, e := range x.Elts {
for _, e := range component.Elts {
kv, ok := e.(*ast.KeyValueExpr)
if !ok {
continue
@ -121,7 +125,7 @@ func main() {
}
}
// Find the Hash field
for _, e := range x.Elts {
for _, e := range component.Elts {
kv, ok := e.(*ast.KeyValueExpr)
if !ok {
continue
@ -141,6 +145,7 @@ func main() {
hash.Value.(*ast.BasicLit).Value = mustGetHash(url.Value.(*ast.BasicLit).Value)
}
}
}
return true
}, nil,
)

View File

@ -9,6 +9,9 @@ package versions
import (
"fmt"
"strings"
"github.com/edgelesssys/constellation/v2/bootstrapper/initproto"
"github.com/edgelesssys/constellation/v2/internal/constants"
)
// ValidK8sVersion represents any of the three currently supported k8s versions.
@ -93,25 +96,37 @@ const (
var VersionConfigs = map[ValidK8sVersion]KubernetesVersion{
V1_23: {
PatchVersion: "v1.23.14", // renovate:kubernetes-release
CNIPlugins: ArtifactVersion{
KubernetesComponents: ComponentVersions{
{
URL: "https://github.com/containernetworking/plugins/releases/download/v1.1.1/cni-plugins-linux-amd64-v1.1.1.tgz", // renovate:cni-plugins-release
Hash: "sha256:b275772da4026d2161bf8a8b41ed4786754c8a93ebfb6564006d5da7f23831e5",
InstallPath: constants.CniPluginsDir,
Extract: true,
},
Crictl: ArtifactVersion{
{
URL: "https://github.com/kubernetes-sigs/cri-tools/releases/download/v1.25.0/crictl-v1.25.0-linux-amd64.tar.gz", // renovate:crictl-release
Hash: "sha256:86ab210c007f521ac4cdcbcf0ae3fb2e10923e65f16de83e0e1db191a07f0235",
InstallPath: constants.BinDir,
Extract: true,
},
Kubelet: ArtifactVersion{
{
URL: "https://storage.googleapis.com/kubernetes-release/release/v1.23.14/bin/linux/amd64/kubelet", // renovate:kubernetes-release
Hash: "sha256:f2bef00508790f632d035a6cfdd31539115611bfc93c5a3266ceb95bb2f27b76",
InstallPath: constants.KubeletPath,
Extract: false,
},
Kubeadm: ArtifactVersion{
{
URL: "https://storage.googleapis.com/kubernetes-release/release/v1.23.14/bin/linux/amd64/kubeadm", // renovate:kubernetes-release
Hash: "sha256:46c847e2699839b9ccf6673f0b946c4778a3a2e8e463d15854ba30d3f0cbd87a",
InstallPath: constants.KubeadmPath,
Extract: false,
},
Kubectl: ArtifactVersion{
{
URL: "https://storage.googleapis.com/kubernetes-release/release/v1.23.14/bin/linux/amd64/kubectl", // renovate:kubernetes-release
Hash: "sha256:13ce4b18ba6e15d5d259249c530637dd7fb9722d121df022099f3ed5f2bd74cd",
InstallPath: constants.KubectlPath,
Extract: false,
},
},
// CloudControllerManagerImageAWS is the CCM image used on AWS.
CloudControllerManagerImageAWS: "registry.k8s.io/provider-aws/cloud-controller-manager:v1.23.2@sha256:5caf74bfe1c6e1b7b7d40345db52b54eeea7229a8fd73c7db9488ef87dc7a496", // renovate:container
@ -127,25 +142,37 @@ var VersionConfigs = map[ValidK8sVersion]KubernetesVersion{
},
V1_24: {
PatchVersion: "v1.24.8", // renovate:kubernetes-release
CNIPlugins: ArtifactVersion{
KubernetesComponents: ComponentVersions{
{
URL: "https://github.com/containernetworking/plugins/releases/download/v1.1.1/cni-plugins-linux-amd64-v1.1.1.tgz", // renovate:cni-plugins-release
Hash: "sha256:b275772da4026d2161bf8a8b41ed4786754c8a93ebfb6564006d5da7f23831e5",
InstallPath: constants.CniPluginsDir,
Extract: true,
},
Crictl: ArtifactVersion{
{
URL: "https://github.com/kubernetes-sigs/cri-tools/releases/download/v1.25.0/crictl-v1.25.0-linux-amd64.tar.gz", // renovate:crictl-release
Hash: "sha256:86ab210c007f521ac4cdcbcf0ae3fb2e10923e65f16de83e0e1db191a07f0235",
InstallPath: constants.BinDir,
Extract: true,
},
Kubelet: ArtifactVersion{
{
URL: "https://storage.googleapis.com/kubernetes-release/release/v1.24.8/bin/linux/amd64/kubelet", // renovate:kubernetes-release
Hash: "sha256:2da0b93857cf352bff5d1eb42e34d398a5971b63a53d8687b45179a78540d6d6",
InstallPath: constants.KubeletPath,
Extract: false,
},
Kubeadm: ArtifactVersion{
{
URL: "https://storage.googleapis.com/kubernetes-release/release/v1.24.8/bin/linux/amd64/kubeadm", // renovate:kubernetes-release
Hash: "sha256:9fea42b4fb5eb2da638d20710ebb791dde221e6477793d3de70134ac058c4cc7",
InstallPath: constants.KubeadmPath,
Extract: false,
},
Kubectl: ArtifactVersion{
{
URL: "https://storage.googleapis.com/kubernetes-release/release/v1.24.8/bin/linux/amd64/kubectl", // renovate:kubernetes-release
Hash: "sha256:f93c18751ec715b4d4437e7ece18fe91948c71be1f24ab02a2dde150f5449855",
InstallPath: constants.KubectlPath,
Extract: false,
},
},
// CloudControllerManagerImageAWS is the CCM image used on AWS.
CloudControllerManagerImageAWS: "registry.k8s.io/provider-aws/cloud-controller-manager:v1.24.1@sha256:4b75b09cc5b3959d06a8c2fb84f165e8163ec0153eaa6a48ece6c8113e78e720", // renovate:container
@ -161,25 +188,37 @@ var VersionConfigs = map[ValidK8sVersion]KubernetesVersion{
},
V1_25: {
PatchVersion: "v1.25.4", // renovate:kubernetes-release
CNIPlugins: ArtifactVersion{
KubernetesComponents: ComponentVersions{
{
URL: "https://github.com/containernetworking/plugins/releases/download/v1.1.1/cni-plugins-linux-amd64-v1.1.1.tgz", // renovate:cni-plugins-release
Hash: "sha256:b275772da4026d2161bf8a8b41ed4786754c8a93ebfb6564006d5da7f23831e5",
InstallPath: constants.CniPluginsDir,
Extract: true,
},
Crictl: ArtifactVersion{
{
URL: "https://github.com/kubernetes-sigs/cri-tools/releases/download/v1.25.0/crictl-v1.25.0-linux-amd64.tar.gz", // renovate:crictl-release
Hash: "sha256:86ab210c007f521ac4cdcbcf0ae3fb2e10923e65f16de83e0e1db191a07f0235",
InstallPath: constants.BinDir,
Extract: true,
},
Kubelet: ArtifactVersion{
{
URL: "https://storage.googleapis.com/kubernetes-release/release/v1.25.4/bin/linux/amd64/kubelet", // renovate:kubernetes-release
Hash: "sha256:7f7437e361f829967ee02e30026d7e85219693432ac5e930cc98dd9c7ddb2fac",
InstallPath: constants.KubeletPath,
Extract: false,
},
Kubeadm: ArtifactVersion{
{
URL: "https://storage.googleapis.com/kubernetes-release/release/v1.25.4/bin/linux/amd64/kubeadm", // renovate:kubernetes-release
Hash: "sha256:b8a6119d2a3a7c6add43dcf8f920436bf7fe71a77a086e96e40aa9d6f70be826",
InstallPath: constants.KubeadmPath,
Extract: false,
},
Kubectl: ArtifactVersion{
{
URL: "https://storage.googleapis.com/kubernetes-release/release/v1.25.4/bin/linux/amd64/kubectl", // renovate:kubernetes-release
Hash: "sha256:e4e569249798a09f37e31b8b33571970fcfbdecdd99b1b81108adc93ca74b522",
InstallPath: constants.KubectlPath,
Extract: false,
},
},
// CloudControllerManagerImageAWS is the CCM image used on AWS.
CloudControllerManagerImageAWS: "registry.k8s.io/provider-aws/cloud-controller-manager:v1.25.1@sha256:85d3f1e9dacc72531445989bb10999e1e70ebc409d11be57e5baa5f031a893b0", // renovate:container
@ -201,11 +240,7 @@ var VersionConfigs = map[ValidK8sVersion]KubernetesVersion{
// KubernetesVersion bundles download URLs to all version-releated binaries necessary for installing/deploying a particular Kubernetes version.
type KubernetesVersion struct {
PatchVersion string
CNIPlugins ArtifactVersion // No k8s version dependency.
Crictl ArtifactVersion // k8s version dependency.
Kubelet ArtifactVersion // k8s version dependency.
Kubeadm ArtifactVersion // k8s version dependency.
Kubectl ArtifactVersion // k8s version dependency.
KubernetesComponents ComponentVersions
CloudControllerManagerImageAWS string // k8s version dependency.
CloudControllerManagerImageGCP string // Using self-built image until resolved: https://github.com/kubernetes/cloud-provider-gcp/issues/289
CloudControllerManagerImageAzure string // k8s version dependency.
@ -213,10 +248,36 @@ type KubernetesVersion struct {
ClusterAutoscalerImage string // Matches k8s versioning scheme.
}
// ArtifactVersion is a version of a particular artifact.
type ArtifactVersion struct {
// ComponentVersion is a version of a particular artifact.
type ComponentVersion struct {
URL string
Hash string
InstallPath string
Extract bool
}
// ComponentVersions is a list of ComponentVersion.
type ComponentVersions []ComponentVersion
// NewComponentVersionsFromProto converts a protobuf KubernetesVersion to ComponentVersions.
func NewComponentVersionsFromProto(protoComponents []*initproto.KubernetesComponent) ComponentVersions {
components := ComponentVersions{}
for _, protoComponent := range protoComponents {
if protoComponent == nil {
continue
}
components = append(components, ComponentVersion{URL: protoComponent.Url, Hash: protoComponent.Hash, InstallPath: protoComponent.InstallPath, Extract: protoComponent.Extract})
}
return components
}
// ToProto converts a ComponentVersions to a protobuf KubernetesVersion.
func (c ComponentVersions) ToProto() []*initproto.KubernetesComponent {
protoComponents := []*initproto.KubernetesComponent{}
for _, component := range c {
protoComponents = append(protoComponents, &initproto.KubernetesComponent{Url: component.URL, Hash: component.Hash, InstallPath: component.InstallPath, Extract: component.Extract})
}
return protoComponents
}
// versionFromDockerImage returns the version tag from the image name, e.g. "v1.22.2" from "foocr.io/org/repo:v1.22.2@sha256:3009fj0...".