mirror of
https://github.com/edgelesssys/constellation.git
synced 2025-01-11 15:39:33 -05:00
joinservice: use configmap for k8s components
This commit is contained in:
parent
e6c4bb3406
commit
0c71cc77f6
@ -28,7 +28,7 @@ func (c *clusterFake) InitCluster(
|
||||
}
|
||||
|
||||
// JoinCluster will fake joining the current node to an existing cluster.
|
||||
func (c *clusterFake) JoinCluster(context.Context, *kubeadm.BootstrapTokenDiscovery, role.Role, string, *logger.Logger) error {
|
||||
func (c *clusterFake) JoinCluster(context.Context, *kubeadm.BootstrapTokenDiscovery, role.Role, string, versions.ComponentVersions, *logger.Logger) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
|
@ -150,7 +150,7 @@ func (s *Server) Init(ctx context.Context, req *initproto.InitRequest) (*initpro
|
||||
s.issuerWrapper.VMType() == vmtype.AzureCVM,
|
||||
req.HelmDeployments,
|
||||
req.ConformanceMode,
|
||||
versions.NewComponentVersionsFromProto(req.KubernetesComponents),
|
||||
versions.NewComponentVersionsFromInitProto(req.KubernetesComponents),
|
||||
s.log,
|
||||
)
|
||||
if err != nil {
|
||||
|
@ -25,6 +25,7 @@ import (
|
||||
"github.com/edgelesssys/constellation/v2/internal/logger"
|
||||
"github.com/edgelesssys/constellation/v2/internal/nodestate"
|
||||
"github.com/edgelesssys/constellation/v2/internal/role"
|
||||
"github.com/edgelesssys/constellation/v2/internal/versions"
|
||||
"github.com/edgelesssys/constellation/v2/joinservice/joinproto"
|
||||
"github.com/spf13/afero"
|
||||
"go.uber.org/zap"
|
||||
@ -286,7 +287,9 @@ func (c *JoinClient) startNodeAndJoin(ticket *joinproto.IssueJoinTicketResponse,
|
||||
Token: ticket.Token,
|
||||
CACertHashes: []string{ticket.DiscoveryTokenCaCertHash},
|
||||
}
|
||||
if err := c.joiner.JoinCluster(ctx, btd, c.role, ticket.KubernetesVersion, c.log); err != nil {
|
||||
k8sComponents := versions.NewComponentVersionsFromJoinProto(ticket.KubernetesComponents)
|
||||
|
||||
if err := c.joiner.JoinCluster(ctx, btd, c.role, ticket.KubernetesVersion, k8sComponents, c.log); err != nil {
|
||||
return fmt.Errorf("joining Kubernetes cluster: %w", err)
|
||||
}
|
||||
|
||||
@ -399,6 +402,7 @@ type ClusterJoiner interface {
|
||||
args *kubeadm.BootstrapTokenDiscovery,
|
||||
peerRole role.Role,
|
||||
k8sVersion string,
|
||||
k8sComponents versions.ComponentVersions,
|
||||
log *logger.Logger,
|
||||
) error
|
||||
}
|
||||
|
@ -23,6 +23,7 @@ import (
|
||||
"github.com/edgelesssys/constellation/v2/internal/grpc/testdialer"
|
||||
"github.com/edgelesssys/constellation/v2/internal/logger"
|
||||
"github.com/edgelesssys/constellation/v2/internal/role"
|
||||
"github.com/edgelesssys/constellation/v2/internal/versions"
|
||||
"github.com/edgelesssys/constellation/v2/joinservice/joinproto"
|
||||
"github.com/spf13/afero"
|
||||
"github.com/stretchr/testify/assert"
|
||||
@ -392,7 +393,7 @@ type stubClusterJoiner struct {
|
||||
joinClusterErr error
|
||||
}
|
||||
|
||||
func (j *stubClusterJoiner) JoinCluster(context.Context, *kubeadm.BootstrapTokenDiscovery, role.Role, string, *logger.Logger) error {
|
||||
func (j *stubClusterJoiner) JoinCluster(context.Context, *kubeadm.BootstrapTokenDiscovery, role.Role, string, versions.ComponentVersions, *logger.Logger) error {
|
||||
j.joinClusterCalled = true
|
||||
return j.joinClusterErr
|
||||
}
|
||||
|
@ -238,7 +238,7 @@ func (k *KubeWrapper) InitCluster(
|
||||
|
||||
// Store the received k8sVersion in a ConfigMap, overwriting existing values (there shouldn't be any).
|
||||
// Joining nodes determine the kubernetes version they will install based on this ConfigMap.
|
||||
if err := k.setupK8sVersionConfigMap(ctx, k8sVersion); err != nil {
|
||||
if err := k.setupK8sVersionConfigMap(ctx, k8sVersion, kubernetesComponents); err != nil {
|
||||
return nil, fmt.Errorf("failed to setup k8s version ConfigMap: %w", err)
|
||||
}
|
||||
|
||||
@ -248,14 +248,22 @@ func (k *KubeWrapper) InitCluster(
|
||||
}
|
||||
|
||||
// JoinCluster joins existing Kubernetes cluster.
|
||||
func (k *KubeWrapper) JoinCluster(ctx context.Context, args *kubeadm.BootstrapTokenDiscovery, peerRole role.Role, versionString string, log *logger.Logger) error {
|
||||
func (k *KubeWrapper) JoinCluster(ctx context.Context, args *kubeadm.BootstrapTokenDiscovery, peerRole role.Role, versionString string, k8sComponents versions.ComponentVersions, log *logger.Logger) error {
|
||||
k8sVersion, err := versions.NewValidK8sVersion(versionString)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
log.With(zap.String("version", string(k8sVersion))).Infof("Installing Kubernetes components")
|
||||
if err := k.clusterUtil.InstallComponents(ctx, k8sVersion); err != nil {
|
||||
return err
|
||||
|
||||
if len(k8sComponents) != 0 {
|
||||
log.With("k8sComponents", k8sComponents).Infof("Using provided kubernetes components")
|
||||
if err := k.clusterUtil.InstallComponentsFromCLI(ctx, k8sComponents); err != nil {
|
||||
return fmt.Errorf("installing kubernetes components: %w", err)
|
||||
}
|
||||
} else {
|
||||
log.With(zap.String("version", string(k8sVersion))).Infof("Installing Kubernetes components")
|
||||
if err := k.clusterUtil.InstallComponents(ctx, k8sVersion); err != nil {
|
||||
return fmt.Errorf("installing kubernetes components: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Step 1: retrieve cloud metadata for Kubernetes configuration
|
||||
@ -312,23 +320,50 @@ func (k *KubeWrapper) GetKubeconfig() ([]byte, error) {
|
||||
}
|
||||
|
||||
// setupK8sVersionConfigMap applies a ConfigMap (cf. server-side apply) to consistently store the installed k8s version.
|
||||
func (k *KubeWrapper) setupK8sVersionConfigMap(ctx context.Context, k8sVersion versions.ValidK8sVersion) error {
|
||||
func (k *KubeWrapper) setupK8sVersionConfigMap(ctx context.Context, k8sVersion versions.ValidK8sVersion, components versions.ComponentVersions) error {
|
||||
componentsMarshalled, err := json.Marshal(components)
|
||||
if err != nil {
|
||||
return fmt.Errorf("marshalling component versions: %w", err)
|
||||
}
|
||||
componentsHash := components.GetHash()
|
||||
componentConfigMapName := fmt.Sprintf("k8s-component-%s", strings.ReplaceAll(componentsHash, ":", "-"))
|
||||
|
||||
componentsConfig := corev1.ConfigMap{
|
||||
TypeMeta: metav1.TypeMeta{
|
||||
APIVersion: "v1",
|
||||
Kind: "ConfigMap",
|
||||
},
|
||||
Immutable: toPtr(true),
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: componentConfigMapName,
|
||||
Namespace: "kube-system",
|
||||
},
|
||||
Data: map[string]string{
|
||||
constants.K8sComponentsFieldName: string(componentsMarshalled),
|
||||
},
|
||||
}
|
||||
|
||||
if err := k.client.CreateConfigMap(ctx, componentsConfig); err != nil {
|
||||
return fmt.Errorf("apply in KubeWrapper.setupK8sVersionConfigMap(..) for components config map failed with: %w", err)
|
||||
}
|
||||
|
||||
config := corev1.ConfigMap{
|
||||
TypeMeta: metav1.TypeMeta{
|
||||
APIVersion: "v1",
|
||||
Kind: "ConfigMap",
|
||||
},
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: "k8s-version",
|
||||
Name: constants.K8sVersionConfigMapName,
|
||||
Namespace: "kube-system",
|
||||
},
|
||||
Data: map[string]string{
|
||||
constants.K8sVersion: string(k8sVersion),
|
||||
constants.K8sVersionFieldName: string(k8sVersion),
|
||||
constants.K8sComponentsFieldName: componentConfigMapName,
|
||||
},
|
||||
}
|
||||
|
||||
if err := k.client.CreateConfigMap(ctx, config); err != nil {
|
||||
return fmt.Errorf("apply in KubeWrapper.setupK8sVersionConfigMap(..) failed with: %w", err)
|
||||
return fmt.Errorf("apply in KubeWrapper.setupK8sVersionConfigMap(..) for version config map failed with: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
@ -519,3 +554,7 @@ type constellationServicesConfig struct {
|
||||
cloudServiceAccountURI string
|
||||
loadBalancerIP string
|
||||
}
|
||||
|
||||
func toPtr[T any](v T) *T {
|
||||
return &v
|
||||
}
|
||||
|
@ -266,14 +266,47 @@ func TestJoinCluster(t *testing.T) {
|
||||
privateIP := "192.0.2.1"
|
||||
k8sVersion := versions.Default
|
||||
|
||||
k8sComponents := versions.ComponentVersions{
|
||||
{
|
||||
URL: "URL",
|
||||
Hash: "Hash",
|
||||
InstallPath: "InstallPath",
|
||||
Extract: true,
|
||||
},
|
||||
}
|
||||
|
||||
testCases := map[string]struct {
|
||||
clusterUtil stubClusterUtil
|
||||
providerMetadata ProviderMetadata
|
||||
wantConfig kubeadm.JoinConfiguration
|
||||
role role.Role
|
||||
wantErr bool
|
||||
clusterUtil stubClusterUtil
|
||||
providerMetadata ProviderMetadata
|
||||
wantConfig kubeadm.JoinConfiguration
|
||||
role role.Role
|
||||
k8sComponents versions.ComponentVersions
|
||||
wantComponentsFromCLI bool
|
||||
wantErr bool
|
||||
}{
|
||||
"kubeadm join worker works with metadata": {
|
||||
"kubeadm join worker works with metadata and remote Kubernetes Components": {
|
||||
clusterUtil: stubClusterUtil{},
|
||||
providerMetadata: &stubProviderMetadata{
|
||||
selfResp: metadata.InstanceMetadata{
|
||||
ProviderID: "provider-id",
|
||||
Name: "metadata-name",
|
||||
VPCIP: "192.0.2.1",
|
||||
},
|
||||
},
|
||||
k8sComponents: k8sComponents,
|
||||
role: role.Worker,
|
||||
wantComponentsFromCLI: true,
|
||||
wantConfig: kubeadm.JoinConfiguration{
|
||||
Discovery: kubeadm.Discovery{
|
||||
BootstrapToken: joinCommand,
|
||||
},
|
||||
NodeRegistration: kubeadm.NodeRegistrationOptions{
|
||||
Name: "metadata-name",
|
||||
KubeletExtraArgs: map[string]string{"node-ip": "192.0.2.1"},
|
||||
},
|
||||
},
|
||||
},
|
||||
"kubeadm join worker works with metadata and local Kubernetes components": {
|
||||
clusterUtil: stubClusterUtil{},
|
||||
providerMetadata: &stubProviderMetadata{
|
||||
selfResp: metadata.InstanceMetadata{
|
||||
@ -340,6 +373,33 @@ func TestJoinCluster(t *testing.T) {
|
||||
SkipPhases: []string{"control-plane-prepare/download-certs"},
|
||||
},
|
||||
},
|
||||
"kubeadm join worker fails when installing remote Kubernetes components": {
|
||||
clusterUtil: stubClusterUtil{installComponentsFromCLIErr: errors.New("error")},
|
||||
providerMetadata: &stubProviderMetadata{
|
||||
selfResp: metadata.InstanceMetadata{
|
||||
ProviderID: "provider-id",
|
||||
Name: "metadata-name",
|
||||
VPCIP: "192.0.2.1",
|
||||
},
|
||||
},
|
||||
k8sComponents: k8sComponents,
|
||||
role: role.Worker,
|
||||
wantComponentsFromCLI: true,
|
||||
wantErr: true,
|
||||
},
|
||||
"kubeadm join worker fails when installing local Kubernetes components": {
|
||||
clusterUtil: stubClusterUtil{installComponentsErr: errors.New("error")},
|
||||
providerMetadata: &stubProviderMetadata{
|
||||
selfResp: metadata.InstanceMetadata{
|
||||
ProviderID: "provider-id",
|
||||
Name: "metadata-name",
|
||||
VPCIP: "192.0.2.1",
|
||||
},
|
||||
},
|
||||
role: role.Worker,
|
||||
wantComponentsFromCLI: true,
|
||||
wantErr: true,
|
||||
},
|
||||
"kubeadm join worker fails when retrieving self metadata": {
|
||||
clusterUtil: stubClusterUtil{},
|
||||
providerMetadata: &stubProviderMetadata{
|
||||
@ -368,7 +428,7 @@ func TestJoinCluster(t *testing.T) {
|
||||
getIPAddr: func() (string, error) { return privateIP, nil },
|
||||
}
|
||||
|
||||
err := kube.JoinCluster(context.Background(), joinCommand, tc.role, string(k8sVersion), logger.NewTest(t))
|
||||
err := kube.JoinCluster(context.Background(), joinCommand, tc.role, string(k8sVersion), tc.k8sComponents, logger.NewTest(t))
|
||||
if tc.wantErr {
|
||||
assert.Error(err)
|
||||
return
|
||||
@ -379,6 +439,8 @@ func TestJoinCluster(t *testing.T) {
|
||||
require.NoError(kubernetes.UnmarshalK8SResources(tc.clusterUtil.joinConfigs[0], &joinYaml))
|
||||
|
||||
assert.Equal(tc.wantConfig, joinYaml.JoinConfiguration)
|
||||
assert.Equal(tc.wantComponentsFromCLI, tc.clusterUtil.calledInstallComponentsFromCLI)
|
||||
assert.Equal(!tc.wantComponentsFromCLI, tc.clusterUtil.calledInstallComponents)
|
||||
})
|
||||
}
|
||||
}
|
||||
@ -428,6 +490,9 @@ type stubClusterUtil struct {
|
||||
joinClusterErr error
|
||||
startKubeletErr error
|
||||
|
||||
calledInstallComponents bool
|
||||
calledInstallComponentsFromCLI bool
|
||||
|
||||
initConfigs [][]byte
|
||||
joinConfigs [][]byte
|
||||
}
|
||||
@ -437,10 +502,12 @@ func (s *stubClusterUtil) SetupKonnectivity(kubectl k8sapi.Client, konnectivityA
|
||||
}
|
||||
|
||||
func (s *stubClusterUtil) InstallComponents(ctx context.Context, version versions.ValidK8sVersion) error {
|
||||
s.calledInstallComponents = true
|
||||
return s.installComponentsErr
|
||||
}
|
||||
|
||||
func (s *stubClusterUtil) InstallComponentsFromCLI(ctx context.Context, kubernetesComponents versions.ComponentVersions) error {
|
||||
s.calledInstallComponentsFromCLI = true
|
||||
return s.installComponentsFromCLIErr
|
||||
}
|
||||
|
||||
|
@ -133,7 +133,7 @@ func initialize(cmd *cobra.Command, newDialer func(validator *cloudcmd.Validator
|
||||
UseExistingKek: false,
|
||||
CloudServiceAccountUri: serviceAccURI,
|
||||
KubernetesVersion: conf.KubernetesVersion,
|
||||
KubernetesComponents: versions.VersionConfigs[k8sVersion].KubernetesComponents.ToProto(),
|
||||
KubernetesComponents: versions.VersionConfigs[k8sVersion].KubernetesComponents.ToInitProto(),
|
||||
HelmDeployments: helmDeployments,
|
||||
EnforcedPcrs: conf.EnforcedPCRs(),
|
||||
EnforceIdkeydigest: conf.EnforcesIDKeyDigest(),
|
||||
|
@ -22,3 +22,11 @@ rules:
|
||||
verbs:
|
||||
- create
|
||||
- update
|
||||
- apiGroups:
|
||||
- ""
|
||||
resources:
|
||||
- configmaps
|
||||
verbs:
|
||||
- get
|
||||
- create
|
||||
- update
|
||||
|
@ -358,7 +358,7 @@ func (i *ChartLoader) loadConstellationServicesHelper(config *config.Config, mas
|
||||
"kmsPort": constants.KMSPort,
|
||||
"serviceBasePath": constants.ServiceBasePath,
|
||||
"joinConfigCMName": constants.JoinConfigMap,
|
||||
"k8sVersionCMName": constants.K8sVersion,
|
||||
"k8sVersionCMName": constants.K8sVersionConfigMapName,
|
||||
"internalCMName": constants.InternalConfigMap,
|
||||
},
|
||||
"kms": map[string]any{
|
||||
|
@ -22,3 +22,11 @@ rules:
|
||||
verbs:
|
||||
- create
|
||||
- update
|
||||
- apiGroups:
|
||||
- ""
|
||||
resources:
|
||||
- configmaps
|
||||
verbs:
|
||||
- get
|
||||
- create
|
||||
- update
|
||||
|
@ -22,3 +22,11 @@ rules:
|
||||
verbs:
|
||||
- create
|
||||
- update
|
||||
- apiGroups:
|
||||
- ""
|
||||
resources:
|
||||
- configmaps
|
||||
verbs:
|
||||
- get
|
||||
- create
|
||||
- update
|
||||
|
@ -22,3 +22,11 @@ rules:
|
||||
verbs:
|
||||
- create
|
||||
- update
|
||||
- apiGroups:
|
||||
- ""
|
||||
resources:
|
||||
- configmaps
|
||||
verbs:
|
||||
- get
|
||||
- create
|
||||
- update
|
||||
|
@ -107,8 +107,18 @@ const (
|
||||
EnforceIDKeyDigestFilename = "enforceIdKeyDigest"
|
||||
// AzureCVM is the name of the file indicating whether the cluster is expected to run on CVMs or not.
|
||||
AzureCVM = "azureCVM"
|
||||
// K8sVersion is the filename of the mapped "k8s-version" configMap file.
|
||||
K8sVersion = "k8s-version"
|
||||
|
||||
// K8sVersionConfigMapName is the filename of the mapped "k8s-version" configMap file.
|
||||
K8sVersionConfigMapName = "k8s-version"
|
||||
|
||||
// K8sVersionFieldName is the key in the "k8s-version" configMap which references the string with the K8s version.
|
||||
K8sVersionFieldName = "k8s-version"
|
||||
|
||||
// K8sComponentsFieldName is the name of the of the key holding the configMap name that holds the components configuration.
|
||||
K8sComponentsFieldName = "components"
|
||||
|
||||
// ComponentsListKey is the name of the key holding the list of components in the components configMap.
|
||||
ComponentsListKey = "components"
|
||||
|
||||
//
|
||||
// CLI.
|
||||
|
@ -7,11 +7,13 @@ SPDX-License-Identifier: AGPL-3.0-only
|
||||
package versions
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"github.com/edgelesssys/constellation/v2/bootstrapper/initproto"
|
||||
"github.com/edgelesssys/constellation/v2/internal/constants"
|
||||
"github.com/edgelesssys/constellation/v2/joinservice/joinproto"
|
||||
)
|
||||
|
||||
// ValidK8sVersion represents any of the three currently supported k8s versions.
|
||||
@ -261,8 +263,8 @@ type ComponentVersion struct {
|
||||
// ComponentVersions is a list of ComponentVersion.
|
||||
type ComponentVersions []ComponentVersion
|
||||
|
||||
// NewComponentVersionsFromProto converts a protobuf KubernetesVersion to ComponentVersions.
|
||||
func NewComponentVersionsFromProto(protoComponents []*initproto.KubernetesComponent) ComponentVersions {
|
||||
// NewComponentVersionsFromInitProto converts a protobuf KubernetesVersion to ComponentVersions.
|
||||
func NewComponentVersionsFromInitProto(protoComponents []*initproto.KubernetesComponent) ComponentVersions {
|
||||
components := ComponentVersions{}
|
||||
for _, protoComponent := range protoComponents {
|
||||
if protoComponent == nil {
|
||||
@ -273,8 +275,20 @@ func NewComponentVersionsFromProto(protoComponents []*initproto.KubernetesCompon
|
||||
return components
|
||||
}
|
||||
|
||||
// ToProto converts a ComponentVersions to a protobuf KubernetesVersion.
|
||||
func (c ComponentVersions) ToProto() []*initproto.KubernetesComponent {
|
||||
// NewComponentVersionsFromJoinProto converts a protobuf KubernetesVersion to ComponentVersions.
|
||||
func NewComponentVersionsFromJoinProto(protoComponents []*joinproto.KubernetesComponent) ComponentVersions {
|
||||
components := ComponentVersions{}
|
||||
for _, protoComponent := range protoComponents {
|
||||
if protoComponent == nil {
|
||||
continue
|
||||
}
|
||||
components = append(components, ComponentVersion{URL: protoComponent.Url, Hash: protoComponent.Hash, InstallPath: protoComponent.InstallPath, Extract: protoComponent.Extract})
|
||||
}
|
||||
return components
|
||||
}
|
||||
|
||||
// ToInitProto converts a ComponentVersions to a protobuf KubernetesVersion.
|
||||
func (c ComponentVersions) ToInitProto() []*initproto.KubernetesComponent {
|
||||
protoComponents := []*initproto.KubernetesComponent{}
|
||||
for _, component := range c {
|
||||
protoComponents = append(protoComponents, &initproto.KubernetesComponent{Url: component.URL, Hash: component.Hash, InstallPath: component.InstallPath, Extract: component.Extract})
|
||||
@ -282,6 +296,25 @@ func (c ComponentVersions) ToProto() []*initproto.KubernetesComponent {
|
||||
return protoComponents
|
||||
}
|
||||
|
||||
// ToJoinProto converts a ComponentVersions to a protobuf KubernetesVersion.
|
||||
func (c ComponentVersions) ToJoinProto() []*joinproto.KubernetesComponent {
|
||||
protoComponents := []*joinproto.KubernetesComponent{}
|
||||
for _, component := range c {
|
||||
protoComponents = append(protoComponents, &joinproto.KubernetesComponent{Url: component.URL, Hash: component.Hash, InstallPath: component.InstallPath, Extract: component.Extract})
|
||||
}
|
||||
return protoComponents
|
||||
}
|
||||
|
||||
// GetHash returns the hash over all component hashes.
|
||||
func (c ComponentVersions) GetHash() string {
|
||||
sha := sha256.New()
|
||||
for _, component := range c {
|
||||
sha.Write([]byte(component.Hash))
|
||||
}
|
||||
|
||||
return fmt.Sprintf("sha256:%x", sha.Sum(nil))
|
||||
}
|
||||
|
||||
// versionFromDockerImage returns the version tag from the image name, e.g. "v1.22.2" from "foocr.io/org/repo:v1.22.2@sha256:3009fj0...".
|
||||
func versionFromDockerImage(imageName string) string {
|
||||
beforeAt, _, _ := strings.Cut(imageName, "@")
|
||||
|
@ -85,7 +85,7 @@ func main() {
|
||||
log.With(zap.Error(err)).Fatalf("Failed to read measurement salt")
|
||||
}
|
||||
|
||||
server := server.New(
|
||||
server, err := server.New(
|
||||
measurementSalt,
|
||||
handler,
|
||||
kubernetesca.New(log.Named("certificateAuthority"), handler),
|
||||
@ -93,6 +93,9 @@ func main() {
|
||||
kms,
|
||||
log.Named("server"),
|
||||
)
|
||||
if err != nil {
|
||||
log.With(zap.Error(err)).Fatalf("Failed to create server")
|
||||
}
|
||||
|
||||
watcher, err := watcher.New(log.Named("fileWatcher"), validator)
|
||||
if err != nil {
|
||||
|
85
joinservice/internal/kubernetes/kubernetes.go
Normal file
85
joinservice/internal/kubernetes/kubernetes.go
Normal file
@ -0,0 +1,85 @@
|
||||
/*
|
||||
Copyright (c) Edgeless Systems GmbH
|
||||
|
||||
SPDX-License-Identifier: AGPL-3.0-only
|
||||
*/
|
||||
|
||||
package kubernetes
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
|
||||
"github.com/edgelesssys/constellation/v2/internal/constants"
|
||||
"github.com/edgelesssys/constellation/v2/internal/versions"
|
||||
corev1 "k8s.io/api/core/v1"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/client-go/kubernetes"
|
||||
"k8s.io/client-go/rest"
|
||||
)
|
||||
|
||||
// Client is a kubernetes client.
|
||||
type Client struct {
|
||||
client *kubernetes.Clientset
|
||||
}
|
||||
|
||||
// New creates a new kubernetes client.
|
||||
func New() (*Client, error) {
|
||||
// creates the in-cluster config
|
||||
config, err := rest.InClusterConfig()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to create in-cluster config: %w", err)
|
||||
}
|
||||
// creates the clientset
|
||||
clientset, err := kubernetes.NewForConfig(config)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to create clientset: %w", err)
|
||||
}
|
||||
return &Client{client: clientset}, nil
|
||||
}
|
||||
|
||||
// GetComponents returns the components of the cluster.
|
||||
func (c *Client) GetComponents(ctx context.Context, configMapName string) (versions.ComponentVersions, error) {
|
||||
componentsRaw, err := c.getConfigMapData(ctx, configMapName, constants.ComponentsListKey)
|
||||
if err != nil {
|
||||
return versions.ComponentVersions{}, fmt.Errorf("failed to get components: %w", err)
|
||||
}
|
||||
var components versions.ComponentVersions
|
||||
if err := json.Unmarshal([]byte(componentsRaw), &components); err != nil {
|
||||
return versions.ComponentVersions{}, fmt.Errorf("failed to unmarshal components %s: %w", componentsRaw, err)
|
||||
}
|
||||
return components, nil
|
||||
}
|
||||
|
||||
func (c *Client) getConfigMapData(ctx context.Context, name, key string) (string, error) {
|
||||
cm, err := c.client.CoreV1().ConfigMaps("kube-system").Get(ctx, name, metav1.GetOptions{})
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to get configmap: %w", err)
|
||||
}
|
||||
|
||||
return cm.Data[key], nil
|
||||
}
|
||||
|
||||
// CreateConfigMap creates the provided configmap.
|
||||
func (c *Client) CreateConfigMap(ctx context.Context, configMap corev1.ConfigMap) error {
|
||||
_, err := c.client.CoreV1().ConfigMaps(configMap.ObjectMeta.Namespace).Create(ctx, &configMap, metav1.CreateOptions{})
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create configmap: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// AddReferenceToK8sVersionConfigMap adds a reference to the provided configmap to the k8s version configmap.
|
||||
func (c *Client) AddReferenceToK8sVersionConfigMap(ctx context.Context, k8sVersionsConfigMapName string, componentsConfigMapName string) error {
|
||||
cm, err := c.client.CoreV1().ConfigMaps("kube-system").Get(ctx, k8sVersionsConfigMapName, metav1.GetOptions{})
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to get configmap: %w", err)
|
||||
}
|
||||
cm.Data[constants.K8sComponentsFieldName] = componentsConfigMapName
|
||||
_, err = c.client.CoreV1().ConfigMaps("kube-system").Update(ctx, cm, metav1.UpdateOptions{})
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to update configmap: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
@ -8,23 +8,32 @@ package server
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"net"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/Azure/azure-sdk-for-go/sdk/azcore/to"
|
||||
"github.com/edgelesssys/constellation/v2/internal/attestation"
|
||||
"github.com/edgelesssys/constellation/v2/internal/constants"
|
||||
"github.com/edgelesssys/constellation/v2/internal/crypto"
|
||||
"github.com/edgelesssys/constellation/v2/internal/file"
|
||||
"github.com/edgelesssys/constellation/v2/internal/grpc/grpclog"
|
||||
"github.com/edgelesssys/constellation/v2/internal/logger"
|
||||
"github.com/edgelesssys/constellation/v2/internal/versions"
|
||||
"github.com/edgelesssys/constellation/v2/joinservice/internal/kubernetes"
|
||||
"github.com/edgelesssys/constellation/v2/joinservice/joinproto"
|
||||
"go.uber.org/zap"
|
||||
"google.golang.org/grpc"
|
||||
"google.golang.org/grpc/codes"
|
||||
"google.golang.org/grpc/credentials"
|
||||
"google.golang.org/grpc/status"
|
||||
corev1 "k8s.io/api/core/v1"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
|
||||
kubeadmv1 "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta3"
|
||||
)
|
||||
@ -38,6 +47,7 @@ type Server struct {
|
||||
joinTokenGetter joinTokenGetter
|
||||
dataKeyGetter dataKeyGetter
|
||||
ca certificateAuthority
|
||||
kubeClient kubeClient
|
||||
joinproto.UnimplementedAPIServer
|
||||
}
|
||||
|
||||
@ -45,7 +55,11 @@ type Server struct {
|
||||
func New(
|
||||
measurementSalt []byte, fileHandler file.Handler, ca certificateAuthority,
|
||||
joinTokenGetter joinTokenGetter, dataKeyGetter dataKeyGetter, log *logger.Logger,
|
||||
) *Server {
|
||||
) (*Server, error) {
|
||||
kubeClient, err := kubernetes.New()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to create kubernetes client: %w", err)
|
||||
}
|
||||
return &Server{
|
||||
measurementSalt: measurementSalt,
|
||||
log: log,
|
||||
@ -53,7 +67,8 @@ func New(
|
||||
joinTokenGetter: joinTokenGetter,
|
||||
dataKeyGetter: dataKeyGetter,
|
||||
ca: ca,
|
||||
}
|
||||
kubeClient: kubeClient,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Run starts the gRPC server on the given port, using the provided tlsConfig.
|
||||
@ -106,12 +121,35 @@ func (s *Server) IssueJoinTicket(ctx context.Context, req *joinproto.IssueJoinTi
|
||||
return nil, status.Errorf(codes.Internal, "unable to generate Kubernetes join arguments: %s", err)
|
||||
}
|
||||
|
||||
log.Infof("Querying K8sVersion ConfigMap")
|
||||
log.Infof("Querying K8sVersion ConfigMap for Kubernetes version")
|
||||
k8sVersion, err := s.getK8sVersion()
|
||||
if err != nil {
|
||||
return nil, status.Errorf(codes.Internal, "unable to get k8s version: %s", err)
|
||||
}
|
||||
|
||||
log.Infof("Querying K8sVersion ConfigMap for components ConfigMap name")
|
||||
componentsConfigMapName, err := s.getK8sComponentsConfigMapName()
|
||||
if errors.Is(err, fs.ErrNotExist) {
|
||||
// If the file does not exist, the Constellation was initialized with a version before 2.3.0
|
||||
// As a migration step, the join service will create the ConfigMap with the K8s components which
|
||||
// match the K8s minor version of the cluster.
|
||||
log.Warnf("Reference to K8sVersion ConfigMap does not exist, creating fallback Components ConfigMap and referencing it in K8sVersion ConfigMap")
|
||||
log.Warnf("This is expected if the Constellation was initialized with a CLI before version 2.3.0")
|
||||
log.Warnf("DEPRECATION WARNING: This is a migration step and will be removed in a future release")
|
||||
componentsConfigMapName, err = s.createFallbackComponentsConfigMap(ctx, k8sVersion)
|
||||
if err != nil {
|
||||
return nil, status.Errorf(codes.Internal, "unable to create fallback k8s components configmap: %s", err)
|
||||
}
|
||||
} else if err != nil {
|
||||
return nil, status.Errorf(codes.Internal, "unable to get components ConfigMap name: %s", err)
|
||||
}
|
||||
|
||||
log.Infof("Querying %s ConfigMap for components", componentsConfigMapName)
|
||||
components, err := s.kubeClient.GetComponents(ctx, componentsConfigMapName)
|
||||
if err != nil {
|
||||
return nil, status.Errorf(codes.Internal, "unable to get components: %s", err)
|
||||
}
|
||||
|
||||
log.Infof("Creating signed kubelet certificate")
|
||||
kubeletCert, err := s.ca.GetCertificate(req.CertificateRequest)
|
||||
if err != nil {
|
||||
@ -146,6 +184,7 @@ func (s *Server) IssueJoinTicket(ctx context.Context, req *joinproto.IssueJoinTi
|
||||
KubeletCert: kubeletCert,
|
||||
ControlPlaneFiles: controlPlaneFiles,
|
||||
KubernetesVersion: k8sVersion,
|
||||
KubernetesComponents: components.ToJoinProto(),
|
||||
}, nil
|
||||
}
|
||||
|
||||
@ -176,15 +215,69 @@ func (s *Server) IssueRejoinTicket(ctx context.Context, req *joinproto.IssueRejo
|
||||
|
||||
// getK8sVersion reads the k8s version from a VolumeMount that is backed by the k8s-version ConfigMap.
|
||||
func (s *Server) getK8sVersion() (string, error) {
|
||||
fileContent, err := s.file.Read(filepath.Join(constants.ServiceBasePath, constants.K8sVersion))
|
||||
fileContent, err := s.file.Read(filepath.Join(constants.ServiceBasePath, constants.K8sVersionConfigMapName))
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("could not read k8s version file: %v", err)
|
||||
return "", fmt.Errorf("could not read k8s version file: %w", err)
|
||||
}
|
||||
k8sVersion := string(fileContent)
|
||||
|
||||
return k8sVersion, nil
|
||||
}
|
||||
|
||||
// getK8sComponentsConfigMapName reads the k8s components config map name from a VolumeMount that is backed by the k8s-version ConfigMap.
|
||||
func (s *Server) getK8sComponentsConfigMapName() (string, error) {
|
||||
fileContent, err := s.file.Read(filepath.Join(constants.ServiceBasePath, constants.K8sComponentsFieldName))
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("could not read k8s version file: %w", err)
|
||||
}
|
||||
componentsConfigMapName := string(fileContent)
|
||||
|
||||
return componentsConfigMapName, nil
|
||||
}
|
||||
|
||||
// This function mimics the creation of the components ConfigMap which is now done in the bootstrapper
|
||||
// during the first initialization of the Constellation .
|
||||
// For more information see setupK8sVersionConfigMap() in bootstrapper/internal/kubernetes/kubernetes.go.
|
||||
// This is a migration step and will be removed in a future release.
|
||||
func (s *Server) createFallbackComponentsConfigMap(ctx context.Context, k8sVersion string) (string, error) {
|
||||
validK8sVersion, err := versions.NewValidK8sVersion(k8sVersion)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("could not create fallback components config map: %w", err)
|
||||
}
|
||||
components := versions.VersionConfigs[validK8sVersion].KubernetesComponents
|
||||
componentsMarshalled, err := json.Marshal(components)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("marshalling component versions: %w", err)
|
||||
}
|
||||
componentsHash := components.GetHash()
|
||||
componentConfigMapName := fmt.Sprintf("k8s-component-%s", strings.ReplaceAll(componentsHash, ":", "-"))
|
||||
|
||||
componentsConfig := corev1.ConfigMap{
|
||||
TypeMeta: metav1.TypeMeta{
|
||||
APIVersion: "v1",
|
||||
Kind: "ConfigMap",
|
||||
},
|
||||
Immutable: to.Ptr(true),
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: componentConfigMapName,
|
||||
Namespace: "kube-system",
|
||||
},
|
||||
Data: map[string]string{
|
||||
constants.K8sComponentsFieldName: string(componentsMarshalled),
|
||||
},
|
||||
}
|
||||
|
||||
if err := s.kubeClient.CreateConfigMap(ctx, componentsConfig); err != nil {
|
||||
return "", fmt.Errorf("creating fallback components config map: %w", err)
|
||||
}
|
||||
|
||||
if err := s.kubeClient.AddReferenceToK8sVersionConfigMap(ctx, "k8s-version", componentConfigMapName); err != nil {
|
||||
return "", fmt.Errorf("adding reference to fallback components config map: %w", err)
|
||||
}
|
||||
|
||||
return componentConfigMapName, nil
|
||||
}
|
||||
|
||||
// joinTokenGetter returns Kubernetes bootstrap (join) tokens.
|
||||
type joinTokenGetter interface {
|
||||
// GetJoinToken returns a bootstrap (join) token.
|
||||
@ -202,3 +295,9 @@ type certificateAuthority interface {
|
||||
// GetCertificate returns a certificate and private key, signed by the issuer.
|
||||
GetCertificate(certificateRequest []byte) (kubeletCert []byte, err error)
|
||||
}
|
||||
|
||||
type kubeClient interface {
|
||||
GetComponents(ctx context.Context, configMapName string) (versions.ComponentVersions, error)
|
||||
CreateConfigMap(ctx context.Context, configMap corev1.ConfigMap) error
|
||||
AddReferenceToK8sVersionConfigMap(ctx context.Context, k8sVersionsConfigMapName string, componentsConfigMapName string) error
|
||||
}
|
||||
|
@ -23,6 +23,7 @@ import (
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"go.uber.org/goleak"
|
||||
corev1 "k8s.io/api/core/v1"
|
||||
kubeadmv1 "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta3"
|
||||
)
|
||||
|
||||
@ -44,12 +45,23 @@ func TestIssueJoinTicket(t *testing.T) {
|
||||
}
|
||||
testK8sVersion := versions.Default
|
||||
|
||||
components := versions.ComponentVersions{
|
||||
{
|
||||
URL: "URL",
|
||||
Hash: "hash",
|
||||
InstallPath: "install-path",
|
||||
Extract: true,
|
||||
},
|
||||
}
|
||||
|
||||
testCases := map[string]struct {
|
||||
isControlPlane bool
|
||||
kubeadm stubTokenGetter
|
||||
kms stubKeyGetter
|
||||
ca stubCA
|
||||
wantErr bool
|
||||
isControlPlane bool
|
||||
kubeadm stubTokenGetter
|
||||
kms stubKeyGetter
|
||||
ca stubCA
|
||||
kubeClient stubKubeClient
|
||||
missingComponentsReferenceFile bool
|
||||
wantErr bool
|
||||
}{
|
||||
"worker node": {
|
||||
kubeadm: stubTokenGetter{token: testJoinToken},
|
||||
@ -57,13 +69,46 @@ func TestIssueJoinTicket(t *testing.T) {
|
||||
uuid: testKey,
|
||||
attestation.MeasurementSecretContext: measurementSecret,
|
||||
}},
|
||||
ca: stubCA{cert: testCert},
|
||||
ca: stubCA{cert: testCert},
|
||||
kubeClient: stubKubeClient{getComponentsVal: components},
|
||||
},
|
||||
"worker node components reference missing": {
|
||||
kubeadm: stubTokenGetter{token: testJoinToken},
|
||||
kms: stubKeyGetter{dataKeys: map[string][]byte{
|
||||
uuid: testKey,
|
||||
attestation.MeasurementSecretContext: measurementSecret,
|
||||
}},
|
||||
ca: stubCA{cert: testCert},
|
||||
kubeClient: stubKubeClient{getComponentsVal: components},
|
||||
missingComponentsReferenceFile: true,
|
||||
},
|
||||
"worker node components reference missing and fallback fails": {
|
||||
kubeadm: stubTokenGetter{token: testJoinToken},
|
||||
kms: stubKeyGetter{dataKeys: map[string][]byte{
|
||||
uuid: testKey,
|
||||
attestation.MeasurementSecretContext: measurementSecret,
|
||||
}},
|
||||
ca: stubCA{cert: testCert},
|
||||
kubeClient: stubKubeClient{createConfigMapErr: someErr},
|
||||
missingComponentsReferenceFile: true,
|
||||
wantErr: true,
|
||||
},
|
||||
"kubeclient fails": {
|
||||
kubeadm: stubTokenGetter{token: testJoinToken},
|
||||
kms: stubKeyGetter{dataKeys: map[string][]byte{
|
||||
uuid: testKey,
|
||||
attestation.MeasurementSecretContext: measurementSecret,
|
||||
}},
|
||||
ca: stubCA{cert: testCert},
|
||||
kubeClient: stubKubeClient{getComponentsErr: someErr},
|
||||
wantErr: true,
|
||||
},
|
||||
"GetDataKey fails": {
|
||||
kubeadm: stubTokenGetter{token: testJoinToken},
|
||||
kms: stubKeyGetter{dataKeys: make(map[string][]byte), getDataKeyErr: someErr},
|
||||
ca: stubCA{cert: testCert},
|
||||
wantErr: true,
|
||||
kubeadm: stubTokenGetter{token: testJoinToken},
|
||||
kms: stubKeyGetter{dataKeys: make(map[string][]byte), getDataKeyErr: someErr},
|
||||
ca: stubCA{cert: testCert},
|
||||
kubeClient: stubKubeClient{getComponentsVal: components},
|
||||
wantErr: true,
|
||||
},
|
||||
"GetJoinToken fails": {
|
||||
kubeadm: stubTokenGetter{getJoinTokenErr: someErr},
|
||||
@ -71,8 +116,9 @@ func TestIssueJoinTicket(t *testing.T) {
|
||||
uuid: testKey,
|
||||
attestation.MeasurementSecretContext: measurementSecret,
|
||||
}},
|
||||
ca: stubCA{cert: testCert},
|
||||
wantErr: true,
|
||||
ca: stubCA{cert: testCert},
|
||||
kubeClient: stubKubeClient{getComponentsVal: components},
|
||||
wantErr: true,
|
||||
},
|
||||
"GetCertificate fails": {
|
||||
kubeadm: stubTokenGetter{token: testJoinToken},
|
||||
@ -80,8 +126,9 @@ func TestIssueJoinTicket(t *testing.T) {
|
||||
uuid: testKey,
|
||||
attestation.MeasurementSecretContext: measurementSecret,
|
||||
}},
|
||||
ca: stubCA{getCertErr: someErr},
|
||||
wantErr: true,
|
||||
ca: stubCA{getCertErr: someErr},
|
||||
kubeClient: stubKubeClient{getComponentsVal: components},
|
||||
wantErr: true,
|
||||
},
|
||||
"control plane": {
|
||||
isControlPlane: true,
|
||||
@ -93,7 +140,8 @@ func TestIssueJoinTicket(t *testing.T) {
|
||||
uuid: testKey,
|
||||
attestation.MeasurementSecretContext: measurementSecret,
|
||||
}},
|
||||
ca: stubCA{cert: testCert},
|
||||
ca: stubCA{cert: testCert},
|
||||
kubeClient: stubKubeClient{getComponentsVal: components},
|
||||
},
|
||||
"GetControlPlaneCertificateKey fails": {
|
||||
isControlPlane: true,
|
||||
@ -102,8 +150,9 @@ func TestIssueJoinTicket(t *testing.T) {
|
||||
uuid: testKey,
|
||||
attestation.MeasurementSecretContext: measurementSecret,
|
||||
}},
|
||||
ca: stubCA{cert: testCert},
|
||||
wantErr: true,
|
||||
ca: stubCA{cert: testCert},
|
||||
kubeClient: stubKubeClient{getComponentsVal: components},
|
||||
wantErr: true,
|
||||
},
|
||||
}
|
||||
|
||||
@ -114,17 +163,23 @@ func TestIssueJoinTicket(t *testing.T) {
|
||||
|
||||
handler := file.NewHandler(afero.NewMemMapFs())
|
||||
// IssueJoinTicket tries to read the k8s-version ConfigMap from a mounted file.
|
||||
require.NoError(handler.Write(filepath.Join(constants.ServiceBasePath, constants.K8sVersion), []byte(testK8sVersion), file.OptNone))
|
||||
require.NoError(handler.Write(filepath.Join(constants.ServiceBasePath, constants.K8sVersionConfigMapName), []byte(testK8sVersion), file.OptNone))
|
||||
|
||||
if !tc.missingComponentsReferenceFile {
|
||||
require.NoError(handler.Write(filepath.Join(constants.ServiceBasePath, constants.K8sComponentsFieldName), []byte(testK8sVersion), file.OptNone))
|
||||
}
|
||||
|
||||
salt := []byte{0xA, 0xB, 0xC}
|
||||
|
||||
api := New(
|
||||
salt,
|
||||
handler,
|
||||
tc.ca,
|
||||
tc.kubeadm,
|
||||
tc.kms,
|
||||
logger.NewTest(t),
|
||||
)
|
||||
api := Server{
|
||||
measurementSalt: salt,
|
||||
file: handler,
|
||||
ca: tc.ca,
|
||||
joinTokenGetter: tc.kubeadm,
|
||||
dataKeyGetter: tc.kms,
|
||||
kubeClient: tc.kubeClient,
|
||||
log: logger.NewTest(t),
|
||||
}
|
||||
|
||||
req := &joinproto.IssueJoinTicketRequest{
|
||||
DiskUuid: "uuid",
|
||||
@ -144,6 +199,7 @@ func TestIssueJoinTicket(t *testing.T) {
|
||||
assert.Equal(tc.kubeadm.token.CACertHashes[0], resp.DiscoveryTokenCaCertHash)
|
||||
assert.Equal(tc.kubeadm.token.Token, resp.Token)
|
||||
assert.Equal(tc.ca.cert, resp.KubeletCert)
|
||||
assert.Equal(tc.kubeClient.getComponentsVal.ToJoinProto(), resp.KubernetesComponents)
|
||||
|
||||
if tc.isControlPlane {
|
||||
assert.Len(resp.ControlPlaneFiles, len(tc.kubeadm.files))
|
||||
@ -181,14 +237,13 @@ func TestIssueRejoinTicker(t *testing.T) {
|
||||
assert := assert.New(t)
|
||||
require := require.New(t)
|
||||
|
||||
api := New(
|
||||
nil,
|
||||
file.Handler{},
|
||||
stubCA{},
|
||||
stubTokenGetter{},
|
||||
tc.keyGetter,
|
||||
logger.NewTest(t),
|
||||
)
|
||||
api := Server{
|
||||
file: file.Handler{},
|
||||
ca: stubCA{},
|
||||
joinTokenGetter: stubTokenGetter{},
|
||||
dataKeyGetter: tc.keyGetter,
|
||||
log: logger.NewTest(t),
|
||||
}
|
||||
|
||||
req := &joinproto.IssueRejoinTicketRequest{
|
||||
DiskUuid: uuid,
|
||||
@ -238,3 +293,24 @@ type stubCA struct {
|
||||
func (f stubCA) GetCertificate(csr []byte) ([]byte, error) {
|
||||
return f.cert, f.getCertErr
|
||||
}
|
||||
|
||||
type stubKubeClient struct {
|
||||
getComponentsVal versions.ComponentVersions
|
||||
getComponentsErr error
|
||||
|
||||
createConfigMapErr error
|
||||
|
||||
AddReferenceToK8sVersionConfigMapErr error
|
||||
}
|
||||
|
||||
func (s stubKubeClient) GetComponents(ctx context.Context, configMapName string) (versions.ComponentVersions, error) {
|
||||
return s.getComponentsVal, s.getComponentsErr
|
||||
}
|
||||
|
||||
func (s stubKubeClient) CreateConfigMap(ctx context.Context, configMap corev1.ConfigMap) error {
|
||||
return s.createConfigMapErr
|
||||
}
|
||||
|
||||
func (s stubKubeClient) AddReferenceToK8sVersionConfigMap(ctx context.Context, k8sVersionsConfigMapName string, componentsConfigMapName string) error {
|
||||
return s.AddReferenceToK8sVersionConfigMapErr
|
||||
}
|
||||
|
@ -97,6 +97,7 @@ type IssueJoinTicketResponse struct {
|
||||
DiscoveryTokenCaCertHash string `protobuf:"bytes,7,opt,name=discovery_token_ca_cert_hash,json=discoveryTokenCaCertHash,proto3" json:"discovery_token_ca_cert_hash,omitempty"`
|
||||
ControlPlaneFiles []*ControlPlaneCertOrKey `protobuf:"bytes,8,rep,name=control_plane_files,json=controlPlaneFiles,proto3" json:"control_plane_files,omitempty"`
|
||||
KubernetesVersion string `protobuf:"bytes,9,opt,name=kubernetes_version,json=kubernetesVersion,proto3" json:"kubernetes_version,omitempty"`
|
||||
KubernetesComponents []*KubernetesComponent `protobuf:"bytes,10,rep,name=kubernetes_components,json=kubernetesComponents,proto3" json:"kubernetes_components,omitempty"`
|
||||
}
|
||||
|
||||
func (x *IssueJoinTicketResponse) Reset() {
|
||||
@ -194,6 +195,13 @@ func (x *IssueJoinTicketResponse) GetKubernetesVersion() string {
|
||||
return ""
|
||||
}
|
||||
|
||||
func (x *IssueJoinTicketResponse) GetKubernetesComponents() []*KubernetesComponent {
|
||||
if x != nil {
|
||||
return x.KubernetesComponents
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
type ControlPlaneCertOrKey struct {
|
||||
state protoimpl.MessageState
|
||||
sizeCache protoimpl.SizeCache
|
||||
@ -351,6 +359,78 @@ func (x *IssueRejoinTicketResponse) GetMeasurementSecret() []byte {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Discuss if we want to import the init proto instead of duplicating it
|
||||
type KubernetesComponent struct {
|
||||
state protoimpl.MessageState
|
||||
sizeCache protoimpl.SizeCache
|
||||
unknownFields protoimpl.UnknownFields
|
||||
|
||||
Url string `protobuf:"bytes,1,opt,name=url,proto3" json:"url,omitempty"`
|
||||
Hash string `protobuf:"bytes,2,opt,name=hash,proto3" json:"hash,omitempty"`
|
||||
InstallPath string `protobuf:"bytes,3,opt,name=install_path,json=installPath,proto3" json:"install_path,omitempty"`
|
||||
Extract bool `protobuf:"varint,4,opt,name=extract,proto3" json:"extract,omitempty"`
|
||||
}
|
||||
|
||||
func (x *KubernetesComponent) Reset() {
|
||||
*x = KubernetesComponent{}
|
||||
if protoimpl.UnsafeEnabled {
|
||||
mi := &file_join_proto_msgTypes[5]
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
ms.StoreMessageInfo(mi)
|
||||
}
|
||||
}
|
||||
|
||||
func (x *KubernetesComponent) String() string {
|
||||
return protoimpl.X.MessageStringOf(x)
|
||||
}
|
||||
|
||||
func (*KubernetesComponent) ProtoMessage() {}
|
||||
|
||||
func (x *KubernetesComponent) ProtoReflect() protoreflect.Message {
|
||||
mi := &file_join_proto_msgTypes[5]
|
||||
if protoimpl.UnsafeEnabled && x != nil {
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
if ms.LoadMessageInfo() == nil {
|
||||
ms.StoreMessageInfo(mi)
|
||||
}
|
||||
return ms
|
||||
}
|
||||
return mi.MessageOf(x)
|
||||
}
|
||||
|
||||
// Deprecated: Use KubernetesComponent.ProtoReflect.Descriptor instead.
|
||||
func (*KubernetesComponent) Descriptor() ([]byte, []int) {
|
||||
return file_join_proto_rawDescGZIP(), []int{5}
|
||||
}
|
||||
|
||||
func (x *KubernetesComponent) GetUrl() string {
|
||||
if x != nil {
|
||||
return x.Url
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func (x *KubernetesComponent) GetHash() string {
|
||||
if x != nil {
|
||||
return x.Hash
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func (x *KubernetesComponent) GetInstallPath() string {
|
||||
if x != nil {
|
||||
return x.InstallPath
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func (x *KubernetesComponent) GetExtract() bool {
|
||||
if x != nil {
|
||||
return x.Extract
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
var File_join_proto protoreflect.FileDescriptor
|
||||
|
||||
var file_join_proto_rawDesc = []byte{
|
||||
@ -364,7 +444,7 @@ var file_join_proto_rawDesc = []byte{
|
||||
0x63, 0x61, 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x28, 0x0a, 0x10, 0x69,
|
||||
0x73, 0x5f, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x5f, 0x70, 0x6c, 0x61, 0x6e, 0x65, 0x18,
|
||||
0x03, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0e, 0x69, 0x73, 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c,
|
||||
0x50, 0x6c, 0x61, 0x6e, 0x65, 0x22, 0xc2, 0x03, 0x0a, 0x17, 0x49, 0x73, 0x73, 0x75, 0x65, 0x4a,
|
||||
0x50, 0x6c, 0x61, 0x6e, 0x65, 0x22, 0x92, 0x04, 0x0a, 0x17, 0x49, 0x73, 0x73, 0x75, 0x65, 0x4a,
|
||||
0x6f, 0x69, 0x6e, 0x54, 0x69, 0x63, 0x6b, 0x65, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73,
|
||||
0x65, 0x12, 0x24, 0x0a, 0x0e, 0x73, 0x74, 0x61, 0x74, 0x65, 0x5f, 0x64, 0x69, 0x73, 0x6b, 0x5f,
|
||||
0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0c, 0x73, 0x74, 0x61, 0x74, 0x65,
|
||||
@ -392,7 +472,12 @@ var file_join_proto_rawDesc = []byte{
|
||||
0x6c, 0x50, 0x6c, 0x61, 0x6e, 0x65, 0x46, 0x69, 0x6c, 0x65, 0x73, 0x12, 0x2d, 0x0a, 0x12, 0x6b,
|
||||
0x75, 0x62, 0x65, 0x72, 0x6e, 0x65, 0x74, 0x65, 0x73, 0x5f, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f,
|
||||
0x6e, 0x18, 0x09, 0x20, 0x01, 0x28, 0x09, 0x52, 0x11, 0x6b, 0x75, 0x62, 0x65, 0x72, 0x6e, 0x65,
|
||||
0x74, 0x65, 0x73, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x22, 0x43, 0x0a, 0x19, 0x63, 0x6f,
|
||||
0x74, 0x65, 0x73, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x4e, 0x0a, 0x15, 0x6b, 0x75,
|
||||
0x62, 0x65, 0x72, 0x6e, 0x65, 0x74, 0x65, 0x73, 0x5f, 0x63, 0x6f, 0x6d, 0x70, 0x6f, 0x6e, 0x65,
|
||||
0x6e, 0x74, 0x73, 0x18, 0x0a, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x6a, 0x6f, 0x69, 0x6e,
|
||||
0x2e, 0x4b, 0x75, 0x62, 0x65, 0x72, 0x6e, 0x65, 0x74, 0x65, 0x73, 0x43, 0x6f, 0x6d, 0x70, 0x6f,
|
||||
0x6e, 0x65, 0x6e, 0x74, 0x52, 0x14, 0x6b, 0x75, 0x62, 0x65, 0x72, 0x6e, 0x65, 0x74, 0x65, 0x73,
|
||||
0x43, 0x6f, 0x6d, 0x70, 0x6f, 0x6e, 0x65, 0x6e, 0x74, 0x73, 0x22, 0x43, 0x0a, 0x19, 0x63, 0x6f,
|
||||
0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x5f, 0x70, 0x6c, 0x61, 0x6e, 0x65, 0x5f, 0x63, 0x65, 0x72, 0x74,
|
||||
0x5f, 0x6f, 0x72, 0x5f, 0x6b, 0x65, 0x79, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18,
|
||||
0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x64,
|
||||
@ -407,23 +492,30 @@ var file_join_proto_rawDesc = []byte{
|
||||
0x74, 0x61, 0x74, 0x65, 0x44, 0x69, 0x73, 0x6b, 0x4b, 0x65, 0x79, 0x12, 0x2d, 0x0a, 0x12, 0x6d,
|
||||
0x65, 0x61, 0x73, 0x75, 0x72, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x5f, 0x73, 0x65, 0x63, 0x72, 0x65,
|
||||
0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x11, 0x6d, 0x65, 0x61, 0x73, 0x75, 0x72, 0x65,
|
||||
0x6d, 0x65, 0x6e, 0x74, 0x53, 0x65, 0x63, 0x72, 0x65, 0x74, 0x32, 0xab, 0x01, 0x0a, 0x03, 0x41,
|
||||
0x50, 0x49, 0x12, 0x4e, 0x0a, 0x0f, 0x49, 0x73, 0x73, 0x75, 0x65, 0x4a, 0x6f, 0x69, 0x6e, 0x54,
|
||||
0x69, 0x63, 0x6b, 0x65, 0x74, 0x12, 0x1c, 0x2e, 0x6a, 0x6f, 0x69, 0x6e, 0x2e, 0x49, 0x73, 0x73,
|
||||
0x75, 0x65, 0x4a, 0x6f, 0x69, 0x6e, 0x54, 0x69, 0x63, 0x6b, 0x65, 0x74, 0x52, 0x65, 0x71, 0x75,
|
||||
0x65, 0x73, 0x74, 0x1a, 0x1d, 0x2e, 0x6a, 0x6f, 0x69, 0x6e, 0x2e, 0x49, 0x73, 0x73, 0x75, 0x65,
|
||||
0x4a, 0x6f, 0x69, 0x6e, 0x54, 0x69, 0x63, 0x6b, 0x65, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e,
|
||||
0x73, 0x65, 0x12, 0x54, 0x0a, 0x11, 0x49, 0x73, 0x73, 0x75, 0x65, 0x52, 0x65, 0x6a, 0x6f, 0x69,
|
||||
0x6e, 0x54, 0x69, 0x63, 0x6b, 0x65, 0x74, 0x12, 0x1e, 0x2e, 0x6a, 0x6f, 0x69, 0x6e, 0x2e, 0x49,
|
||||
0x73, 0x73, 0x75, 0x65, 0x52, 0x65, 0x6a, 0x6f, 0x69, 0x6e, 0x54, 0x69, 0x63, 0x6b, 0x65, 0x74,
|
||||
0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1f, 0x2e, 0x6a, 0x6f, 0x69, 0x6e, 0x2e, 0x49,
|
||||
0x73, 0x73, 0x75, 0x65, 0x52, 0x65, 0x6a, 0x6f, 0x69, 0x6e, 0x54, 0x69, 0x63, 0x6b, 0x65, 0x74,
|
||||
0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x42, 0x3f, 0x5a, 0x3d, 0x67, 0x69, 0x74, 0x68,
|
||||
0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x65, 0x64, 0x67, 0x65, 0x6c, 0x65, 0x73, 0x73, 0x73,
|
||||
0x79, 0x73, 0x2f, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x65, 0x6c, 0x6c, 0x61, 0x74, 0x69, 0x6f, 0x6e,
|
||||
0x2f, 0x76, 0x32, 0x2f, 0x6a, 0x6f, 0x69, 0x6e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x2f,
|
||||
0x6a, 0x6f, 0x69, 0x6e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f,
|
||||
0x33,
|
||||
0x6d, 0x65, 0x6e, 0x74, 0x53, 0x65, 0x63, 0x72, 0x65, 0x74, 0x22, 0x78, 0x0a, 0x13, 0x4b, 0x75,
|
||||
0x62, 0x65, 0x72, 0x6e, 0x65, 0x74, 0x65, 0x73, 0x43, 0x6f, 0x6d, 0x70, 0x6f, 0x6e, 0x65, 0x6e,
|
||||
0x74, 0x12, 0x10, 0x0a, 0x03, 0x75, 0x72, 0x6c, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03,
|
||||
0x75, 0x72, 0x6c, 0x12, 0x12, 0x0a, 0x04, 0x68, 0x61, 0x73, 0x68, 0x18, 0x02, 0x20, 0x01, 0x28,
|
||||
0x09, 0x52, 0x04, 0x68, 0x61, 0x73, 0x68, 0x12, 0x21, 0x0a, 0x0c, 0x69, 0x6e, 0x73, 0x74, 0x61,
|
||||
0x6c, 0x6c, 0x5f, 0x70, 0x61, 0x74, 0x68, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x69,
|
||||
0x6e, 0x73, 0x74, 0x61, 0x6c, 0x6c, 0x50, 0x61, 0x74, 0x68, 0x12, 0x18, 0x0a, 0x07, 0x65, 0x78,
|
||||
0x74, 0x72, 0x61, 0x63, 0x74, 0x18, 0x04, 0x20, 0x01, 0x28, 0x08, 0x52, 0x07, 0x65, 0x78, 0x74,
|
||||
0x72, 0x61, 0x63, 0x74, 0x32, 0xab, 0x01, 0x0a, 0x03, 0x41, 0x50, 0x49, 0x12, 0x4e, 0x0a, 0x0f,
|
||||
0x49, 0x73, 0x73, 0x75, 0x65, 0x4a, 0x6f, 0x69, 0x6e, 0x54, 0x69, 0x63, 0x6b, 0x65, 0x74, 0x12,
|
||||
0x1c, 0x2e, 0x6a, 0x6f, 0x69, 0x6e, 0x2e, 0x49, 0x73, 0x73, 0x75, 0x65, 0x4a, 0x6f, 0x69, 0x6e,
|
||||
0x54, 0x69, 0x63, 0x6b, 0x65, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1d, 0x2e,
|
||||
0x6a, 0x6f, 0x69, 0x6e, 0x2e, 0x49, 0x73, 0x73, 0x75, 0x65, 0x4a, 0x6f, 0x69, 0x6e, 0x54, 0x69,
|
||||
0x63, 0x6b, 0x65, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x54, 0x0a, 0x11,
|
||||
0x49, 0x73, 0x73, 0x75, 0x65, 0x52, 0x65, 0x6a, 0x6f, 0x69, 0x6e, 0x54, 0x69, 0x63, 0x6b, 0x65,
|
||||
0x74, 0x12, 0x1e, 0x2e, 0x6a, 0x6f, 0x69, 0x6e, 0x2e, 0x49, 0x73, 0x73, 0x75, 0x65, 0x52, 0x65,
|
||||
0x6a, 0x6f, 0x69, 0x6e, 0x54, 0x69, 0x63, 0x6b, 0x65, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73,
|
||||
0x74, 0x1a, 0x1f, 0x2e, 0x6a, 0x6f, 0x69, 0x6e, 0x2e, 0x49, 0x73, 0x73, 0x75, 0x65, 0x52, 0x65,
|
||||
0x6a, 0x6f, 0x69, 0x6e, 0x54, 0x69, 0x63, 0x6b, 0x65, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e,
|
||||
0x73, 0x65, 0x42, 0x3f, 0x5a, 0x3d, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d,
|
||||
0x2f, 0x65, 0x64, 0x67, 0x65, 0x6c, 0x65, 0x73, 0x73, 0x73, 0x79, 0x73, 0x2f, 0x63, 0x6f, 0x6e,
|
||||
0x73, 0x74, 0x65, 0x6c, 0x6c, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x2f, 0x76, 0x32, 0x2f, 0x6a, 0x6f,
|
||||
0x69, 0x6e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x2f, 0x6a, 0x6f, 0x69, 0x6e, 0x70, 0x72,
|
||||
0x6f, 0x74, 0x6f, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
|
||||
}
|
||||
|
||||
var (
|
||||
@ -438,25 +530,27 @@ func file_join_proto_rawDescGZIP() []byte {
|
||||
return file_join_proto_rawDescData
|
||||
}
|
||||
|
||||
var file_join_proto_msgTypes = make([]protoimpl.MessageInfo, 5)
|
||||
var file_join_proto_msgTypes = make([]protoimpl.MessageInfo, 6)
|
||||
var file_join_proto_goTypes = []interface{}{
|
||||
(*IssueJoinTicketRequest)(nil), // 0: join.IssueJoinTicketRequest
|
||||
(*IssueJoinTicketResponse)(nil), // 1: join.IssueJoinTicketResponse
|
||||
(*ControlPlaneCertOrKey)(nil), // 2: join.control_plane_cert_or_key
|
||||
(*IssueRejoinTicketRequest)(nil), // 3: join.IssueRejoinTicketRequest
|
||||
(*IssueRejoinTicketResponse)(nil), // 4: join.IssueRejoinTicketResponse
|
||||
(*KubernetesComponent)(nil), // 5: join.KubernetesComponent
|
||||
}
|
||||
var file_join_proto_depIdxs = []int32{
|
||||
2, // 0: join.IssueJoinTicketResponse.control_plane_files:type_name -> join.control_plane_cert_or_key
|
||||
0, // 1: join.API.IssueJoinTicket:input_type -> join.IssueJoinTicketRequest
|
||||
3, // 2: join.API.IssueRejoinTicket:input_type -> join.IssueRejoinTicketRequest
|
||||
1, // 3: join.API.IssueJoinTicket:output_type -> join.IssueJoinTicketResponse
|
||||
4, // 4: join.API.IssueRejoinTicket:output_type -> join.IssueRejoinTicketResponse
|
||||
3, // [3:5] is the sub-list for method output_type
|
||||
1, // [1:3] is the sub-list for method input_type
|
||||
1, // [1:1] is the sub-list for extension type_name
|
||||
1, // [1:1] is the sub-list for extension extendee
|
||||
0, // [0:1] is the sub-list for field type_name
|
||||
5, // 1: join.IssueJoinTicketResponse.kubernetes_components:type_name -> join.KubernetesComponent
|
||||
0, // 2: join.API.IssueJoinTicket:input_type -> join.IssueJoinTicketRequest
|
||||
3, // 3: join.API.IssueRejoinTicket:input_type -> join.IssueRejoinTicketRequest
|
||||
1, // 4: join.API.IssueJoinTicket:output_type -> join.IssueJoinTicketResponse
|
||||
4, // 5: join.API.IssueRejoinTicket:output_type -> join.IssueRejoinTicketResponse
|
||||
4, // [4:6] is the sub-list for method output_type
|
||||
2, // [2:4] is the sub-list for method input_type
|
||||
2, // [2:2] is the sub-list for extension type_name
|
||||
2, // [2:2] is the sub-list for extension extendee
|
||||
0, // [0:2] is the sub-list for field type_name
|
||||
}
|
||||
|
||||
func init() { file_join_proto_init() }
|
||||
@ -525,6 +619,18 @@ func file_join_proto_init() {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
file_join_proto_msgTypes[5].Exporter = func(v interface{}, i int) interface{} {
|
||||
switch v := v.(*KubernetesComponent); i {
|
||||
case 0:
|
||||
return &v.state
|
||||
case 1:
|
||||
return &v.sizeCache
|
||||
case 2:
|
||||
return &v.unknownFields
|
||||
default:
|
||||
return nil
|
||||
}
|
||||
}
|
||||
}
|
||||
type x struct{}
|
||||
out := protoimpl.TypeBuilder{
|
||||
@ -532,7 +638,7 @@ func file_join_proto_init() {
|
||||
GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
|
||||
RawDescriptor: file_join_proto_rawDesc,
|
||||
NumEnums: 0,
|
||||
NumMessages: 5,
|
||||
NumMessages: 6,
|
||||
NumExtensions: 0,
|
||||
NumServices: 1,
|
||||
},
|
||||
|
@ -26,6 +26,7 @@ message IssueJoinTicketResponse {
|
||||
string discovery_token_ca_cert_hash = 7;
|
||||
repeated control_plane_cert_or_key control_plane_files = 8;
|
||||
string kubernetes_version = 9;
|
||||
repeated KubernetesComponent kubernetes_components = 10;
|
||||
}
|
||||
|
||||
message control_plane_cert_or_key {
|
||||
@ -41,3 +42,11 @@ message IssueRejoinTicketResponse {
|
||||
bytes state_disk_key = 1;
|
||||
bytes measurement_secret = 2;
|
||||
}
|
||||
|
||||
// Discuss if we want to import the init proto instead of duplicating it
|
||||
message KubernetesComponent {
|
||||
string url = 1;
|
||||
string hash = 2;
|
||||
string install_path = 3;
|
||||
bool extract = 4;
|
||||
}
|
||||
|
Loading…
Reference in New Issue
Block a user