Git-Mirrors/constellation
1
0
Fork 0
mirror of https://github.com/edgelesssys/constellation.git synced 2025-12-15 16:09:39 -05:00
Code Issues Projects Releases Packages Wiki Activity

AB#2589: Deploy operators via Helm (#575)

Browse source 
* Only deploy operators on GCP/Azure.
* cert-manager is now deployed by default (GCP/Azure)
* remove OLM
This commit is contained in:
Otto Bittner 2022-11-21 10:35:40 +01:00 • committed by GitHub
parent b8d991f84c
commit bdd9dd922b
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
146 changed files with 12799 additions and 8706 deletions
Download patch file Download diff file Expand all files Collapse all files

15
bootstrapper/internal/kubernetes/k8sapi/k8sutil.go
View file

@ -42,8 +42,6 @@ import (
const (
// kubeletStartTimeout is the maximum time given to the kubelet service to (re)start.
kubeletStartTimeout = 10 * time.Minute
// crdTimeout is the maximum time given to the CRDs to be created.
crdTimeout = 30 * time.Second
)
// Client provides the functions to talk to the k8s API.
@ -326,19 +324,6 @@ func (k *KubernetesUtil) SetupVerificationService(kubectl Client, verificationSe
return kubectl.Apply(verificationServiceConfiguration, true)
}
// SetupOperatorLifecycleManager deploys operator lifecycle manager.
func (k *KubernetesUtil) SetupOperatorLifecycleManager(ctx context.Context, kubectl Client, olmCRDs, olmConfiguration kubernetes.Marshaler, crdNames []string) error {
if err := kubectl.Apply(olmCRDs, true); err != nil {
return fmt.Errorf("applying OLM CRDs: %w", err)
}
crdReadyTimeout, cancel := context.WithTimeout(ctx, crdTimeout)
defer cancel()
if err := kubectl.WaitForCRDs(crdReadyTimeout, crdNames); err != nil {
return fmt.Errorf("waiting for OLM CRDs: %w", err)
}
return kubectl.Apply(olmConfiguration, true)
}
// SetupNodeMaintenanceOperator deploys node maintenance operator.
func (k *KubernetesUtil) SetupNodeMaintenanceOperator(kubectl Client, nodeMaintenanceOperatorConfiguration kubernetes.Marshaler) error {
return kubectl.Apply(nodeMaintenanceOperatorConfiguration, true)

87
bootstrapper/internal/kubernetes/k8sapi/resources/node_maintenance_operator.go
View file

@ -1,87 +0,0 @@
/*
Copyright (c) Edgeless Systems GmbH
SPDX-License-Identifier: AGPL-3.0-only
*/
package resources
import (
"time"
"github.com/edgelesssys/constellation/v2/internal/kubernetes"
"github.com/edgelesssys/constellation/v2/internal/versions"
operatorsv1 "github.com/operator-framework/api/pkg/operators/v1"
operatorsv1alpha1 "github.com/operator-framework/api/pkg/operators/v1alpha1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)
const (
nodeMaintenanceOperatorNamespace = "kube-system"
nodeMaintenanceOperatorCatalogNamespace = "olm"
)
// NodeMaintenanceOperatorDeployment groups all deployments for node maintenance operator.
type NodeMaintenanceOperatorDeployment struct {
CatalogSource operatorsv1alpha1.CatalogSource
OperatorGroup operatorsv1.OperatorGroup
Subscription operatorsv1alpha1.Subscription
}
// NewNodeMaintenanceOperatorDeployment creates a new node maintenance operator (NMO) deployment.
// See https://github.com/medik8s/node-maintenance-operator for more information.
func NewNodeMaintenanceOperatorDeployment() *NodeMaintenanceOperatorDeployment {
return &NodeMaintenanceOperatorDeployment{
CatalogSource: operatorsv1alpha1.CatalogSource{
TypeMeta: metav1.TypeMeta{APIVersion: "operators.coreos.com/v1alpha1", Kind: "CatalogSource"},
ObjectMeta: metav1.ObjectMeta{
Name: "node-maintenance-operator-catalog",
Namespace: nodeMaintenanceOperatorCatalogNamespace,
},
Spec: operatorsv1alpha1.CatalogSourceSpec{
SourceType: "grpc",
Image: versions.NodeMaintenanceOperatorCatalogImage,
DisplayName: "Node Maintenance Operator",
Publisher: "Medik8s Team",
UpdateStrategy: &operatorsv1alpha1.UpdateStrategy{
RegistryPoll: &operatorsv1alpha1.RegistryPoll{
RawInterval: "1m0s",
Interval: &metav1.Duration{
Duration: time.Minute,
},
},
},
},
},
OperatorGroup: operatorsv1.OperatorGroup{
TypeMeta: metav1.TypeMeta{APIVersion: "operators.coreos.com/v1", Kind: "OperatorGroup"},
ObjectMeta: metav1.ObjectMeta{
Name: "constellation-og",
Namespace: nodeMaintenanceOperatorNamespace,
},
Spec: operatorsv1.OperatorGroupSpec{
UpgradeStrategy: operatorsv1.UpgradeStrategyDefault,
},
},
Subscription: operatorsv1alpha1.Subscription{
TypeMeta: metav1.TypeMeta{APIVersion: "operators.coreos.com/v1alpha1", Kind: "Subscription"},
ObjectMeta: metav1.ObjectMeta{
Name: "node-maintenance-operator-sub",
Namespace: nodeMaintenanceOperatorNamespace,
},
Spec: &operatorsv1alpha1.SubscriptionSpec{
Channel: "stable",
Package: "node-maintenance-operator",
CatalogSource: "node-maintenance-operator-catalog",
CatalogSourceNamespace: "olm",
InstallPlanApproval: operatorsv1alpha1.ApprovalAutomatic,
StartingCSV: "node-maintenance-operator." + versions.NodeMaintenanceOperatorVersion,
},
},
}
}
// Marshal to Kubernetes YAML.
func (c *NodeMaintenanceOperatorDeployment) Marshal() ([]byte, error) {
return kubernetes.MarshalK8SResources(c)
}

28
bootstrapper/internal/kubernetes/k8sapi/resources/node_maintenance_operator_test.go
View file

@ -1,28 +0,0 @@
/*
Copyright (c) Edgeless Systems GmbH
SPDX-License-Identifier: AGPL-3.0-only
*/
package resources
import (
"testing"
"github.com/edgelesssys/constellation/v2/internal/kubernetes"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestNodeMaintenanceOperatorMarshalUnmarshal(t *testing.T) {
require := require.New(t)
assert := assert.New(t)
nmoDepl := NewNodeMaintenanceOperatorDeployment()
data, err := nmoDepl.Marshal()
require.NoError(err)
var recreated NodeMaintenanceOperatorDeployment
require.NoError(kubernetes.UnmarshalK8SResources(data, &recreated))
assert.Equal(nmoDepl, &recreated)
}

92
bootstrapper/internal/kubernetes/k8sapi/resources/node_operator.go
View file

@ -1,92 +0,0 @@
/*
Copyright (c) Edgeless Systems GmbH
SPDX-License-Identifier: AGPL-3.0-only
*/
package resources
import (
"time"
"github.com/edgelesssys/constellation/v2/internal/kubernetes"
"github.com/edgelesssys/constellation/v2/internal/versions"
operatorsv1 "github.com/operator-framework/api/pkg/operators/v1"
operatorsv1alpha1 "github.com/operator-framework/api/pkg/operators/v1alpha1"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)
const (
nodeOperatorNamespace = "kube-system"
nodeOperatorCatalogNamespace = "olm"
)
// NodeOperatorDeployment groups all deployments for node operator.
type NodeOperatorDeployment struct {
CatalogSource operatorsv1alpha1.CatalogSource
OperatorGroup operatorsv1.OperatorGroup
Subscription operatorsv1alpha1.Subscription
}
// NewNodeOperatorDeployment creates a new constellation node operator deployment.
// See /operators/constellation-node-operator for more information.
func NewNodeOperatorDeployment(cloudProvider string, uid string) *NodeOperatorDeployment {
return &NodeOperatorDeployment{
CatalogSource: operatorsv1alpha1.CatalogSource{
TypeMeta: metav1.TypeMeta{APIVersion: "operators.coreos.com/v1alpha1", Kind: "CatalogSource"},
ObjectMeta: metav1.ObjectMeta{
Name: "constellation-node-operator-catalog",
Namespace: nodeOperatorCatalogNamespace,
},
Spec: operatorsv1alpha1.CatalogSourceSpec{
SourceType: "grpc",
Image: versions.NodeOperatorCatalogImage,
DisplayName: "Constellation Node Operator",
Publisher: "Edgeless Systems",
UpdateStrategy: &operatorsv1alpha1.UpdateStrategy{
RegistryPoll: &operatorsv1alpha1.RegistryPoll{
RawInterval: "1m0s",
Interval: &metav1.Duration{Duration: 1 * time.Minute},
},
},
},
},
OperatorGroup: operatorsv1.OperatorGroup{
TypeMeta: metav1.TypeMeta{APIVersion: "operators.coreos.com/v1", Kind: "OperatorGroup"},
ObjectMeta: metav1.ObjectMeta{
Name: "constellation-og",
Namespace: nodeOperatorNamespace,
},
Spec: operatorsv1.OperatorGroupSpec{
UpgradeStrategy: operatorsv1.UpgradeStrategyDefault,
},
},
Subscription: operatorsv1alpha1.Subscription{
TypeMeta: metav1.TypeMeta{APIVersion: "operators.coreos.com/v1alpha1", Kind: "Subscription"},
ObjectMeta: metav1.ObjectMeta{
Name: "constellation-node-operator-sub",
Namespace: nodeOperatorNamespace,
},
Spec: &operatorsv1alpha1.SubscriptionSpec{
Channel: "alpha",
Package: "node-operator",
CatalogSource: "constellation-node-operator-catalog",
CatalogSourceNamespace: "olm",
InstallPlanApproval: operatorsv1alpha1.ApprovalAutomatic,
StartingCSV: "node-operator." + versions.NodeOperatorVersion,
Config: &operatorsv1alpha1.SubscriptionConfig{
Env: []corev1.EnvVar{
{Name: "CONSTEL_CSP", Value: cloudProvider},
{Name: "constellation-uid", Value: uid},
},
},
},
},
}
}
// Marshal to Kubernetes YAML.
func (c *NodeOperatorDeployment) Marshal() ([]byte, error) {
return kubernetes.MarshalK8SResources(c)
}

28
bootstrapper/internal/kubernetes/k8sapi/resources/node_operator_test.go
View file

@ -1,28 +0,0 @@
/*
Copyright (c) Edgeless Systems GmbH
SPDX-License-Identifier: AGPL-3.0-only
*/
package resources
import (
"testing"
"github.com/edgelesssys/constellation/v2/internal/kubernetes"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestNodeOperatorMarshalUnmarshal(t *testing.T) {
require := require.New(t)
assert := assert.New(t)
nmoDepl := NewNodeOperatorDeployment("csp", "uid")
data, err := nmoDepl.Marshal()
require.NoError(err)
var recreated NodeOperatorDeployment
require.NoError(kubernetes.UnmarshalK8SResources(data, &recreated))
assert.Equal(nmoDepl, &recreated)
}

37
bootstrapper/internal/kubernetes/k8sapi/resources/operator_lifecycle_manager.go
View file

@ -1,37 +0,0 @@
/*
Copyright (c) Edgeless Systems GmbH
SPDX-License-Identifier: AGPL-3.0-only
*/
package resources
import "github.com/edgelesssys/constellation/v2/internal/crds"
// OLMCRDNames are the names of the custom resource definitions that are used by the olm operator.
var OLMCRDNames = []string{
"catalogsources.operators.coreos.com",
"clusterserviceversions.operators.coreos.com",
"installplans.operators.coreos.com",
"olmconfigs.operators.coreos.com",
"operatorconditions.operators.coreos.com",
"operatorgroups.operators.coreos.com",
"operators.operators.coreos.com",
"subscriptions.operators.coreos.com",
}
// OperatorLifecycleManagerCRDs contains custom resource definitions used by the olm operator.
type OperatorLifecycleManagerCRDs struct{}
// Marshal returns the already marshalled CRDs.
func (m *OperatorLifecycleManagerCRDs) Marshal() ([]byte, error) {
return crds.OLMCRDs, nil
}
// OperatorLifecycleManager is the deployment of the olm operator.
type OperatorLifecycleManager struct{}
// Marshal returns the already marshalled deployment yaml.
func (m *OperatorLifecycleManager) Marshal() ([]byte, error) {
return crds.OLM, nil
}

5
bootstrapper/internal/kubernetes/k8sutil.go
View file

@ -25,9 +25,6 @@ type clusterUtil interface {
SetupKonnectivity(kubectl k8sapi.Client, konnectivityAgentsDaemonSet kubernetes.Marshaler) error
SetupVerificationService(kubectl k8sapi.Client, verificationServiceConfiguration kubernetes.Marshaler) error
SetupGCPGuestAgent(kubectl k8sapi.Client, gcpGuestAgentConfiguration kubernetes.Marshaler) error
SetupOperatorLifecycleManager(ctx context.Context, kubectl k8sapi.Client, olmCRDs, olmConfiguration kubernetes.Marshaler, crdNames []string) error
SetupNodeMaintenanceOperator(kubectl k8sapi.Client, nodeMaintenanceOperatorConfiguration kubernetes.Marshaler) error
SetupNodeOperator(ctx context.Context, kubectl k8sapi.Client, nodeOperatorConfiguration kubernetes.Marshaler) error
FixCilium(log *logger.Logger)
StartKubelet() error
}
@ -37,5 +34,7 @@ type clusterUtil interface {
// Naming is inspired by Helm.
type helmClient interface {
InstallCilium(context.Context, k8sapi.Client, helm.Release, k8sapi.SetupPodNetworkInput) error
InstallCertManager(ctx context.Context, release helm.Release) error
InstallOperators(ctx context.Context, release helm.Release, extraVals map[string]any) error
InstallConstellationServices(ctx context.Context, release helm.Release, extraVals map[string]any) error
}

52
bootstrapper/internal/kubernetes/kubernetes.go
View file

@ -211,8 +211,21 @@ func (k *KubeWrapper) InitCluster(
return nil, fmt.Errorf("failed to setup verification service: %w", err)
}
if err := k.setupOperators(ctx); err != nil {
return nil, fmt.Errorf("setting up operators: %w", err)
// cert-manager is necessary for our operator deployments.
// They are currently only deployed on GCP & Azure. This is why we deploy cert-manager only on GCP & Azure.
if k.cloudProvider == "gcp" || k.cloudProvider == "azure" {
if err = k.helmClient.InstallCertManager(ctx, helmReleases.CertManager); err != nil {
return nil, fmt.Errorf("installing cert-manager: %w", err)
}
}
operatorVals, err := k.setupOperatorVals(ctx)
if err != nil {
return nil, fmt.Errorf("setting up operator vals: %w", err)
}
if err = k.helmClient.InstallOperators(ctx, helmReleases.Operators, operatorVals); err != nil {
return nil, fmt.Errorf("installing operators: %w", err)
}
if k.cloudProvider == "gcp" {
@ -346,28 +359,6 @@ func (k *KubeWrapper) setupInternalConfigMap(ctx context.Context, azureCVM strin
return nil
}
// setupOperators deploys the operator lifecycle manager and subscriptions to operators.
func (k *KubeWrapper) setupOperators(ctx context.Context) error {
if err := k.clusterUtil.SetupOperatorLifecycleManager(ctx, k.client, &resources.OperatorLifecycleManagerCRDs{}, &resources.OperatorLifecycleManager{}, resources.OLMCRDNames); err != nil {
return fmt.Errorf("setting up OLM: %w", err)
}
if err := k.clusterUtil.SetupNodeMaintenanceOperator(k.client, resources.NewNodeMaintenanceOperatorDeployment()); err != nil {
return fmt.Errorf("setting up node maintenance operator: %w", err)
}
uid, err := k.providerMetadata.UID(ctx)
if err != nil {
return fmt.Errorf("retrieving constellation UID: %w", err)
}
if err := k.clusterUtil.SetupNodeOperator(ctx, k.client, resources.NewNodeOperatorDeployment(k.cloudProvider, uid)); err != nil {
return fmt.Errorf("setting up constellation node operator: %w", err)
}
return nil
}
// k8sCompliantHostname transforms a hostname to an RFC 1123 compliant, lowercase subdomain as required by Kubernetes node names.
// The following regex is used by k8s for validation: /^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$/ .
// Only a simple heuristic is used for now (to lowercase, replace underscores).
@ -497,6 +488,19 @@ func (k *KubeWrapper) setupExtraVals(ctx context.Context, initialMeasurementsJSO
return extraVals, nil
}
func (k *KubeWrapper) setupOperatorVals(ctx context.Context) (map[string]any, error) {
uid, err := k.providerMetadata.UID(ctx)
if err != nil {
return nil, fmt.Errorf("retrieving constellation UID: %w", err)
}
return map[string]any{
"constellation-operator": map[string]any{
"constellationUID": uid,
},
}, nil
}
type ccmConfigGetter interface {
GetCCMConfig(ctx context.Context, providerID, cloudServiceAccountURI string) ([]byte, error)
}

14
bootstrapper/internal/kubernetes/kubernetes_test.go
View file

@ -551,14 +551,24 @@ func (s *stubKubeconfigReader) ReadKubeconfig() ([]byte, error) {
}
type stubHelmClient struct {
ciliumError error
servicesError error
ciliumError error
certManagerError error
operatorsError error
servicesError error
}
func (s *stubHelmClient) InstallCilium(ctx context.Context, kubectl k8sapi.Client, release helm.Release, in k8sapi.SetupPodNetworkInput) error {
return s.ciliumError
}
func (s *stubHelmClient) InstallCertManager(ctx context.Context, release helm.Release) error {
return s.certManagerError
}
func (s *stubHelmClient) InstallOperators(ctx context.Context, release helm.Release, extraVals map[string]any) error {
return s.operatorsError
}
func (s *stubHelmClient) InstallConstellationServices(ctx context.Context, release helm.Release, extraVals map[string]any) error {
return s.servicesError
}