constellation/operators/constellation-node-operator/internal/deploy/deploy.go

/*
Copyright (c) Edgeless Systems GmbH

SPDX-License-Identifier: AGPL-3.0-only
*/

// Package deploy provides functions to deploy initial resources for the node operator.
package deploy

import (
	"context"
	"errors"
	"fmt"
	"strings"

	updatev1alpha1 "github.com/edgelesssys/constellation/operators/constellation-node-operator/v2/api/v1alpha1"
	"github.com/edgelesssys/constellation/operators/constellation-node-operator/v2/internal/constants"
	k8sErrors "k8s.io/apimachinery/pkg/api/errors"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"sigs.k8s.io/controller-runtime/pkg/client"
	"sigs.k8s.io/controller-runtime/pkg/log"
)

// InitialResources creates the initial resources for the node operator.
func InitialResources(ctx context.Context, k8sClient client.Writer, imageInfo imageInfoGetter, scalingGroupGetter scalingGroupGetter, uid string) error {
	logr := log.FromContext(ctx)
	controlPlaneGroupIDs, workerGroupIDs, err := scalingGroupGetter.ListScalingGroups(ctx, uid)
	if err != nil {
		return fmt.Errorf("listing scaling groups: %w", err)
	}
	if len(controlPlaneGroupIDs) == 0 {
		return errors.New("determining initial node image: no control plane scaling group found")
	}
	if len(workerGroupIDs) == 0 {
		return errors.New("determining initial node image: no worker scaling group found")
	}

	if err := createAutoscalingStrategy(ctx, k8sClient, scalingGroupGetter.AutoscalingCloudProvider()); err != nil {
		return fmt.Errorf("creating initial autoscaling strategy: %w", err)
	}
	imageReference, err := scalingGroupGetter.GetScalingGroupImage(ctx, controlPlaneGroupIDs[0])
	if err != nil {
		return fmt.Errorf("determining initial node image: %w", err)
	}
	imageVersion, err := imageInfo.ImageVersion(imageReference)
	if err != nil {
		// do not fail if the image version cannot be determined
		// this is important for backwards compatibility
		logr.Error(err, "determining initial node image version")
		imageVersion = ""
	}

	if err := createNodeImage(ctx, k8sClient, imageReference, imageVersion); err != nil {
		return fmt.Errorf("creating initial node image %q: %w", imageReference, err)
	}
	for _, groupID := range controlPlaneGroupIDs {
		groupName, err := scalingGroupGetter.GetScalingGroupName(groupID)
		if err != nil {
			return fmt.Errorf("determining scaling group name of %q: %w", groupID, err)
		}
		autoscalingGroupName, err := scalingGroupGetter.GetAutoscalingGroupName(groupID)
		if err != nil {
			return fmt.Errorf("determining autoscaling group name of %q: %w", groupID, err)
		}
		newScalingGroupConfig := newScalingGroupConfig{k8sClient, groupID, groupName, autoscalingGroupName, updatev1alpha1.ControlPlaneRole}
		if err := createScalingGroup(ctx, newScalingGroupConfig); err != nil {
			return fmt.Errorf("creating initial control plane scaling group: %w", err)
		}
	}
	for _, groupID := range workerGroupIDs {
		groupName, err := scalingGroupGetter.GetScalingGroupName(groupID)
		if err != nil {
			return fmt.Errorf("determining scaling group name of %q: %w", groupID, err)
		}
		autoscalingGroupName, err := scalingGroupGetter.GetAutoscalingGroupName(groupID)
		if err != nil {
			return fmt.Errorf("determining autoscaling group name of %q: %w", groupID, err)
		}
		newScalingGroupConfig := newScalingGroupConfig{k8sClient, groupID, groupName, autoscalingGroupName, updatev1alpha1.WorkerRole}
		if err := createScalingGroup(ctx, newScalingGroupConfig); err != nil {
			return fmt.Errorf("creating initial worker scaling group: %w", err)
		}
	}
	return nil
}

// createAutoscalingStrategy creates the autoscaling strategy resource if it does not exist yet.
func createAutoscalingStrategy(ctx context.Context, k8sClient client.Writer, provider string) error {
	err := k8sClient.Create(ctx, &updatev1alpha1.AutoscalingStrategy{
		TypeMeta: metav1.TypeMeta{APIVersion: "update.edgeless.systems/v1alpha1", Kind: "AutoscalingStrategy"},
		ObjectMeta: metav1.ObjectMeta{
			Name: constants.AutoscalingStrategyResourceName,
		},
		Spec: updatev1alpha1.AutoscalingStrategySpec{
			Enabled:             true,
			DeploymentName:      "constellation-cluster-autoscaler",
			DeploymentNamespace: "kube-system",
			AutoscalerExtraArgs: map[string]string{
				"cloud-provider":  provider,
				"logtostderr":     "true",
				"stderrthreshold": "info",
				"v":               "2",
				"namespace":       "kube-system",
			},
		},
	})
	if k8sErrors.IsAlreadyExists(err) {
		return nil
	}
	return err
}

// createNodeImage creates the initial nodeimage resource if it does not exist yet.
func createNodeImage(ctx context.Context, k8sClient client.Writer, imageReference, imageVersion string) error {
	err := k8sClient.Create(ctx, &updatev1alpha1.NodeImage{
		TypeMeta: metav1.TypeMeta{APIVersion: "update.edgeless.systems/v1alpha1", Kind: "NodeImage"},
		ObjectMeta: metav1.ObjectMeta{
			Name: constants.NodeImageResourceName,
		},
		Spec: updatev1alpha1.NodeImageSpec{
			ImageReference: imageReference,
			ImageVersion:   imageVersion,
		},
	})
	if k8sErrors.IsAlreadyExists(err) {
		return nil
	}
	return err
}

// createScalingGroup creates an initial scaling group resource if it does not exist yet.
func createScalingGroup(ctx context.Context, config newScalingGroupConfig) error {
	err := config.k8sClient.Create(ctx, &updatev1alpha1.ScalingGroup{
		TypeMeta: metav1.TypeMeta{APIVersion: "update.edgeless.systems/v1alpha1", Kind: "ScalingGroup"},
		ObjectMeta: metav1.ObjectMeta{
			Name: strings.ToLower(config.groupName),
		},
		Spec: updatev1alpha1.ScalingGroupSpec{
			NodeImage:           constants.NodeImageResourceName,
			GroupID:             config.groupID,
			AutoscalerGroupName: config.autoscalingGroupName,
			Min:                 1,
			Max:                 10,
			Role:                config.role,
		},
	})
	if k8sErrors.IsAlreadyExists(err) {
		return nil
	}
	return err
}

type imageInfoGetter interface {
	ImageVersion(imageReference string) (string, error)
}

type scalingGroupGetter interface {
	// GetScalingGroupImage retrieves the image currently used by a scaling group.
	GetScalingGroupImage(ctx context.Context, scalingGroupID string) (string, error)
	// GetScalingGroupName retrieves the name of a scaling group.
	GetScalingGroupName(scalingGroupID string) (string, error)
	// GetScalingGroupName retrieves the name of a scaling group as needed by the cluster-autoscaler.
	GetAutoscalingGroupName(scalingGroupID string) (string, error)
	// ListScalingGroups retrieves a list of scaling groups for the cluster.
	ListScalingGroups(ctx context.Context, uid string) (controlPlaneGroupIDs []string, workerGroupIDs []string, err error)
	// AutoscalingCloudProvider returns the cloud-provider name as used by k8s cluster-autoscaler.
	AutoscalingCloudProvider() string
}

type newScalingGroupConfig struct {
	k8sClient            client.Writer
	groupID              string
	groupName            string
	autoscalingGroupName string
	role                 updatev1alpha1.NodeRole
}