diff --git a/operators/constellation-node-operator/config/manager/manager.yaml b/operators/constellation-node-operator/config/manager/manager.yaml index 74bea8480..cd8703d93 100644 --- a/operators/constellation-node-operator/config/manager/manager.yaml +++ b/operators/constellation-node-operator/config/manager/manager.yaml @@ -53,8 +53,9 @@ spec: - mountPath: /etc/azure name: azureconfig readOnly: true - # TODO(user): Configure the resources accordingly based on the project requirements. - # More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + - mountPath: /etc/gce + name: gceconf + readOnly: true resources: limits: cpu: 500m @@ -71,6 +72,10 @@ spec: secret: secretName: azureconfig optional: true + - name: gceconf + configMap: + name: gceconf + optional: true nodeSelector: node-role.kubernetes.io/control-plane: "" imagePullSecrets: diff --git a/operators/constellation-node-operator/internal/azure/client/api.go b/operators/constellation-node-operator/internal/azure/client/api.go index 19bce1210..6091fd48f 100644 --- a/operators/constellation-node-operator/internal/azure/client/api.go +++ b/operators/constellation-node-operator/internal/azure/client/api.go @@ -30,6 +30,8 @@ type scaleSetsAPI interface { BeginDeleteInstances(ctx context.Context, resourceGroupName string, vmScaleSetName string, vmInstanceIDs armcomputev2.VirtualMachineScaleSetVMInstanceRequiredIDs, options *armcomputev2.VirtualMachineScaleSetsClientBeginDeleteInstancesOptions, ) (*runtime.Poller[armcomputev2.VirtualMachineScaleSetsClientDeleteInstancesResponse], error) + NewListPager(resourceGroupName string, options *armcomputev2.VirtualMachineScaleSetsClientListOptions, + ) *runtime.Pager[armcomputev2.VirtualMachineScaleSetsClientListResponse] } type capacityPoller interface { diff --git a/operators/constellation-node-operator/internal/azure/client/client.go b/operators/constellation-node-operator/internal/azure/client/client.go index 45b785cc0..bfd93040e 100644 --- a/operators/constellation-node-operator/internal/azure/client/client.go +++ b/operators/constellation-node-operator/internal/azure/client/client.go @@ -9,6 +9,7 @@ import ( // Client is a client for the Azure Cloud. type Client struct { + config cloudConfig scaleSetsAPI virtualMachineScaleSetVMsAPI capacityPollerGenerator func(resourceGroup, scaleSet string, wantedCapacity int64) capacityPoller @@ -37,6 +38,7 @@ func NewFromDefault(configPath string) (*Client, error) { } return &Client{ + config: *config, scaleSetsAPI: scaleSetAPI, virtualMachineScaleSetVMsAPI: virtualMachineScaleSetVMsAPI, capacityPollerGenerator: func(resourceGroup, scaleSet string, wantedCapacity int64) capacityPoller { diff --git a/operators/constellation-node-operator/internal/azure/client/client_test.go b/operators/constellation-node-operator/internal/azure/client/client_test.go index 8765826c3..d99077772 100644 --- a/operators/constellation-node-operator/internal/azure/client/client_test.go +++ b/operators/constellation-node-operator/internal/azure/client/client_test.go @@ -16,6 +16,7 @@ type stubScaleSetsAPI struct { deleteResponse armcomputev2.VirtualMachineScaleSetsClientDeleteInstancesResponse deleteErr error resultErr error + pager *stubVMSSPager } func (a *stubScaleSetsAPI) Get(ctx context.Context, resourceGroupName string, vmScaleSetName string, @@ -54,12 +55,20 @@ func (a *stubScaleSetsAPI) BeginDeleteInstances(ctx context.Context, resourceGro return poller, a.deleteErr } +func (a *stubScaleSetsAPI) NewListPager(resourceGroupName string, options *armcomputev2.VirtualMachineScaleSetsClientListOptions, +) *runtime.Pager[armcomputev2.VirtualMachineScaleSetsClientListResponse] { + return runtime.NewPager(runtime.PagingHandler[armcomputev2.VirtualMachineScaleSetsClientListResponse]{ + More: a.pager.moreFunc(), + Fetcher: a.pager.fetcherFunc(), + }) +} + type stubvirtualMachineScaleSetVMsAPI struct { scaleSetVM armcomputev2.VirtualMachineScaleSetVMsClientGetResponse getErr error instanceView armcomputev2.VirtualMachineScaleSetVMsClientGetInstanceViewResponse instanceViewErr error - pager *stubPager + pager *stubVMSSVMPager } func (a *stubvirtualMachineScaleSetVMsAPI) Get(ctx context.Context, resourceGroupName string, vmScaleSetName string, instanceID string, @@ -102,19 +111,19 @@ func (p *stubPoller[T]) Result(ctx context.Context, out *T) error { return p.resultErr } -type stubPager struct { +type stubVMSSVMPager struct { list []armcomputev2.VirtualMachineScaleSetVM fetchErr error more bool } -func (p *stubPager) moreFunc() func(armcomputev2.VirtualMachineScaleSetVMsClientListResponse) bool { +func (p *stubVMSSVMPager) moreFunc() func(armcomputev2.VirtualMachineScaleSetVMsClientListResponse) bool { return func(armcomputev2.VirtualMachineScaleSetVMsClientListResponse) bool { return p.more } } -func (p *stubPager) fetcherFunc() func(context.Context, *armcomputev2.VirtualMachineScaleSetVMsClientListResponse) (armcomputev2.VirtualMachineScaleSetVMsClientListResponse, error) { +func (p *stubVMSSVMPager) fetcherFunc() func(context.Context, *armcomputev2.VirtualMachineScaleSetVMsClientListResponse) (armcomputev2.VirtualMachineScaleSetVMsClientListResponse, error) { return func(context.Context, *armcomputev2.VirtualMachineScaleSetVMsClientListResponse) (armcomputev2.VirtualMachineScaleSetVMsClientListResponse, error) { page := make([]*armcomputev2.VirtualMachineScaleSetVM, len(p.list)) for i := range p.list { @@ -127,3 +136,29 @@ func (p *stubPager) fetcherFunc() func(context.Context, *armcomputev2.VirtualMac }, p.fetchErr } } + +type stubVMSSPager struct { + list []armcomputev2.VirtualMachineScaleSet + fetchErr error + more bool +} + +func (p *stubVMSSPager) moreFunc() func(armcomputev2.VirtualMachineScaleSetsClientListResponse) bool { + return func(armcomputev2.VirtualMachineScaleSetsClientListResponse) bool { + return p.more + } +} + +func (p *stubVMSSPager) fetcherFunc() func(context.Context, *armcomputev2.VirtualMachineScaleSetsClientListResponse) (armcomputev2.VirtualMachineScaleSetsClientListResponse, error) { + return func(context.Context, *armcomputev2.VirtualMachineScaleSetsClientListResponse) (armcomputev2.VirtualMachineScaleSetsClientListResponse, error) { + page := make([]*armcomputev2.VirtualMachineScaleSet, len(p.list)) + for i := range p.list { + page[i] = &p.list[i] + } + return armcomputev2.VirtualMachineScaleSetsClientListResponse{ + VirtualMachineScaleSetListResult: armcomputev2.VirtualMachineScaleSetListResult{ + Value: page, + }, + }, p.fetchErr + } +} diff --git a/operators/constellation-node-operator/internal/azure/client/config.go b/operators/constellation-node-operator/internal/azure/client/config.go index c39886aea..60ab547a2 100644 --- a/operators/constellation-node-operator/internal/azure/client/config.go +++ b/operators/constellation-node-operator/internal/azure/client/config.go @@ -13,6 +13,7 @@ import ( type cloudConfig struct { TenantID string `json:"tenantId,omitempty"` SubscriptionID string `json:"subscriptionId,omitempty"` + ResourceGroup string `json:"resourceGroup,omitempty"` } // loadConfig loads the cloud config from the given path. diff --git a/operators/constellation-node-operator/internal/azure/client/instanceview_test.go b/operators/constellation-node-operator/internal/azure/client/instanceview_test.go index 3d2ae4500..700d4899f 100644 --- a/operators/constellation-node-operator/internal/azure/client/instanceview_test.go +++ b/operators/constellation-node-operator/internal/azure/client/instanceview_test.go @@ -11,7 +11,7 @@ import ( ) // this state is included in most VMs but not needed -// to determine the node state as every provisioned VM also has a power state +// to determine the node state as every provisioned VM also has a power state. const provisioningStateSucceeded = "ProvisioningState/succeeded" func TestNodeStateFromStatuses(t *testing.T) { diff --git a/operators/constellation-node-operator/internal/azure/client/nodeimage_test.go b/operators/constellation-node-operator/internal/azure/client/nodeimage_test.go index bd6d3819b..29a85bfe9 100644 --- a/operators/constellation-node-operator/internal/azure/client/nodeimage_test.go +++ b/operators/constellation-node-operator/internal/azure/client/nodeimage_test.go @@ -199,7 +199,7 @@ func TestCreateNode(t *testing.T) { assert := assert.New(t) require := require.New(t) - pager := &stubPager{ + pager := &stubVMSSVMPager{ list: tc.preexistingVMs, fetchErr: tc.fetchErr, } @@ -325,7 +325,7 @@ func TestCapacityPollingHandler(t *testing.T) { assert.Error(handler.Poll(context.Background())) // let Poll finish - handler.scaleSetsAPI.(*stubScaleSetsAPI).scaleSet.SKU = &armcomputev2.SKU{Capacity: to.Ptr(int64(wantCapacity))} + handler.scaleSetsAPI.(*stubScaleSetsAPI).scaleSet.SKU = &armcomputev2.SKU{Capacity: to.Ptr(wantCapacity)} assert.NoError(handler.Poll(context.Background())) assert.True(handler.Done()) assert.NoError(handler.Result(context.Background(), &gotCapacity)) diff --git a/operators/constellation-node-operator/internal/azure/client/scaleset.go b/operators/constellation-node-operator/internal/azure/client/scaleset.go new file mode 100644 index 000000000..d82991af7 --- /dev/null +++ b/operators/constellation-node-operator/internal/azure/client/scaleset.go @@ -0,0 +1,26 @@ +package client + +import ( + "context" + "fmt" +) + +// getScaleSets retrieves the IDs of all scale sets of a resource group. +func (c *Client) getScaleSets(ctx context.Context) ([]string, error) { + pager := c.scaleSetsAPI.NewListPager(c.config.ResourceGroup, nil) + var scaleSets []string + + for pager.More() { + page, err := pager.NextPage(ctx) + if err != nil { + return nil, fmt.Errorf("paging scale sets: %w", err) + } + for _, scaleSet := range page.Value { + if scaleSet == nil || scaleSet.ID == nil { + continue + } + scaleSets = append(scaleSets, *scaleSet.ID) + } + } + return scaleSets, nil +} diff --git a/operators/constellation-node-operator/internal/azure/client/scaleset_test.go b/operators/constellation-node-operator/internal/azure/client/scaleset_test.go new file mode 100644 index 000000000..58a9d7b95 --- /dev/null +++ b/operators/constellation-node-operator/internal/azure/client/scaleset_test.go @@ -0,0 +1,56 @@ +package client + +import ( + "context" + "errors" + "testing" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore/to" + armcomputev2 "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v2" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestGetScaleSets(t *testing.T) { + testCases := map[string]struct { + scaleSet armcomputev2.VirtualMachineScaleSet + fetchPageErr error + wantScaleSets []string + wantErr bool + }{ + "fetching scale sets works": { + scaleSet: armcomputev2.VirtualMachineScaleSet{ + ID: to.Ptr("/subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/virtualMachineScaleSets/scale-set-name"), + }, + wantScaleSets: []string{"/subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/virtualMachineScaleSets/scale-set-name"}, + }, + "fetching scale sets fails": { + fetchPageErr: errors.New("fetch page error"), + wantErr: true, + }, + "scale set is invalid": {}, + } + + for name, tc := range testCases { + t.Run(name, func(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + + client := Client{ + scaleSetsAPI: &stubScaleSetsAPI{ + pager: &stubVMSSPager{ + list: []armcomputev2.VirtualMachineScaleSet{tc.scaleSet}, + fetchErr: tc.fetchPageErr, + }, + }, + } + gotScaleSets, err := client.getScaleSets(context.Background()) + if tc.wantErr { + assert.Error(err) + return + } + require.NoError(err) + assert.ElementsMatch(tc.wantScaleSets, gotScaleSets) + }) + } +} diff --git a/operators/constellation-node-operator/internal/azure/client/scalinggroup.go b/operators/constellation-node-operator/internal/azure/client/scalinggroup.go index d177546dd..bad123d73 100644 --- a/operators/constellation-node-operator/internal/azure/client/scalinggroup.go +++ b/operators/constellation-node-operator/internal/azure/client/scalinggroup.go @@ -3,6 +3,7 @@ package client import ( "context" "fmt" + "strings" "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v2" ) @@ -52,3 +53,32 @@ func (c *Client) SetScalingGroupImage(ctx context.Context, scalingGroupID, image } return nil } + +// GetScalingGroupName retrieves the name of a scaling group. +func (c *Client) GetScalingGroupName(ctx context.Context, scalingGroupID string) (string, error) { + _, _, scaleSet, err := splitVMSSID(scalingGroupID) + if err != nil { + return "", fmt.Errorf("getting scaling group name: %w", err) + } + return strings.ToLower(scaleSet), nil +} + +// ListScalingGroups retrieves a list of scaling groups for the cluster. +func (c *Client) ListScalingGroups(ctx context.Context, uid string) (controlPlaneGroupIDs []string, workerGroupIDs []string, err error) { + scaleSetIDs, err := c.getScaleSets(ctx) + if err != nil { + return nil, nil, fmt.Errorf("listing scaling groups: %w", err) + } + for _, scaleSetID := range scaleSetIDs { + _, _, scaleSet, err := splitVMSSID(scaleSetID) + if err != nil { + return nil, nil, fmt.Errorf("getting scaling group name: %w", err) + } + if scaleSet == "constellation-scale-set-controlplanes-"+uid { + controlPlaneGroupIDs = append(controlPlaneGroupIDs, scaleSetID) + } else if strings.HasPrefix(scaleSet, "constellation-scale-set-workers-"+uid) { + workerGroupIDs = append(workerGroupIDs, scaleSetID) + } + } + return controlPlaneGroupIDs, workerGroupIDs, nil +} diff --git a/operators/constellation-node-operator/internal/azure/client/scalinggroup_test.go b/operators/constellation-node-operator/internal/azure/client/scalinggroup_test.go index fcf1ad1eb..b1c0f8de4 100644 --- a/operators/constellation-node-operator/internal/azure/client/scalinggroup_test.go +++ b/operators/constellation-node-operator/internal/azure/client/scalinggroup_test.go @@ -123,3 +123,97 @@ func TestSetScalingGroupImage(t *testing.T) { }) } } + +func TestGetScalingGroupName(t *testing.T) { + testCases := map[string]struct { + scalingGroupID string + wantName string + wantErr bool + }{ + "getting name works": { + scalingGroupID: "/subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/virtualMachineScaleSets/scale-set-name", + wantName: "scale-set-name", + }, + "uppercase name is lowercased": { + scalingGroupID: "/subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/virtualMachineScaleSets/SCALE-SET-NAME", + wantName: "scale-set-name", + }, + "splitting scalingGroupID fails": { + scalingGroupID: "invalid", + wantErr: true, + }, + } + + for name, tc := range testCases { + t.Run(name, func(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + + client := Client{} + gotName, err := client.GetScalingGroupName(context.Background(), tc.scalingGroupID) + if tc.wantErr { + assert.Error(err) + return + } + require.NoError(err) + assert.Equal(tc.wantName, gotName) + }) + } +} + +func TestListScalingGroups(t *testing.T) { + testCases := map[string]struct { + scaleSet armcomputev2.VirtualMachineScaleSet + fetchPageErr error + wantControlPlanes []string + wantWorkers []string + wantErr bool + }{ + "listing control-plane works": { + scaleSet: armcomputev2.VirtualMachineScaleSet{ + ID: to.Ptr("/subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/virtualMachineScaleSets/constellation-scale-set-controlplanes-uid"), + }, + wantControlPlanes: []string{"/subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/virtualMachineScaleSets/constellation-scale-set-controlplanes-uid"}, + }, + "listing worker works": { + scaleSet: armcomputev2.VirtualMachineScaleSet{ + ID: to.Ptr("/subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/virtualMachineScaleSets/constellation-scale-set-workers-uid"), + }, + wantWorkers: []string{"/subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/virtualMachineScaleSets/constellation-scale-set-workers-uid"}, + }, + "listing other works": { + scaleSet: armcomputev2.VirtualMachineScaleSet{ + ID: to.Ptr("/subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/virtualMachineScaleSets/other"), + }, + }, + "fetching scale sets fails": { + fetchPageErr: errors.New("fetch page error"), + wantErr: true, + }, + "scale set is invalid": {}, + } + + for name, tc := range testCases { + t.Run(name, func(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + + client := Client{ + scaleSetsAPI: &stubScaleSetsAPI{ + pager: &stubVMSSPager{ + list: []armcomputev2.VirtualMachineScaleSet{tc.scaleSet}, + fetchErr: tc.fetchPageErr, + }, + }, + } + gotControlPlanes, gotWorkers, err := client.ListScalingGroups(context.Background(), "uid") + if tc.wantErr { + assert.Error(err) + return + } + require.NoError(err) + assert.ElementsMatch(tc.wantControlPlanes, gotControlPlanes) + assert.ElementsMatch(tc.wantWorkers, gotWorkers) + }) + } +} diff --git a/operators/constellation-node-operator/internal/azure/client/vmss.go b/operators/constellation-node-operator/internal/azure/client/vmss.go index 48a9445b0..745c23994 100644 --- a/operators/constellation-node-operator/internal/azure/client/vmss.go +++ b/operators/constellation-node-operator/internal/azure/client/vmss.go @@ -9,7 +9,7 @@ import ( var vmssIDRegexp = regexp.MustCompile(`^/subscriptions/([^/]+)/resourceGroups/([^/]+)/providers/Microsoft.Compute/virtualMachineScaleSets/([^/]+)$`) // joinVMSSID joins scale set parameters to generate a virtual machine scale set (VMSS) ID. -// Format: /subscriptions//resourceGroups//providers/Microsoft.Compute/virtualMachineScaleSets/ +// Format: /subscriptions//resourceGroups//providers/Microsoft.Compute/virtualMachineScaleSets/ . func joinVMSSID(subscriptionID, resourceGroup, scaleSet string) string { return fmt.Sprintf("/subscriptions/%s/resourceGroups/%s/providers/Microsoft.Compute/virtualMachineScaleSets/%s", subscriptionID, resourceGroup, scaleSet) } diff --git a/operators/constellation-node-operator/internal/constants/constants.go b/operators/constellation-node-operator/internal/constants/constants.go new file mode 100644 index 000000000..6697b439a --- /dev/null +++ b/operators/constellation-node-operator/internal/constants/constants.go @@ -0,0 +1,8 @@ +package constants + +const ( + AutoscalingStrategyResourceName = "autoscalingstrategy" + NodeImageResourceName = "constellation-coreos" + ControlPlaneScalingGroupResourceName = "scalinggroup-controlplane" + WorkerScalingGroupResourceName = "scalinggroup-worker" +) diff --git a/operators/constellation-node-operator/internal/deploy/deploy.go b/operators/constellation-node-operator/internal/deploy/deploy.go new file mode 100644 index 000000000..59a2f5b9b --- /dev/null +++ b/operators/constellation-node-operator/internal/deploy/deploy.go @@ -0,0 +1,122 @@ +// Package deploy provides functions to deploy initial resources for the node operator. +package deploy + +import ( + "context" + "errors" + "fmt" + + updatev1alpha1 "github.com/edgelesssys/constellation/operators/constellation-node-operator/api/v1alpha1" + "github.com/edgelesssys/constellation/operators/constellation-node-operator/internal/constants" + k8sErrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +// InitialResources creates the initial resources for the node operator. +func InitialResources(ctx context.Context, k8sClient client.Writer, scalingGroupGetter scalingGroupGetter, uid string) error { + controlPlaneGroupIDs, workerGroupIDs, err := scalingGroupGetter.ListScalingGroups(ctx, uid) + if err != nil { + return fmt.Errorf("listing scaling groups: %w", err) + } + if len(controlPlaneGroupIDs) == 0 { + return errors.New("determining initial node image: no control plane scaling group found") + } + if len(workerGroupIDs) == 0 { + return errors.New("determining initial node image: no worker scaling group found") + } + + if err := createAutoscalingStrategy(ctx, k8sClient); err != nil { + return fmt.Errorf("creating initial autoscaling strategy: %w", err) + } + imageReference, err := scalingGroupGetter.GetScalingGroupImage(ctx, controlPlaneGroupIDs[0]) + if err != nil { + return fmt.Errorf("determining initial node image: %w", err) + } + if err := createNodeImage(ctx, k8sClient, imageReference); err != nil { + return fmt.Errorf("creating initial node image %q: %w", imageReference, err) + } + for _, groupID := range controlPlaneGroupIDs { + groupName, err := scalingGroupGetter.GetScalingGroupName(ctx, groupID) + if err != nil { + return fmt.Errorf("determining scaling group name of %q: %w", groupID, err) + } + if err := createScalingGroup(ctx, k8sClient, groupID, groupName, false); err != nil { + return fmt.Errorf("creating initial control plane scaling group: %w", err) + } + } + for _, groupID := range workerGroupIDs { + groupName, err := scalingGroupGetter.GetScalingGroupName(ctx, groupID) + if err != nil { + return fmt.Errorf("determining scaling group name of %q: %w", groupID, err) + } + if err := createScalingGroup(ctx, k8sClient, groupID, groupName, true); err != nil { + return fmt.Errorf("creating initial worker scaling group: %w", err) + } + } + return nil +} + +// createAutoscalingStrategy creates the autoscaling strategy resource if it does not exist yet. +func createAutoscalingStrategy(ctx context.Context, k8sClient client.Writer) error { + err := k8sClient.Create(ctx, &updatev1alpha1.AutoscalingStrategy{ + TypeMeta: metav1.TypeMeta{APIVersion: "update.edgeless.systems/v1alpha1", Kind: "AutoscalingStrategy"}, + ObjectMeta: metav1.ObjectMeta{ + Name: constants.AutoscalingStrategyResourceName, + }, + Spec: updatev1alpha1.AutoscalingStrategySpec{ + Enabled: true, + DeploymentName: "constellation-cluster-autoscaler", + DeploymentNamespace: "kube-system", + }, + }) + if k8sErrors.IsAlreadyExists(err) { + return nil + } + return err +} + +// createNodeImage creates the initial nodeimage resource if it does not exist yet. +func createNodeImage(ctx context.Context, k8sClient client.Writer, imageReference string) error { + err := k8sClient.Create(ctx, &updatev1alpha1.NodeImage{ + TypeMeta: metav1.TypeMeta{APIVersion: "update.edgeless.systems/v1alpha1", Kind: "NodeImage"}, + ObjectMeta: metav1.ObjectMeta{ + Name: constants.NodeImageResourceName, + }, + Spec: updatev1alpha1.NodeImageSpec{ + ImageReference: imageReference, + }, + }) + if k8sErrors.IsAlreadyExists(err) { + return nil + } + return err +} + +// createScalingGroup creates an initial scaling group resource if it does not exist yet. +func createScalingGroup(ctx context.Context, k8sClient client.Writer, groupID, groupName string, autoscaling bool) error { + err := k8sClient.Create(ctx, &updatev1alpha1.ScalingGroup{ + TypeMeta: metav1.TypeMeta{APIVersion: "update.edgeless.systems/v1alpha1", Kind: "ScalingGroup"}, + ObjectMeta: metav1.ObjectMeta{ + Name: groupName, + }, + Spec: updatev1alpha1.ScalingGroupSpec{ + NodeImage: constants.NodeImageResourceName, + GroupID: groupID, + Autoscaling: autoscaling, + }, + }) + if k8sErrors.IsAlreadyExists(err) { + return nil + } + return err +} + +type scalingGroupGetter interface { + // GetScalingGroupImage retrieves the image currently used by a scaling group. + GetScalingGroupImage(ctx context.Context, scalingGroupID string) (string, error) + // GetScalingGroupName retrieves the name of a scaling group. + GetScalingGroupName(ctx context.Context, scalingGroupID string) (string, error) + // ListScalingGroups retrieves a list of scaling groups for the cluster. + ListScalingGroups(ctx context.Context, uid string) (controlPlaneGroupIDs []string, workerGroupIDs []string, err error) +} diff --git a/operators/constellation-node-operator/internal/deploy/deploy_test.go b/operators/constellation-node-operator/internal/deploy/deploy_test.go new file mode 100644 index 000000000..65272c48b --- /dev/null +++ b/operators/constellation-node-operator/internal/deploy/deploy_test.go @@ -0,0 +1,317 @@ +package deploy + +import ( + "context" + "errors" + "testing" + + updatev1alpha1 "github.com/edgelesssys/constellation/operators/constellation-node-operator/api/v1alpha1" + "github.com/edgelesssys/constellation/operators/constellation-node-operator/internal/constants" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + k8sErrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +func TestInitialResources(t *testing.T) { + testCases := map[string]struct { + items []scalingGroupStoreItem + imageErr error + nameErr error + listErr error + createErr error + wantResources int + wantErr bool + }{ + "creating initial resources works": { + items: []scalingGroupStoreItem{ + {groupID: "control-plane", image: "image-1", name: "control-plane", isControlPlane: true}, + {groupID: "worker", image: "image-1", name: "worker"}, + }, + wantResources: 4, + }, + "missing control planes": { + items: []scalingGroupStoreItem{ + {groupID: "worker", image: "image-1", name: "worker"}, + }, + wantErr: true, + }, + "missing workers": { + items: []scalingGroupStoreItem{ + {groupID: "control-plane", image: "image-1", name: "control-plane", isControlPlane: true}, + }, + wantErr: true, + }, + "listing groups fails": { + listErr: errors.New("list failed"), + wantErr: true, + }, + "creating resources fails": { + items: []scalingGroupStoreItem{ + {groupID: "control-plane", image: "image-1", name: "control-plane", isControlPlane: true}, + {groupID: "worker", image: "image-1", name: "worker"}, + }, + createErr: errors.New("create failed"), + wantErr: true, + }, + "getting image fails": { + items: []scalingGroupStoreItem{ + {groupID: "control-plane", image: "image-1", name: "control-plane", isControlPlane: true}, + {groupID: "worker", image: "image-1", name: "worker"}, + }, + imageErr: errors.New("getting image failed"), + wantErr: true, + }, + "getting name fails": { + items: []scalingGroupStoreItem{ + {groupID: "control-plane", image: "image-1", name: "control-plane", isControlPlane: true}, + {groupID: "worker", image: "image-1", name: "worker"}, + }, + nameErr: errors.New("getting name failed"), + wantErr: true, + }, + } + + for name, tc := range testCases { + t.Run(name, func(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + + k8sClient := &stubK8sClient{createErr: tc.createErr} + scalingGroupGetter := newScalingGroupGetter(tc.items, tc.imageErr, tc.nameErr, tc.listErr) + err := InitialResources(context.Background(), k8sClient, scalingGroupGetter, "uid") + if tc.wantErr { + assert.Error(err) + return + } + require.NoError(err) + assert.Len(k8sClient.createdObjects, tc.wantResources) + }) + } +} + +func TestCreateAutoscalingStrategy(t *testing.T) { + testCases := map[string]struct { + createErr error + wantStrategy *updatev1alpha1.AutoscalingStrategy + wantErr bool + }{ + "create works": { + wantStrategy: &updatev1alpha1.AutoscalingStrategy{ + TypeMeta: metav1.TypeMeta{APIVersion: "update.edgeless.systems/v1alpha1", Kind: "AutoscalingStrategy"}, + ObjectMeta: metav1.ObjectMeta{ + Name: constants.AutoscalingStrategyResourceName, + }, + Spec: updatev1alpha1.AutoscalingStrategySpec{ + Enabled: true, + DeploymentName: "constellation-cluster-autoscaler", + DeploymentNamespace: "kube-system", + }, + }, + }, + "create fails": { + createErr: errors.New("create failed"), + wantErr: true, + }, + "strategy exists": { + createErr: k8sErrors.NewAlreadyExists(schema.GroupResource{}, constants.AutoscalingStrategyResourceName), + wantStrategy: &updatev1alpha1.AutoscalingStrategy{ + TypeMeta: metav1.TypeMeta{APIVersion: "update.edgeless.systems/v1alpha1", Kind: "AutoscalingStrategy"}, + ObjectMeta: metav1.ObjectMeta{ + Name: constants.AutoscalingStrategyResourceName, + }, + Spec: updatev1alpha1.AutoscalingStrategySpec{ + Enabled: true, + DeploymentName: "constellation-cluster-autoscaler", + DeploymentNamespace: "kube-system", + }, + }, + }, + } + + for name, tc := range testCases { + t.Run(name, func(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + + k8sClient := &stubK8sClient{createErr: tc.createErr} + err := createAutoscalingStrategy(context.Background(), k8sClient) + if tc.wantErr { + assert.Error(err) + return + } + require.NoError(err) + assert.Len(k8sClient.createdObjects, 1) + assert.Equal(tc.wantStrategy, k8sClient.createdObjects[0]) + }) + } +} + +func TestCreateNodeImage(t *testing.T) { + testCases := map[string]struct { + createErr error + wantNodeImage *updatev1alpha1.NodeImage + wantErr bool + }{ + "create works": { + wantNodeImage: &updatev1alpha1.NodeImage{ + TypeMeta: metav1.TypeMeta{APIVersion: "update.edgeless.systems/v1alpha1", Kind: "NodeImage"}, + ObjectMeta: metav1.ObjectMeta{ + Name: constants.NodeImageResourceName, + }, + Spec: updatev1alpha1.NodeImageSpec{ + ImageReference: "image-reference", + }, + }, + }, + "create fails": { + createErr: errors.New("create failed"), + wantErr: true, + }, + "image exists": { + createErr: k8sErrors.NewAlreadyExists(schema.GroupResource{}, constants.AutoscalingStrategyResourceName), + wantNodeImage: &updatev1alpha1.NodeImage{ + TypeMeta: metav1.TypeMeta{APIVersion: "update.edgeless.systems/v1alpha1", Kind: "NodeImage"}, + ObjectMeta: metav1.ObjectMeta{ + Name: constants.NodeImageResourceName, + }, + Spec: updatev1alpha1.NodeImageSpec{ + ImageReference: "image-reference", + }, + }, + }, + } + + for name, tc := range testCases { + t.Run(name, func(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + + k8sClient := &stubK8sClient{createErr: tc.createErr} + err := createNodeImage(context.Background(), k8sClient, "image-reference") + if tc.wantErr { + assert.Error(err) + return + } + require.NoError(err) + assert.Len(k8sClient.createdObjects, 1) + assert.Equal(tc.wantNodeImage, k8sClient.createdObjects[0]) + }) + } +} + +func TestCreateScalingGroup(t *testing.T) { + testCases := map[string]struct { + createErr error + wantScalingGroup *updatev1alpha1.ScalingGroup + wantErr bool + }{ + "create works": { + wantScalingGroup: &updatev1alpha1.ScalingGroup{ + TypeMeta: metav1.TypeMeta{APIVersion: "update.edgeless.systems/v1alpha1", Kind: "ScalingGroup"}, + ObjectMeta: metav1.ObjectMeta{ + Name: "group-name", + }, + Spec: updatev1alpha1.ScalingGroupSpec{ + NodeImage: constants.NodeImageResourceName, + GroupID: "group-id", + Autoscaling: true, + }, + }, + }, + "create fails": { + createErr: errors.New("create failed"), + wantErr: true, + }, + "image exists": { + createErr: k8sErrors.NewAlreadyExists(schema.GroupResource{}, constants.AutoscalingStrategyResourceName), + wantScalingGroup: &updatev1alpha1.ScalingGroup{ + TypeMeta: metav1.TypeMeta{APIVersion: "update.edgeless.systems/v1alpha1", Kind: "ScalingGroup"}, + ObjectMeta: metav1.ObjectMeta{ + Name: "group-name", + }, + Spec: updatev1alpha1.ScalingGroupSpec{ + NodeImage: constants.NodeImageResourceName, + GroupID: "group-id", + Autoscaling: true, + }, + }, + }, + } + + for name, tc := range testCases { + t.Run(name, func(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + + k8sClient := &stubK8sClient{createErr: tc.createErr} + err := createScalingGroup(context.Background(), k8sClient, "group-id", "group-name", true) + if tc.wantErr { + assert.Error(err) + return + } + require.NoError(err) + assert.Len(k8sClient.createdObjects, 1) + assert.Equal(tc.wantScalingGroup, k8sClient.createdObjects[0]) + }) + } +} + +type stubK8sClient struct { + createdObjects []client.Object + createErr error + client.Writer +} + +func (s *stubK8sClient) Create(ctx context.Context, obj client.Object, opts ...client.CreateOption) error { + s.createdObjects = append(s.createdObjects, obj) + return s.createErr +} + +type stubScalingGroupGetter struct { + store map[string]scalingGroupStoreItem + imageErr error + nameErr error + listErr error +} + +func newScalingGroupGetter(items []scalingGroupStoreItem, imageErr, nameErr, listErr error) *stubScalingGroupGetter { + store := make(map[string]scalingGroupStoreItem) + for _, item := range items { + store[item.groupID] = item + } + return &stubScalingGroupGetter{ + store: store, + imageErr: imageErr, + nameErr: nameErr, + listErr: listErr, + } +} + +func (g *stubScalingGroupGetter) GetScalingGroupImage(ctx context.Context, scalingGroupID string) (string, error) { + return g.store[scalingGroupID].image, g.imageErr +} + +func (g *stubScalingGroupGetter) GetScalingGroupName(ctx context.Context, scalingGroupID string) (string, error) { + return g.store[scalingGroupID].name, g.nameErr +} + +func (g *stubScalingGroupGetter) ListScalingGroups(ctx context.Context, uid string) (controlPlaneGroupIDs []string, workerGroupIDs []string, err error) { + for _, item := range g.store { + if item.isControlPlane { + controlPlaneGroupIDs = append(controlPlaneGroupIDs, item.groupID) + } else { + workerGroupIDs = append(workerGroupIDs, item.groupID) + } + } + return controlPlaneGroupIDs, workerGroupIDs, g.listErr +} + +type scalingGroupStoreItem struct { + groupID string + name string + image string + isControlPlane bool +} diff --git a/operators/constellation-node-operator/internal/gcp/client/api.go b/operators/constellation-node-operator/internal/gcp/client/api.go index e01a7fe00..dfade861e 100644 --- a/operators/constellation-node-operator/internal/gcp/client/api.go +++ b/operators/constellation-node-operator/internal/gcp/client/api.go @@ -3,6 +3,7 @@ package client import ( "context" + compute "cloud.google.com/go/compute/apiv1" "github.com/googleapis/gax-go/v2" computepb "google.golang.org/genproto/googleapis/cloud/compute/v1" ) @@ -27,6 +28,8 @@ type instanceGroupManagersAPI interface { Close() error Get(ctx context.Context, req *computepb.GetInstanceGroupManagerRequest, opts ...gax.CallOption) (*computepb.InstanceGroupManager, error) + AggregatedList(ctx context.Context, req *computepb.AggregatedListInstanceGroupManagersRequest, + opts ...gax.CallOption) InstanceGroupManagerScopedListIterator SetInstanceTemplate(ctx context.Context, req *computepb.SetInstanceTemplateInstanceGroupManagerRequest, opts ...gax.CallOption) (Operation, error) CreateInstances(ctx context.Context, req *computepb.CreateInstancesInstanceGroupManagerRequest, @@ -47,6 +50,10 @@ type Operation interface { Wait(ctx context.Context, opts ...gax.CallOption) error } +type InstanceGroupManagerScopedListIterator interface { + Next() (compute.InstanceGroupManagersScopedListPair, error) +} + type prng interface { // Intn returns, as an int, a non-negative pseudo-random number in the half-open interval [0,n). It panics if n <= 0. Intn(n int) int diff --git a/operators/constellation-node-operator/internal/gcp/client/client.go b/operators/constellation-node-operator/internal/gcp/client/client.go index 4b44a19cd..d2f3f8993 100644 --- a/operators/constellation-node-operator/internal/gcp/client/client.go +++ b/operators/constellation-node-operator/internal/gcp/client/client.go @@ -6,11 +6,13 @@ import ( "time" compute "cloud.google.com/go/compute/apiv1" + "github.com/spf13/afero" "go.uber.org/multierr" ) // Client is a client for the Google Compute Engine. type Client struct { + projectID string instanceAPI instanceTemplateAPI instanceGroupManagersAPI @@ -20,7 +22,12 @@ type Client struct { } // New creates a new client for the Google Compute Engine. -func New(ctx context.Context) (*Client, error) { +func New(ctx context.Context, configPath string) (*Client, error) { + projectID, err := loadProjectID(afero.NewOsFs(), configPath) + if err != nil { + return nil, err + } + var closers []closer insAPI, err := compute.NewInstancesRESTClient(ctx) if err != nil { @@ -44,8 +51,8 @@ func New(ctx context.Context) (*Client, error) { _ = closeAll(closers) return nil, err } - return &Client{ + projectID: projectID, instanceAPI: insAPI, instanceTemplateAPI: &instanceTemplateClient{templAPI}, instanceGroupManagersAPI: &instanceGroupManagersClient{groupAPI}, diff --git a/operators/constellation-node-operator/internal/gcp/client/client_test.go b/operators/constellation-node-operator/internal/gcp/client/client_test.go index ec5a1b396..12e4bc998 100644 --- a/operators/constellation-node-operator/internal/gcp/client/client_test.go +++ b/operators/constellation-node-operator/internal/gcp/client/client_test.go @@ -3,7 +3,9 @@ package client import ( "context" + compute "cloud.google.com/go/compute/apiv1" "github.com/googleapis/gax-go/v2" + "google.golang.org/api/iterator" computepb "google.golang.org/genproto/googleapis/cloud/compute/v1" "google.golang.org/protobuf/proto" ) @@ -63,6 +65,7 @@ func (a stubInstanceTemplateAPI) Insert(ctx context.Context, req *computepb.Inse type stubInstanceGroupManagersAPI struct { instanceGroupManager *computepb.InstanceGroupManager getErr error + aggregatedListErr error setInstanceTemplateErr error createInstancesErr error deleteInstancesErr error @@ -78,6 +81,24 @@ func (a stubInstanceGroupManagersAPI) Get(ctx context.Context, req *computepb.Ge return a.instanceGroupManager, a.getErr } +func (a stubInstanceGroupManagersAPI) AggregatedList(ctx context.Context, req *computepb.AggregatedListInstanceGroupManagersRequest, + opts ...gax.CallOption, +) InstanceGroupManagerScopedListIterator { + return &stubInstanceGroupManagerScopedListIterator{ + pairs: []compute.InstanceGroupManagersScopedListPair{ + { + Key: "key", + Value: &computepb.InstanceGroupManagersScopedList{ + InstanceGroupManagers: []*computepb.InstanceGroupManager{ + a.instanceGroupManager, + }, + }, + }, + }, + nextErr: a.aggregatedListErr, + } +} + func (a stubInstanceGroupManagersAPI) SetInstanceTemplate(ctx context.Context, req *computepb.SetInstanceTemplateInstanceGroupManagerRequest, opts ...gax.CallOption, ) (Operation, error) { @@ -141,3 +162,22 @@ func (o *stubOperation) Done() bool { func (o *stubOperation) Wait(ctx context.Context, opts ...gax.CallOption) error { return nil } + +type stubInstanceGroupManagerScopedListIterator struct { + pairs []compute.InstanceGroupManagersScopedListPair + nextErr error + + internalCounter int +} + +func (i *stubInstanceGroupManagerScopedListIterator) Next() (compute.InstanceGroupManagersScopedListPair, error) { + if i.nextErr != nil { + return compute.InstanceGroupManagersScopedListPair{}, i.nextErr + } + if i.internalCounter >= len(i.pairs) { + return compute.InstanceGroupManagersScopedListPair{}, iterator.Done + } + pair := i.pairs[i.internalCounter] + i.internalCounter++ + return pair, nil +} diff --git a/operators/constellation-node-operator/internal/gcp/client/config.go b/operators/constellation-node-operator/internal/gcp/client/config.go new file mode 100644 index 000000000..f2cbe6cfb --- /dev/null +++ b/operators/constellation-node-operator/internal/gcp/client/config.go @@ -0,0 +1,24 @@ +package client + +import ( + "errors" + "regexp" + + "github.com/spf13/afero" +) + +var projectIDRegex = regexp.MustCompile(`(?m)^project-id = (.*)$`) + +// loadProjectID loads the project id from the gce config file. +func loadProjectID(fs afero.Fs, path string) (string, error) { + rawConfig, err := afero.ReadFile(fs, path) + if err != nil { + return "", err + } + // find project-id line + matches := projectIDRegex.FindStringSubmatch(string(rawConfig)) + if len(matches) != 2 { + return "", errors.New("invalid config: project-id not found") + } + return matches[1], nil +} diff --git a/operators/constellation-node-operator/internal/gcp/client/config_test.go b/operators/constellation-node-operator/internal/gcp/client/config_test.go new file mode 100644 index 000000000..82dc88d9f --- /dev/null +++ b/operators/constellation-node-operator/internal/gcp/client/config_test.go @@ -0,0 +1,53 @@ +package client + +import ( + "testing" + + "github.com/spf13/afero" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestLoadProjectID(t *testing.T) { + testCases := map[string]struct { + rawConfig string + skipWrite bool + wantProjectID string + wantErr bool + }{ + "valid config": { + rawConfig: `project-id = project-id`, + wantProjectID: "project-id", + }, + "invalid config": { + rawConfig: `x = y`, + wantErr: true, + }, + "config is empty": { + wantErr: true, + }, + "config does not exist": { + skipWrite: true, + wantErr: true, + }, + } + + for name, tc := range testCases { + t.Run(name, func(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + + fs := afero.NewMemMapFs() + if !tc.skipWrite { + require.NoError(afero.WriteFile(fs, "gce.conf", []byte(tc.rawConfig), 0o644)) + } + gotProjectID, err := loadProjectID(fs, "gce.conf") + if tc.wantErr { + assert.Error(err) + return + } + require.NoError(err) + assert.Equal(tc.wantProjectID, gotProjectID) + }) + } +} diff --git a/operators/constellation-node-operator/internal/gcp/client/gcpwrappers.go b/operators/constellation-node-operator/internal/gcp/client/gcpwrappers.go index a72cfcd1c..b70bf9c91 100644 --- a/operators/constellation-node-operator/internal/gcp/client/gcpwrappers.go +++ b/operators/constellation-node-operator/internal/gcp/client/gcpwrappers.go @@ -42,6 +42,12 @@ func (c *instanceGroupManagersClient) Get(ctx context.Context, req *computepb.Ge return c.InstanceGroupManagersClient.Get(ctx, req, opts...) } +func (c *instanceGroupManagersClient) AggregatedList(ctx context.Context, req *computepb.AggregatedListInstanceGroupManagersRequest, + opts ...gax.CallOption, +) InstanceGroupManagerScopedListIterator { + return c.InstanceGroupManagersClient.AggregatedList(ctx, req, opts...) +} + func (c *instanceGroupManagersClient) SetInstanceTemplate(ctx context.Context, req *computepb.SetInstanceTemplateInstanceGroupManagerRequest, opts ...gax.CallOption, ) (Operation, error) { diff --git a/operators/constellation-node-operator/internal/gcp/client/instancegroupmanagers.go b/operators/constellation-node-operator/internal/gcp/client/instancegroupmanagers.go index 9cbc06610..c1baba59a 100644 --- a/operators/constellation-node-operator/internal/gcp/client/instancegroupmanagers.go +++ b/operators/constellation-node-operator/internal/gcp/client/instancegroupmanagers.go @@ -5,7 +5,11 @@ import ( "regexp" ) -var instanceGroupIDRegex = regexp.MustCompile(`^projects/([^/]+)/zones/([^/]+)/instanceGroupManagers/([^/]+)$`) +var ( + instanceGroupIDRegex = regexp.MustCompile(`^projects/([^/]+)/zones/([^/]+)/instanceGroupManagers/([^/]+)$`) + controlPlaneInstanceGroupNameRegex = regexp.MustCompile(`^(.*)control-plane(.*)$`) + workerInstanceGroupNameRegex = regexp.MustCompile(`^(.*)worker(.*)$`) +) // splitInstanceGroupID splits an instance group ID into core components. func splitInstanceGroupID(instanceGroupID string) (project, zone, instanceGroup string, err error) { @@ -16,8 +20,18 @@ func splitInstanceGroupID(instanceGroupID string) (project, zone, instanceGroup return matches[1], matches[2], matches[3], nil } +// isControlPlaneInstanceGroup returns true if the instance group is a control plane instance group. +func isControlPlaneInstanceGroup(instanceGroupName string) bool { + return controlPlaneInstanceGroupNameRegex.MatchString(instanceGroupName) +} + +// isWorkerInstanceGroup returns true if the instance group is a worker instance group. +func isWorkerInstanceGroup(instanceGroupName string) bool { + return workerInstanceGroupNameRegex.MatchString(instanceGroupName) +} + // generateInstanceName generates a random instance name. -func generateInstanceName(baseInstanceName string, random prng) (string, error) { +func generateInstanceName(baseInstanceName string, random prng) string { letters := []byte("abcdefghijklmnopqrstuvwxyz0123456789") const uidLen = 4 uid := make([]byte, 0, uidLen) @@ -25,5 +39,5 @@ func generateInstanceName(baseInstanceName string, random prng) (string, error) n := random.Intn(len(letters)) uid = append(uid, letters[n]) } - return baseInstanceName + "-" + string(uid), nil + return baseInstanceName + "-" + string(uid) } diff --git a/operators/constellation-node-operator/internal/gcp/client/instancegroupmanagers_test.go b/operators/constellation-node-operator/internal/gcp/client/instancegroupmanagers_test.go index 3b202c401..e5a7e33a5 100644 --- a/operators/constellation-node-operator/internal/gcp/client/instancegroupmanagers_test.go +++ b/operators/constellation-node-operator/internal/gcp/client/instancegroupmanagers_test.go @@ -49,21 +49,17 @@ func TestSplitInstanceGroupID(t *testing.T) { func TestGenerateInstanceName(t *testing.T) { assert := assert.New(t) - require := require.New(t) baseInstanceName := "base" - gotInstanceName, err := generateInstanceName(baseInstanceName, &stubRng{result: 0}) - require.NoError(err) + gotInstanceName := generateInstanceName(baseInstanceName, &stubRng{result: 0}) assert.Equal("base-aaaa", gotInstanceName) } func TestGenerateInstanceNameRandomTest(t *testing.T) { assert := assert.New(t) - require := require.New(t) instanceNameRegexp := regexp.MustCompile(`^base-[0-9a-z]{4}$`) baseInstanceName := "base" random := rand.New(rand.NewSource(int64(time.Now().Nanosecond()))) - gotInstanceName, err := generateInstanceName(baseInstanceName, random) - require.NoError(err) + gotInstanceName := generateInstanceName(baseInstanceName, random) assert.Regexp(instanceNameRegexp, gotInstanceName) } diff --git a/operators/constellation-node-operator/internal/gcp/client/nodeimage.go b/operators/constellation-node-operator/internal/gcp/client/nodeimage.go index d995cb071..25527a4f4 100644 --- a/operators/constellation-node-operator/internal/gcp/client/nodeimage.go +++ b/operators/constellation-node-operator/internal/gcp/client/nodeimage.go @@ -81,10 +81,7 @@ func (c *Client) CreateNode(ctx context.Context, scalingGroupID string) (nodeNam if instanceGroupManager.BaseInstanceName == nil { return "", "", fmt.Errorf("instance group manager %q has no base instance name", instanceGroupName) } - instanceName, err := generateInstanceName(*instanceGroupManager.BaseInstanceName, c.prng) - if err != nil { - return "", "", err - } + instanceName := generateInstanceName(*instanceGroupManager.BaseInstanceName, c.prng) op, err := c.instanceGroupManagersAPI.CreateInstances(ctx, &computepb.CreateInstancesInstanceGroupManagerRequest{ InstanceGroupManager: instanceGroupName, Project: project, diff --git a/operators/constellation-node-operator/internal/gcp/client/scalinggroup.go b/operators/constellation-node-operator/internal/gcp/client/scalinggroup.go index 5ff75bf85..04a7a3d9c 100644 --- a/operators/constellation-node-operator/internal/gcp/client/scalinggroup.go +++ b/operators/constellation-node-operator/internal/gcp/client/scalinggroup.go @@ -4,8 +4,11 @@ import ( "context" "errors" "fmt" + "strings" + "google.golang.org/api/iterator" computepb "google.golang.org/genproto/googleapis/cloud/compute/v1" + "google.golang.org/protobuf/proto" ) // GetScalingGroupImage returns the image URI of the scaling group. @@ -77,6 +80,47 @@ func (c *Client) SetScalingGroupImage(ctx context.Context, scalingGroupID, image return nil } +// GetScalingGroupName retrieves the name of a scaling group. +func (c *Client) GetScalingGroupName(ctx context.Context, scalingGroupID string) (string, error) { + _, _, instanceGroupName, err := splitInstanceGroupID(scalingGroupID) + if err != nil { + return "", fmt.Errorf("getting scaling group name: %w", err) + } + return strings.ToLower(instanceGroupName), nil +} + +// ListScalingGroups retrieves a list of scaling groups for the cluster. +func (c *Client) ListScalingGroups(ctx context.Context, uid string) (controlPlaneGroupIDs []string, workerGroupIDs []string, err error) { + iter := c.instanceGroupManagersAPI.AggregatedList(ctx, &computepb.AggregatedListInstanceGroupManagersRequest{ + Filter: proto.String(fmt.Sprintf("name eq \".+-.+-%s\"", uid)), // filter by constellation UID + Project: c.projectID, + }) + for instanceGroupManagerScopedListPair, err := iter.Next(); ; instanceGroupManagerScopedListPair, err = iter.Next() { + if err == iterator.Done { + break + } + if err != nil { + return nil, nil, fmt.Errorf("listing instance group managers: %w", err) + } + if instanceGroupManagerScopedListPair.Value == nil { + continue + } + for _, instanceGroupManager := range instanceGroupManagerScopedListPair.Value.InstanceGroupManagers { + if instanceGroupManager == nil || instanceGroupManager.Name == nil || instanceGroupManager.SelfLink == nil { + continue + } + groupID := uriNormalize(*instanceGroupManager.SelfLink) + + if isControlPlaneInstanceGroup(*instanceGroupManager.Name) { + controlPlaneGroupIDs = append(controlPlaneGroupIDs, groupID) + } else if isWorkerInstanceGroup(*instanceGroupManager.Name) { + workerGroupIDs = append(workerGroupIDs, groupID) + } + } + } + return controlPlaneGroupIDs, workerGroupIDs, nil +} + func (c *Client) getScalingGroupTemplate(ctx context.Context, scalingGroupID string) (*computepb.InstanceTemplate, error) { project, zone, instanceGroupName, err := splitInstanceGroupID(scalingGroupID) if err != nil { diff --git a/operators/constellation-node-operator/internal/gcp/client/scalinggroup_test.go b/operators/constellation-node-operator/internal/gcp/client/scalinggroup_test.go index 61c127abb..c5b74d976 100644 --- a/operators/constellation-node-operator/internal/gcp/client/scalinggroup_test.go +++ b/operators/constellation-node-operator/internal/gcp/client/scalinggroup_test.go @@ -282,3 +282,96 @@ func TestSetScalingGroupImage(t *testing.T) { }) } } + +func TestGetScalingGroupName(t *testing.T) { + testCases := map[string]struct { + scalingGroupID string + wantName string + wantErr bool + }{ + "valid scaling group ID": { + scalingGroupID: "projects/project/zones/zone/instanceGroupManagers/instance-group", + wantName: "instance-group", + }, + "invalid scaling group ID": { + scalingGroupID: "invalid", + wantErr: true, + }, + } + + for name, tc := range testCases { + t.Run(name, func(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + + client := Client{} + gotName, err := client.GetScalingGroupName(context.Background(), tc.scalingGroupID) + if tc.wantErr { + assert.Error(err) + return + } + require.NoError(err) + assert.Equal(tc.wantName, gotName) + }) + } +} + +func TestListScalingGroups(t *testing.T) { + testCases := map[string]struct { + name *string + groupID *string + listInstanceGroupManagersErr error + wantControlPlanes []string + wantWorkers []string + wantErr bool + }{ + "list instance group managers fails": { + listInstanceGroupManagersErr: errors.New("list instance group managers error"), + wantErr: true, + }, + "list instance group managers for control plane": { + name: proto.String("test-control-plane-uid"), + groupID: proto.String("projects/project/zones/zone/instanceGroupManagers/test-control-plane-uid"), + wantControlPlanes: []string{ + "projects/project/zones/zone/instanceGroupManagers/test-control-plane-uid", + }, + }, + "list instance group managers for worker": { + name: proto.String("test-worker-uid"), + groupID: proto.String("projects/project/zones/zone/instanceGroupManagers/test-worker-uid"), + wantWorkers: []string{ + "projects/project/zones/zone/instanceGroupManagers/test-worker-uid", + }, + }, + "unrelated instance group manager": { + name: proto.String("test-unrelated-uid"), + groupID: proto.String("projects/project/zones/zone/instanceGroupManagers/test-unrelated-uid"), + }, + "invalid instance group manager": {}, + } + + for name, tc := range testCases { + t.Run(name, func(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + + client := Client{ + instanceGroupManagersAPI: &stubInstanceGroupManagersAPI{ + aggregatedListErr: tc.listInstanceGroupManagersErr, + instanceGroupManager: &computepb.InstanceGroupManager{ + Name: tc.name, + SelfLink: tc.groupID, + }, + }, + } + gotControlPlanes, gotWorkers, err := client.ListScalingGroups(context.Background(), "uid") + if tc.wantErr { + assert.Error(err) + return + } + require.NoError(err) + assert.ElementsMatch(tc.wantControlPlanes, gotControlPlanes) + assert.ElementsMatch(tc.wantWorkers, gotWorkers) + }) + } +} diff --git a/operators/constellation-node-operator/main.go b/operators/constellation-node-operator/main.go index 2dafbff00..db57e5535 100644 --- a/operators/constellation-node-operator/main.go +++ b/operators/constellation-node-operator/main.go @@ -20,6 +20,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/log/zap" azureclient "github.com/edgelesssys/constellation/operators/constellation-node-operator/internal/azure/client" + "github.com/edgelesssys/constellation/operators/constellation-node-operator/internal/deploy" gcpclient "github.com/edgelesssys/constellation/operators/constellation-node-operator/internal/gcp/client" updatev1alpha1 "github.com/edgelesssys/constellation/operators/constellation-node-operator/api/v1alpha1" @@ -36,8 +37,11 @@ var ( const ( defaultAzureCloudConfigPath = "/etc/azure/azure.json" + defaultGCPCloudConfigPath = "/etc/gce/gce.conf" // constellationCSP is the environment variable stating which Cloud Service Provider Constellation is running on. constellationCSP = "CONSTEL_CSP" + // constellationUID is the environment variable stating which uid is used to tag / label cloud provider resources belonging to one constellation. + constellationUID = "constellation-uid" ) func init() { @@ -80,7 +84,10 @@ func main() { os.Exit(1) } case "gcp": - cspClient, clientErr = gcpclient.New(context.Background()) + if cloudConfigPath == "" { + cloudConfigPath = defaultGCPCloudConfigPath + } + cspClient, clientErr = gcpclient.New(context.Background(), cloudConfigPath) if clientErr != nil { setupLog.Error(clientErr, "unable to create GCP client") os.Exit(1) @@ -115,6 +122,11 @@ func main() { } defer etcdClient.Close() + if err := deploy.InitialResources(context.Background(), k8sClient, cspClient, os.Getenv(constellationUID)); err != nil { + setupLog.Error(err, "Unable to deploy initial resources") + os.Exit(1) + } + if err = controllers.NewNodeImageReconciler( cspClient, etcdClient, mgr.GetClient(), mgr.GetScheme(), ).SetupWithManager(mgr); err != nil { @@ -173,4 +185,8 @@ type cspAPI interface { GetScalingGroupImage(ctx context.Context, scalingGroupID string) (string, error) // SetScalingGroupImage sets the image to be used by newly created nodes in a scaling group. SetScalingGroupImage(ctx context.Context, scalingGroupID, imageURI string) error + // GetScalingGroupName retrieves the name of a scaling group. + GetScalingGroupName(ctx context.Context, scalingGroupID string) (string, error) + // ListScalingGroups retrieves a list of scaling groups for the cluster. + ListScalingGroups(ctx context.Context, uid string) (controlPlaneGroupIDs []string, workerGroupIDs []string, err error) }