operator: always create initial resources (#858)

* operator: move csp clients to own path

* operator: use cloudfake as default csp
This commit is contained in:
3u13r 2023-01-04 16:44:47 +01:00 committed by GitHub
parent f720726074
commit 98316b5248
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
49 changed files with 112 additions and 108 deletions

View file

@ -0,0 +1,45 @@
/*
Copyright (c) Edgeless Systems GmbH
SPDX-License-Identifier: AGPL-3.0-only
*/
package client
import (
"context"
"github.com/Azure/azure-sdk-for-go/sdk/azcore/runtime"
armcomputev2 "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v2"
"github.com/edgelesssys/constellation/operators/constellation-node-operator/v2/internal/poller"
)
type virtualMachineScaleSetVMsAPI interface {
Get(ctx context.Context, resourceGroupName string, vmScaleSetName string, instanceID string,
options *armcomputev2.VirtualMachineScaleSetVMsClientGetOptions,
) (armcomputev2.VirtualMachineScaleSetVMsClientGetResponse, error)
GetInstanceView(ctx context.Context, resourceGroupName string, vmScaleSetName string, instanceID string,
options *armcomputev2.VirtualMachineScaleSetVMsClientGetInstanceViewOptions,
) (armcomputev2.VirtualMachineScaleSetVMsClientGetInstanceViewResponse, error)
NewListPager(resourceGroupName string, virtualMachineScaleSetName string,
options *armcomputev2.VirtualMachineScaleSetVMsClientListOptions,
) *runtime.Pager[armcomputev2.VirtualMachineScaleSetVMsClientListResponse]
}
type scaleSetsAPI interface {
Get(ctx context.Context, resourceGroupName string, vmScaleSetName string,
options *armcomputev2.VirtualMachineScaleSetsClientGetOptions,
) (armcomputev2.VirtualMachineScaleSetsClientGetResponse, error)
BeginUpdate(ctx context.Context, resourceGroupName string, vmScaleSetName string, parameters armcomputev2.VirtualMachineScaleSetUpdate,
options *armcomputev2.VirtualMachineScaleSetsClientBeginUpdateOptions,
) (*runtime.Poller[armcomputev2.VirtualMachineScaleSetsClientUpdateResponse], error)
BeginDeleteInstances(ctx context.Context, resourceGroupName string, vmScaleSetName string, vmInstanceIDs armcomputev2.VirtualMachineScaleSetVMInstanceRequiredIDs,
options *armcomputev2.VirtualMachineScaleSetsClientBeginDeleteInstancesOptions,
) (*runtime.Poller[armcomputev2.VirtualMachineScaleSetsClientDeleteInstancesResponse], error)
NewListPager(resourceGroupName string, options *armcomputev2.VirtualMachineScaleSetsClientListOptions,
) *runtime.Pager[armcomputev2.VirtualMachineScaleSetsClientListResponse]
}
type capacityPoller interface {
PollUntilDone(context.Context, *poller.PollUntilDoneOptions) (int64, error)
}

View file

@ -0,0 +1,12 @@
/*
Copyright (c) Edgeless Systems GmbH
SPDX-License-Identifier: AGPL-3.0-only
*/
package client
// AutoscalingCloudProvider returns the cloud-provider name as used by k8s cluster-autoscaler.
func (c *Client) AutoscalingCloudProvider() string {
return "azure"
}

View file

@ -0,0 +1,59 @@
/*
Copyright (c) Edgeless Systems GmbH
SPDX-License-Identifier: AGPL-3.0-only
*/
package client
import (
"github.com/Azure/azure-sdk-for-go/sdk/azidentity"
armcomputev2 "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v2"
"github.com/edgelesssys/constellation/operators/constellation-node-operator/v2/internal/poller"
"github.com/spf13/afero"
)
// Client is a client for the Azure Cloud.
type Client struct {
config cloudConfig
scaleSetsAPI
virtualMachineScaleSetVMsAPI
capacityPollerGenerator func(resourceGroup, scaleSet string, wantedCapacity int64) capacityPoller
pollerOptions *poller.PollUntilDoneOptions
}
// NewFromDefault creates a client with initialized clients.
func NewFromDefault(configPath string) (*Client, error) {
config, err := loadConfig(afero.NewOsFs(), configPath)
if err != nil {
return nil, err
}
cred, err := azidentity.NewDefaultAzureCredential(nil)
if err != nil {
return nil, err
}
scaleSetAPI, err := armcomputev2.NewVirtualMachineScaleSetsClient(config.SubscriptionID, cred, nil)
if err != nil {
return nil, err
}
virtualMachineScaleSetVMsAPI, err := armcomputev2.NewVirtualMachineScaleSetVMsClient(config.SubscriptionID, cred, nil)
if err != nil {
return nil, err
}
return &Client{
config: *config,
scaleSetsAPI: scaleSetAPI,
virtualMachineScaleSetVMsAPI: virtualMachineScaleSetVMsAPI,
capacityPollerGenerator: func(resourceGroup, scaleSet string, wantedCapacity int64) capacityPoller {
return poller.New[int64](&capacityPollingHandler{
resourceGroup: resourceGroup,
scaleSet: scaleSet,
wantedCapacity: wantedCapacity,
scaleSetsAPI: scaleSetAPI,
})
},
}, nil
}

View file

@ -0,0 +1,170 @@
/*
Copyright (c) Edgeless Systems GmbH
SPDX-License-Identifier: AGPL-3.0-only
*/
package client
import (
"context"
"net/http"
"github.com/Azure/azure-sdk-for-go/sdk/azcore/runtime"
armcomputev2 "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v2"
)
type stubScaleSetsAPI struct {
scaleSet armcomputev2.VirtualMachineScaleSetsClientGetResponse
getErr error
updateResponse armcomputev2.VirtualMachineScaleSetsClientUpdateResponse
updateErr error
deleteResponse armcomputev2.VirtualMachineScaleSetsClientDeleteInstancesResponse
deleteErr error
resultErr error
pager *stubVMSSPager
}
func (a *stubScaleSetsAPI) Get(ctx context.Context, resourceGroupName string, vmScaleSetName string,
options *armcomputev2.VirtualMachineScaleSetsClientGetOptions,
) (armcomputev2.VirtualMachineScaleSetsClientGetResponse, error) {
return a.scaleSet, a.getErr
}
func (a *stubScaleSetsAPI) BeginUpdate(ctx context.Context, resourceGroupName string, vmScaleSetName string, parameters armcomputev2.VirtualMachineScaleSetUpdate,
options *armcomputev2.VirtualMachineScaleSetsClientBeginUpdateOptions,
) (*runtime.Poller[armcomputev2.VirtualMachineScaleSetsClientUpdateResponse], error) {
poller, err := runtime.NewPoller(nil, runtime.NewPipeline("", "", runtime.PipelineOptions{}, nil), &runtime.NewPollerOptions[armcomputev2.VirtualMachineScaleSetsClientUpdateResponse]{
Handler: &stubPoller[armcomputev2.VirtualMachineScaleSetsClientUpdateResponse]{
result: a.updateResponse,
resultErr: a.resultErr,
},
})
if err != nil {
panic(err)
}
return poller, a.updateErr
}
func (a *stubScaleSetsAPI) BeginDeleteInstances(ctx context.Context, resourceGroupName string, vmScaleSetName string, vmInstanceIDs armcomputev2.VirtualMachineScaleSetVMInstanceRequiredIDs,
options *armcomputev2.VirtualMachineScaleSetsClientBeginDeleteInstancesOptions,
) (*runtime.Poller[armcomputev2.VirtualMachineScaleSetsClientDeleteInstancesResponse], error) {
poller, err := runtime.NewPoller(nil, runtime.NewPipeline("", "", runtime.PipelineOptions{}, nil), &runtime.NewPollerOptions[armcomputev2.VirtualMachineScaleSetsClientDeleteInstancesResponse]{
Handler: &stubPoller[armcomputev2.VirtualMachineScaleSetsClientDeleteInstancesResponse]{
result: a.deleteResponse,
resultErr: a.resultErr,
},
})
if err != nil {
panic(err)
}
return poller, a.deleteErr
}
func (a *stubScaleSetsAPI) NewListPager(resourceGroupName string, options *armcomputev2.VirtualMachineScaleSetsClientListOptions,
) *runtime.Pager[armcomputev2.VirtualMachineScaleSetsClientListResponse] {
return runtime.NewPager(runtime.PagingHandler[armcomputev2.VirtualMachineScaleSetsClientListResponse]{
More: a.pager.moreFunc(),
Fetcher: a.pager.fetcherFunc(),
})
}
type stubvirtualMachineScaleSetVMsAPI struct {
scaleSetVM armcomputev2.VirtualMachineScaleSetVMsClientGetResponse
getErr error
instanceView armcomputev2.VirtualMachineScaleSetVMsClientGetInstanceViewResponse
instanceViewErr error
pager *stubVMSSVMPager
}
func (a *stubvirtualMachineScaleSetVMsAPI) Get(ctx context.Context, resourceGroupName string, vmScaleSetName string, instanceID string,
options *armcomputev2.VirtualMachineScaleSetVMsClientGetOptions,
) (armcomputev2.VirtualMachineScaleSetVMsClientGetResponse, error) {
return a.scaleSetVM, a.getErr
}
func (a *stubvirtualMachineScaleSetVMsAPI) GetInstanceView(ctx context.Context, resourceGroupName string, vmScaleSetName string, instanceID string,
options *armcomputev2.VirtualMachineScaleSetVMsClientGetInstanceViewOptions,
) (armcomputev2.VirtualMachineScaleSetVMsClientGetInstanceViewResponse, error) {
return a.instanceView, a.instanceViewErr
}
func (a *stubvirtualMachineScaleSetVMsAPI) NewListPager(resourceGroupName string, virtualMachineScaleSetName string,
options *armcomputev2.VirtualMachineScaleSetVMsClientListOptions,
) *runtime.Pager[armcomputev2.VirtualMachineScaleSetVMsClientListResponse] {
return runtime.NewPager(runtime.PagingHandler[armcomputev2.VirtualMachineScaleSetVMsClientListResponse]{
More: a.pager.moreFunc(),
Fetcher: a.pager.fetcherFunc(),
})
}
type stubPoller[T any] struct {
result T
pollErr error
resultErr error
}
func (p *stubPoller[T]) Done() bool {
return true
}
func (p *stubPoller[T]) Poll(context.Context) (*http.Response, error) {
return nil, p.pollErr
}
func (p *stubPoller[T]) Result(ctx context.Context, out *T) error {
*out = p.result
return p.resultErr
}
type stubVMSSVMPager struct {
list []armcomputev2.VirtualMachineScaleSetVM
fetchErr error
more bool
}
func (p *stubVMSSVMPager) moreFunc() func(armcomputev2.VirtualMachineScaleSetVMsClientListResponse) bool {
return func(armcomputev2.VirtualMachineScaleSetVMsClientListResponse) bool {
return p.more
}
}
func (p *stubVMSSVMPager) fetcherFunc() func(context.Context, *armcomputev2.VirtualMachineScaleSetVMsClientListResponse) (armcomputev2.VirtualMachineScaleSetVMsClientListResponse, error) {
return func(context.Context, *armcomputev2.VirtualMachineScaleSetVMsClientListResponse) (armcomputev2.VirtualMachineScaleSetVMsClientListResponse, error) {
page := make([]*armcomputev2.VirtualMachineScaleSetVM, len(p.list))
for i := range p.list {
page[i] = &p.list[i]
}
return armcomputev2.VirtualMachineScaleSetVMsClientListResponse{
VirtualMachineScaleSetVMListResult: armcomputev2.VirtualMachineScaleSetVMListResult{
Value: page,
},
}, p.fetchErr
}
}
type stubVMSSPager struct {
list []armcomputev2.VirtualMachineScaleSet
fetchErr error
more bool
}
func (p *stubVMSSPager) moreFunc() func(armcomputev2.VirtualMachineScaleSetsClientListResponse) bool {
return func(armcomputev2.VirtualMachineScaleSetsClientListResponse) bool {
return p.more
}
}
func (p *stubVMSSPager) fetcherFunc() func(context.Context, *armcomputev2.VirtualMachineScaleSetsClientListResponse) (armcomputev2.VirtualMachineScaleSetsClientListResponse, error) {
return func(context.Context, *armcomputev2.VirtualMachineScaleSetsClientListResponse) (armcomputev2.VirtualMachineScaleSetsClientListResponse, error) {
page := make([]*armcomputev2.VirtualMachineScaleSet, len(p.list))
for i := range p.list {
page[i] = &p.list[i]
}
return armcomputev2.VirtualMachineScaleSetsClientListResponse{
VirtualMachineScaleSetListResult: armcomputev2.VirtualMachineScaleSetListResult{
Value: page,
},
}, p.fetchErr
}
}

View file

@ -0,0 +1,39 @@
/*
Copyright (c) Edgeless Systems GmbH
SPDX-License-Identifier: AGPL-3.0-only
*/
package client
import (
"encoding/json"
"errors"
"github.com/spf13/afero"
)
// cloudConfig uses same format as azure cloud controller manager to share the same kubernetes secret.
// this definition only contains the fields we need.
// reference: https://cloud-provider-azure.sigs.k8s.io/install/configs/ .
type cloudConfig struct {
TenantID string `json:"tenantId,omitempty"`
SubscriptionID string `json:"subscriptionId,omitempty"`
ResourceGroup string `json:"resourceGroup,omitempty"`
}
// loadConfig loads the cloud config from the given path.
func loadConfig(fs afero.Fs, path string) (*cloudConfig, error) {
rawConfig, err := afero.ReadFile(fs, path)
if err != nil {
return nil, err
}
var config cloudConfig
if err := json.Unmarshal(rawConfig, &config); err != nil {
return nil, err
}
if config.TenantID == "" || config.SubscriptionID == "" {
return nil, errors.New("invalid config: tenantId and subscriptionId are required")
}
return &config, nil
}

View file

@ -0,0 +1,62 @@
/*
Copyright (c) Edgeless Systems GmbH
SPDX-License-Identifier: AGPL-3.0-only
*/
package client
import (
"testing"
"github.com/spf13/afero"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestLoadConfig(t *testing.T) {
testCases := map[string]struct {
rawConfig string
skipWrite bool
wantTenantID string
wantSubscriptionID string
wantErr bool
}{
"valid config": {
rawConfig: `{"tenantId":"tenantId","subscriptionId":"subscriptionId"}`,
wantTenantID: "tenantId",
wantSubscriptionID: "subscriptionId",
},
"invalid config": {
rawConfig: `{"tenantId":"tenantId","subscriptionId":""}`,
wantErr: true,
},
"config is empty": {
wantErr: true,
},
"config does not exist": {
skipWrite: true,
wantErr: true,
},
}
for name, tc := range testCases {
t.Run(name, func(t *testing.T) {
assert := assert.New(t)
require := require.New(t)
fs := afero.NewMemMapFs()
if !tc.skipWrite {
require.NoError(afero.WriteFile(fs, "config.json", []byte(tc.rawConfig), 0o644))
}
gotConfig, err := loadConfig(fs, "config.json")
if tc.wantErr {
assert.Error(err)
return
}
require.NoError(err)
assert.Equal(tc.wantTenantID, gotConfig.TenantID)
assert.Equal(tc.wantSubscriptionID, gotConfig.SubscriptionID)
})
}
}

View file

@ -0,0 +1,63 @@
/*
Copyright (c) Edgeless Systems GmbH
SPDX-License-Identifier: AGPL-3.0-only
*/
package client
import (
armcomputev2 "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v2"
updatev1alpha1 "github.com/edgelesssys/constellation/operators/constellation-node-operator/v2/api/v1alpha1"
)
const (
provisioningStateCreating = "ProvisioningState/creating"
provisioningStateUpdating = "ProvisioningState/updating"
provisioningStateMigrating = "ProvisioningState/migrating"
provisioningStateFailed = "ProvisioningState/failed"
provisioningStateDeleting = "ProvisioningState/deleting"
powerStateStarting = "PowerState/starting"
powerStateStopping = "PowerState/stopping"
powerStateStopped = "PowerState/stopped"
powerStateDeallocating = "PowerState/deallocating"
powerStateDeallocated = "PowerState/deallocated"
powerStateRunning = "PowerState/running"
)
// nodeStateFromStatuses returns the node state from instance view statuses.
// reference:
// - https://docs.microsoft.com/en-us/azure/virtual-machines/states-billing#provisioning-states
// - https://docs.microsoft.com/en-us/azure/virtual-machines/states-billing#power-states-and-billing
func nodeStateFromStatuses(statuses []*armcomputev2.InstanceViewStatus) updatev1alpha1.CSPNodeState {
for _, status := range statuses {
if status == nil || status.Code == nil {
continue
}
switch *status.Code {
case provisioningStateCreating:
return updatev1alpha1.NodeStateCreating
case provisioningStateUpdating:
return updatev1alpha1.NodeStateStopped
case provisioningStateMigrating:
return updatev1alpha1.NodeStateStopped
case provisioningStateFailed:
return updatev1alpha1.NodeStateFailed
case provisioningStateDeleting:
return updatev1alpha1.NodeStateTerminating
case powerStateStarting:
return updatev1alpha1.NodeStateCreating
case powerStateStopping:
return updatev1alpha1.NodeStateStopped
case powerStateStopped:
return updatev1alpha1.NodeStateStopped
case powerStateDeallocating:
return updatev1alpha1.NodeStateStopped
case powerStateDeallocated:
return updatev1alpha1.NodeStateStopped
case powerStateRunning:
return updatev1alpha1.NodeStateReady
}
}
return updatev1alpha1.NodeStateUnknown
}

View file

@ -0,0 +1,110 @@
/*
Copyright (c) Edgeless Systems GmbH
SPDX-License-Identifier: AGPL-3.0-only
*/
package client
import (
"testing"
"github.com/Azure/azure-sdk-for-go/sdk/azcore/to"
armcomputev2 "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v2"
"github.com/stretchr/testify/assert"
updatev1alpha1 "github.com/edgelesssys/constellation/operators/constellation-node-operator/v2/api/v1alpha1"
)
// this state is included in most VMs but not needed
// to determine the node state as every provisioned VM also has a power state.
const provisioningStateSucceeded = "ProvisioningState/succeeded"
func TestNodeStateFromStatuses(t *testing.T) {
testCases := map[string]struct {
statuses []*armcomputev2.InstanceViewStatus
wantState updatev1alpha1.CSPNodeState
}{
"no statuses": {
wantState: updatev1alpha1.NodeStateUnknown,
},
"invalid status": {
statuses: []*armcomputev2.InstanceViewStatus{nil, {Code: nil}},
wantState: updatev1alpha1.NodeStateUnknown,
},
"provisioning creating": {
statuses: []*armcomputev2.InstanceViewStatus{{Code: to.Ptr(provisioningStateCreating)}},
wantState: updatev1alpha1.NodeStateCreating,
},
"provisioning updating": {
statuses: []*armcomputev2.InstanceViewStatus{{Code: to.Ptr(provisioningStateUpdating)}},
wantState: updatev1alpha1.NodeStateStopped,
},
"provisioning migrating": {
statuses: []*armcomputev2.InstanceViewStatus{{Code: to.Ptr(provisioningStateMigrating)}},
wantState: updatev1alpha1.NodeStateStopped,
},
"provisioning failed": {
statuses: []*armcomputev2.InstanceViewStatus{{Code: to.Ptr(provisioningStateFailed)}},
wantState: updatev1alpha1.NodeStateFailed,
},
"provisioning deleting": {
statuses: []*armcomputev2.InstanceViewStatus{{Code: to.Ptr(provisioningStateDeleting)}},
wantState: updatev1alpha1.NodeStateTerminating,
},
"provisioning succeeded (but no power state)": {
statuses: []*armcomputev2.InstanceViewStatus{{Code: to.Ptr(provisioningStateSucceeded)}},
wantState: updatev1alpha1.NodeStateUnknown,
},
"power starting": {
statuses: []*armcomputev2.InstanceViewStatus{
{Code: to.Ptr(provisioningStateSucceeded)},
{Code: to.Ptr(powerStateStarting)},
},
wantState: updatev1alpha1.NodeStateCreating,
},
"power stopping": {
statuses: []*armcomputev2.InstanceViewStatus{
{Code: to.Ptr(provisioningStateSucceeded)},
{Code: to.Ptr(powerStateStopping)},
},
wantState: updatev1alpha1.NodeStateStopped,
},
"power stopped": {
statuses: []*armcomputev2.InstanceViewStatus{
{Code: to.Ptr(provisioningStateSucceeded)},
{Code: to.Ptr(powerStateStopped)},
},
wantState: updatev1alpha1.NodeStateStopped,
},
"power deallocating": {
statuses: []*armcomputev2.InstanceViewStatus{
{Code: to.Ptr(provisioningStateSucceeded)},
{Code: to.Ptr(powerStateDeallocating)},
},
wantState: updatev1alpha1.NodeStateStopped,
},
"power deallocated": {
statuses: []*armcomputev2.InstanceViewStatus{
{Code: to.Ptr(provisioningStateSucceeded)},
{Code: to.Ptr(powerStateDeallocated)},
},
wantState: updatev1alpha1.NodeStateStopped,
},
"power running": {
statuses: []*armcomputev2.InstanceViewStatus{
{Code: to.Ptr(provisioningStateSucceeded)},
{Code: to.Ptr(powerStateRunning)},
},
wantState: updatev1alpha1.NodeStateReady,
},
}
for name, tc := range testCases {
t.Run(name, func(t *testing.T) {
assert := assert.New(t)
gotState := nodeStateFromStatuses(tc.statuses)
assert.Equal(tc.wantState, gotState)
})
}
}

View file

@ -0,0 +1,165 @@
/*
Copyright (c) Edgeless Systems GmbH
SPDX-License-Identifier: AGPL-3.0-only
*/
package client
import (
"context"
"fmt"
"strings"
"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v2"
)
// GetNodeImage returns the image name of the node.
func (c *Client) GetNodeImage(ctx context.Context, providerID string) (string, error) {
_, resourceGroup, scaleSet, instanceID, err := scaleSetInformationFromProviderID(providerID)
if err != nil {
return "", err
}
resp, err := c.virtualMachineScaleSetVMsAPI.Get(ctx, resourceGroup, scaleSet, instanceID, nil)
if err != nil {
return "", err
}
if resp.Properties == nil ||
resp.Properties.StorageProfile == nil ||
resp.Properties.StorageProfile.ImageReference == nil ||
resp.Properties.StorageProfile.ImageReference.ID == nil && resp.Properties.StorageProfile.ImageReference.CommunityGalleryImageID == nil {
return "", fmt.Errorf("node %q does not have valid image reference", providerID)
}
if resp.Properties.StorageProfile.ImageReference.ID != nil {
return *resp.Properties.StorageProfile.ImageReference.ID, nil
}
return *resp.Properties.StorageProfile.ImageReference.CommunityGalleryImageID, nil
}
// GetScalingGroupID returns the scaling group ID of the node.
func (c *Client) GetScalingGroupID(ctx context.Context, providerID string) (string, error) {
subscriptionID, resourceGroup, scaleSet, _, err := scaleSetInformationFromProviderID(providerID)
if err != nil {
return "", err
}
return joinVMSSID(subscriptionID, resourceGroup, scaleSet), nil
}
// CreateNode creates a node in the specified scaling group.
func (c *Client) CreateNode(ctx context.Context, scalingGroupID string) (nodeName, providerID string, err error) {
_, resourceGroup, scaleSet, err := splitVMSSID(scalingGroupID)
if err != nil {
return "", "", err
}
// get list of instance IDs before scaling,
var oldVMIDs []string
pager := c.virtualMachineScaleSetVMsAPI.NewListPager(resourceGroup, scaleSet, nil)
for pager.More() {
page, err := pager.NextPage(ctx)
if err != nil {
return "", "", err
}
for _, vm := range page.Value {
if vm == nil || vm.ID == nil {
continue
}
oldVMIDs = append(oldVMIDs, *vm.ID)
}
}
// increase the number of instances by one
resp, err := c.scaleSetsAPI.Get(ctx, resourceGroup, scaleSet, nil)
if err != nil {
return "", "", err
}
if resp.SKU == nil || resp.SKU.Capacity == nil {
return "", "", fmt.Errorf("scale set %q does not have valid capacity", scaleSet)
}
wantedCapacity := *resp.SKU.Capacity + 1
_, err = c.scaleSetsAPI.BeginUpdate(ctx, resourceGroup, scaleSet, armcompute.VirtualMachineScaleSetUpdate{
SKU: &armcompute.SKU{
Capacity: &wantedCapacity,
},
}, nil)
if err != nil {
return "", "", err
}
poller := c.capacityPollerGenerator(resourceGroup, scaleSet, wantedCapacity)
if _, err := poller.PollUntilDone(ctx, c.pollerOptions); err != nil {
return "", "", err
}
// get the list of instances again
// and find the new instance id by comparing the old and new lists
pager = c.virtualMachineScaleSetVMsAPI.NewListPager(resourceGroup, scaleSet, nil)
for pager.More() {
page, err := pager.NextPage(ctx)
if err != nil {
return "", "", err
}
for _, vm := range page.Value {
// check if the instance already existed in the old list
if !hasKnownVMID(vm, oldVMIDs) {
return strings.ToLower(*vm.Properties.OSProfile.ComputerName), "azure://" + *vm.ID, nil
}
}
}
return "", "", fmt.Errorf("failed to find new node after scaling up")
}
// DeleteNode deletes a node specified by its provider ID.
func (c *Client) DeleteNode(ctx context.Context, providerID string) error {
_, resourceGroup, scaleSet, instanceID, err := scaleSetInformationFromProviderID(providerID)
if err != nil {
return err
}
ids := armcompute.VirtualMachineScaleSetVMInstanceRequiredIDs{InstanceIDs: []*string{&instanceID}}
_, err = c.scaleSetsAPI.BeginDeleteInstances(ctx, resourceGroup, scaleSet, ids, nil)
return err
}
// capacityPollingHandler polls a scale set
// until its capacity reaches the desired value.
type capacityPollingHandler struct {
done bool
wantedCapacity int64
resourceGroup string
scaleSet string
scaleSetsAPI
}
func (h *capacityPollingHandler) Done() bool {
return h.done
}
func (h *capacityPollingHandler) Poll(ctx context.Context) error {
resp, err := h.scaleSetsAPI.Get(ctx, h.resourceGroup, h.scaleSet, nil)
if err != nil {
return err
}
if resp.SKU == nil || resp.SKU.Capacity == nil {
return fmt.Errorf("scale set %q does not have valid capacity", h.scaleSet)
}
h.done = *resp.SKU.Capacity == h.wantedCapacity
return nil
}
func (h *capacityPollingHandler) Result(ctx context.Context, out *int64) error {
if !h.done {
return fmt.Errorf("failed to scale up")
}
*out = h.wantedCapacity
return nil
}
// hasKnownVMID returns true if the vmID is found in the vm ID list.
func hasKnownVMID(vm *armcompute.VirtualMachineScaleSetVM, vmIDs []string) bool {
for _, id := range vmIDs {
if vm != nil && vm.ID != nil && *vm.ID == id {
return true
}
}
return false
}

View file

@ -0,0 +1,372 @@
/*
Copyright (c) Edgeless Systems GmbH
SPDX-License-Identifier: AGPL-3.0-only
*/
package client
import (
"context"
"errors"
"sync"
"testing"
"github.com/Azure/azure-sdk-for-go/sdk/azcore/to"
armcomputev2 "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v2"
"github.com/edgelesssys/constellation/operators/constellation-node-operator/v2/internal/poller"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestGetNodeImage(t *testing.T) {
testCases := map[string]struct {
providerID string
vm armcomputev2.VirtualMachineScaleSetVM
getScaleSetVMErr error
wantImage string
wantErr bool
}{
"getting node image works": {
providerID: "azure:///subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/virtualMachineScaleSets/scale-set-name/virtualMachines/instance-id",
vm: armcomputev2.VirtualMachineScaleSetVM{
Properties: &armcomputev2.VirtualMachineScaleSetVMProperties{
StorageProfile: &armcomputev2.StorageProfile{
ImageReference: &armcomputev2.ImageReference{
ID: to.Ptr("/subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/images/image-name"),
},
},
},
},
wantImage: "/subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/images/image-name",
},
"getting community node image works": {
providerID: "azure:///subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/virtualMachineScaleSets/scale-set-name/virtualMachines/instance-id",
vm: armcomputev2.VirtualMachineScaleSetVM{
Properties: &armcomputev2.VirtualMachineScaleSetVMProperties{
StorageProfile: &armcomputev2.StorageProfile{
ImageReference: &armcomputev2.ImageReference{
CommunityGalleryImageID: to.Ptr("/CommunityGalleries/gallery-name/Images/image-name/Versions/1.2.3"),
},
},
},
},
wantImage: "/CommunityGalleries/gallery-name/Images/image-name/Versions/1.2.3",
},
"splitting providerID fails": {
providerID: "invalid",
wantErr: true,
},
"get scale set vm fails": {
providerID: "azure:///subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/virtualMachineScaleSets/scale-set-name/virtualMachines/instance-id",
getScaleSetVMErr: errors.New("get scale set vm error"),
wantErr: true,
},
"scale set vm does not have valid image reference": {
providerID: "azure:///subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/virtualMachineScaleSets/scale-set-name/virtualMachines/instance-id",
vm: armcomputev2.VirtualMachineScaleSetVM{},
wantErr: true,
},
}
for name, tc := range testCases {
t.Run(name, func(t *testing.T) {
assert := assert.New(t)
require := require.New(t)
client := Client{
virtualMachineScaleSetVMsAPI: &stubvirtualMachineScaleSetVMsAPI{
scaleSetVM: armcomputev2.VirtualMachineScaleSetVMsClientGetResponse{
VirtualMachineScaleSetVM: tc.vm,
},
getErr: tc.getScaleSetVMErr,
},
}
gotImage, err := client.GetNodeImage(context.Background(), tc.providerID)
if tc.wantErr {
assert.Error(err)
return
}
require.NoError(err)
assert.Equal(tc.wantImage, gotImage)
})
}
}
func TestGetScalingGroupID(t *testing.T) {
testCases := map[string]struct {
providerID string
wantScalingGroupID string
wantErr bool
}{
"getting scaling group id works": {
providerID: "azure:///subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/virtualMachineScaleSets/scale-set-name/virtualMachines/instance-id",
wantScalingGroupID: "/subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/virtualMachineScaleSets/scale-set-name",
},
"splitting providerID fails": {
providerID: "invalid",
wantErr: true,
},
}
for name, tc := range testCases {
t.Run(name, func(t *testing.T) {
assert := assert.New(t)
require := require.New(t)
client := Client{}
gotScalingGroupID, err := client.GetScalingGroupID(context.Background(), tc.providerID)
if tc.wantErr {
assert.Error(err)
return
}
require.NoError(err)
assert.Equal(tc.wantScalingGroupID, gotScalingGroupID)
})
}
}
func TestCreateNode(t *testing.T) {
testCases := map[string]struct {
scalingGroupID string
sku *armcomputev2.SKU
preexistingVMs []armcomputev2.VirtualMachineScaleSetVM
newVM *armcomputev2.VirtualMachineScaleSetVM
fetchErr error
pollErr error
getSKUCapacityErr error
updateScaleSetErr error
wantNodeName string
wantProviderID string
wantEarlyErr bool
wantLateErr bool
}{
"creating node works": {
scalingGroupID: "/subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/virtualMachineScaleSets/scale-set-name",
sku: &armcomputev2.SKU{
Capacity: to.Ptr(int64(0)),
},
newVM: &armcomputev2.VirtualMachineScaleSetVM{
ID: to.Ptr("/subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/virtualMachineScaleSets/scale-set-name/virtualMachines/node-name"),
Properties: &armcomputev2.VirtualMachineScaleSetVMProperties{
OSProfile: &armcomputev2.OSProfile{
ComputerName: to.Ptr("node-name"),
},
},
},
wantNodeName: "node-name",
wantProviderID: "azure:///subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/virtualMachineScaleSets/scale-set-name/virtualMachines/node-name",
},
"creating node works with existing nodes": {
scalingGroupID: "/subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/virtualMachineScaleSets/scale-set-name",
sku: &armcomputev2.SKU{
Capacity: to.Ptr(int64(1)),
},
preexistingVMs: []armcomputev2.VirtualMachineScaleSetVM{
{ID: to.Ptr("/subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/virtualMachineScaleSets/scale-set-name/virtualMachines/preexisting-node")},
},
newVM: &armcomputev2.VirtualMachineScaleSetVM{
ID: to.Ptr("/subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/virtualMachineScaleSets/scale-set-name/virtualMachines/new-node"),
Properties: &armcomputev2.VirtualMachineScaleSetVMProperties{
OSProfile: &armcomputev2.OSProfile{
ComputerName: to.Ptr("new-node"),
},
},
},
wantNodeName: "new-node",
wantProviderID: "azure:///subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/virtualMachineScaleSets/scale-set-name/virtualMachines/new-node",
},
"splitting scalingGroupID fails": {
scalingGroupID: "invalid",
wantEarlyErr: true,
},
"getting node list fails": {
scalingGroupID: "/subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/virtualMachineScaleSets/scale-set-name",
fetchErr: errors.New("get node list error"),
wantEarlyErr: true,
},
"getting SKU capacity fails": {
scalingGroupID: "/subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/virtualMachineScaleSets/scale-set-name",
getSKUCapacityErr: errors.New("get sku capacity error"),
wantEarlyErr: true,
},
"sku is invalid": {
scalingGroupID: "/subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/virtualMachineScaleSets/scale-set-name",
wantEarlyErr: true,
},
"updating scale set capacity fails": {
scalingGroupID: "/subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/virtualMachineScaleSets/scale-set-name",
sku: &armcomputev2.SKU{Capacity: to.Ptr(int64(0))},
updateScaleSetErr: errors.New("update scale set error"),
wantEarlyErr: true,
},
"polling for increased capacity fails": {
scalingGroupID: "/subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/virtualMachineScaleSets/scale-set-name",
sku: &armcomputev2.SKU{Capacity: to.Ptr(int64(0))},
pollErr: errors.New("poll error"),
wantLateErr: true,
},
"new node cannot be found": {
scalingGroupID: "/subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/virtualMachineScaleSets/scale-set-name",
sku: &armcomputev2.SKU{Capacity: to.Ptr(int64(0))},
wantLateErr: true,
},
}
for name, tc := range testCases {
t.Run(name, func(t *testing.T) {
assert := assert.New(t)
require := require.New(t)
pager := &stubVMSSVMPager{
list: tc.preexistingVMs,
fetchErr: tc.fetchErr,
}
poller := newStubCapacityPoller(tc.pollErr)
client := Client{
virtualMachineScaleSetVMsAPI: &stubvirtualMachineScaleSetVMsAPI{
pager: pager,
},
scaleSetsAPI: &stubScaleSetsAPI{
scaleSet: armcomputev2.VirtualMachineScaleSetsClientGetResponse{
VirtualMachineScaleSet: armcomputev2.VirtualMachineScaleSet{
SKU: tc.sku,
},
},
getErr: tc.getSKUCapacityErr,
updateErr: tc.updateScaleSetErr,
},
capacityPollerGenerator: func(resourceGroup, scaleSet string, wantedCapacity int64) capacityPoller {
return poller
},
}
wg := sync.WaitGroup{}
wg.Add(1)
var gotNodeName, gotProviderID string
var createErr error
go func() {
defer wg.Done()
gotNodeName, gotProviderID, createErr = client.CreateNode(context.Background(), tc.scalingGroupID)
}()
// want error before PollUntilDone is called
if tc.wantEarlyErr {
wg.Wait()
assert.Error(createErr)
return
}
// wait for PollUntilDone to be called
<-poller.pollC
// update list of nodes
if tc.newVM != nil {
pager.list = append(pager.list, *tc.newVM)
}
// let PollUntilDone finish
poller.doneC <- struct{}{}
wg.Wait()
if tc.wantLateErr {
assert.Error(createErr)
return
}
require.NoError(createErr)
assert.Equal(tc.wantNodeName, gotNodeName)
assert.Equal(tc.wantProviderID, gotProviderID)
})
}
}
func TestDeleteNode(t *testing.T) {
testCases := map[string]struct {
providerID string
deleteErr error
wantErr bool
}{
"deleting node works": {
providerID: "azure:///subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/virtualMachineScaleSets/scale-set-name/virtualMachines/instance-id",
},
"invalid providerID": {
providerID: "invalid",
wantErr: true,
},
"deleting node fails": {
providerID: "azure:///subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/virtualMachineScaleSets/scale-set-name/virtualMachines/instance-id",
deleteErr: errors.New("delete error"),
wantErr: true,
},
}
for name, tc := range testCases {
t.Run(name, func(t *testing.T) {
assert := assert.New(t)
client := Client{
scaleSetsAPI: &stubScaleSetsAPI{deleteErr: tc.deleteErr},
}
err := client.DeleteNode(context.Background(), tc.providerID)
if tc.wantErr {
assert.Error(err)
return
}
assert.NoError(err)
})
}
}
// TODO: test capacityPollingHandler
func TestCapacityPollingHandler(t *testing.T) {
assert := assert.New(t)
wantCapacity := int64(1)
var gotCapacity int64
handler := capacityPollingHandler{
scaleSetsAPI: &stubScaleSetsAPI{
scaleSet: armcomputev2.VirtualMachineScaleSetsClientGetResponse{
VirtualMachineScaleSet: armcomputev2.VirtualMachineScaleSet{
SKU: &armcomputev2.SKU{Capacity: to.Ptr(int64(0))},
},
},
},
wantedCapacity: wantCapacity,
}
assert.NoError(handler.Poll(context.Background()))
assert.False(handler.Done())
// Calling Result early should error
assert.Error(handler.Result(context.Background(), &gotCapacity))
// let scaleSet API error
handler.scaleSetsAPI.(*stubScaleSetsAPI).getErr = errors.New("get error")
assert.Error(handler.Poll(context.Background()))
handler.scaleSetsAPI.(*stubScaleSetsAPI).getErr = nil
// let scaleSet API return invalid SKU
handler.scaleSetsAPI.(*stubScaleSetsAPI).scaleSet.SKU = nil
assert.Error(handler.Poll(context.Background()))
// let Poll finish
handler.scaleSetsAPI.(*stubScaleSetsAPI).scaleSet.SKU = &armcomputev2.SKU{Capacity: to.Ptr(wantCapacity)}
assert.NoError(handler.Poll(context.Background()))
assert.True(handler.Done())
assert.NoError(handler.Result(context.Background(), &gotCapacity))
assert.Equal(wantCapacity, gotCapacity)
}
type stubCapacityPoller struct {
pollErr error
pollC chan struct{}
doneC chan struct{}
}
func newStubCapacityPoller(pollErr error) *stubCapacityPoller {
return &stubCapacityPoller{
pollErr: pollErr,
pollC: make(chan struct{}),
doneC: make(chan struct{}),
}
}
func (p *stubCapacityPoller) PollUntilDone(context.Context, *poller.PollUntilDoneOptions) (int64, error) {
p.pollC <- struct{}{}
<-p.doneC
return 0, p.pollErr
}

View file

@ -0,0 +1,33 @@
/*
Copyright (c) Edgeless Systems GmbH
SPDX-License-Identifier: AGPL-3.0-only
*/
package client
import (
"context"
"errors"
"net/http"
"github.com/Azure/azure-sdk-for-go/sdk/azcore"
updatev1alpha1 "github.com/edgelesssys/constellation/operators/constellation-node-operator/v2/api/v1alpha1"
)
// GetNodeState returns the state of the node.
func (c *Client) GetNodeState(ctx context.Context, providerID string) (updatev1alpha1.CSPNodeState, error) {
_, resourceGroup, scaleSet, instanceID, err := scaleSetInformationFromProviderID(providerID)
if err != nil {
return updatev1alpha1.NodeStateUnknown, err
}
instanceView, err := c.virtualMachineScaleSetVMsAPI.GetInstanceView(ctx, resourceGroup, scaleSet, instanceID, nil)
if err != nil {
var respErr *azcore.ResponseError
if errors.As(err, &respErr) && respErr.StatusCode == http.StatusNotFound {
return updatev1alpha1.NodeStateTerminated, nil
}
return updatev1alpha1.NodeStateUnknown, err
}
return nodeStateFromStatuses(instanceView.Statuses), nil
}

View file

@ -0,0 +1,79 @@
/*
Copyright (c) Edgeless Systems GmbH
SPDX-License-Identifier: AGPL-3.0-only
*/
package client
import (
"context"
"errors"
"net/http"
"testing"
"github.com/Azure/azure-sdk-for-go/sdk/azcore"
"github.com/Azure/azure-sdk-for-go/sdk/azcore/to"
armcomputev2 "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v2"
updatev1alpha1 "github.com/edgelesssys/constellation/operators/constellation-node-operator/v2/api/v1alpha1"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestGetNodeState(t *testing.T) {
testCases := map[string]struct {
providerID string
instanceView armcomputev2.VirtualMachineScaleSetVMInstanceView
getInstanceViewErr error
wantState updatev1alpha1.CSPNodeState
wantErr bool
}{
"getting node state works": {
providerID: "azure:///subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/virtualMachineScaleSets/scale-set-name/virtualMachines/instance-id",
instanceView: armcomputev2.VirtualMachineScaleSetVMInstanceView{
Statuses: []*armcomputev2.InstanceViewStatus{
{Code: to.Ptr("ProvisioningState/succeeded")},
{Code: to.Ptr("PowerState/running")},
},
},
wantState: updatev1alpha1.NodeStateReady,
},
"splitting providerID fails": {
providerID: "invalid",
wantErr: true,
},
"get instance view fails": {
providerID: "azure:///subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/virtualMachineScaleSets/scale-set-name/virtualMachines/instance-id",
getInstanceViewErr: errors.New("get instance view error"),
wantErr: true,
},
"get instance view returns 404": {
providerID: "azure:///subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/virtualMachineScaleSets/scale-set-name/virtualMachines/instance-id",
getInstanceViewErr: &azcore.ResponseError{StatusCode: http.StatusNotFound},
wantState: updatev1alpha1.NodeStateTerminated,
},
}
for name, tc := range testCases {
t.Run(name, func(t *testing.T) {
assert := assert.New(t)
require := require.New(t)
client := Client{
virtualMachineScaleSetVMsAPI: &stubvirtualMachineScaleSetVMsAPI{
instanceView: armcomputev2.VirtualMachineScaleSetVMsClientGetInstanceViewResponse{
VirtualMachineScaleSetVMInstanceView: tc.instanceView,
},
instanceViewErr: tc.getInstanceViewErr,
},
}
gotState, err := client.GetNodeState(context.Background(), tc.providerID)
if tc.wantErr {
assert.Error(err)
return
}
require.NoError(err)
assert.Equal(tc.wantState, gotState)
})
}
}

View file

@ -0,0 +1,26 @@
/*
Copyright (c) Edgeless Systems GmbH
SPDX-License-Identifier: AGPL-3.0-only
*/
package client
import (
"errors"
"regexp"
)
// azureVMSSProviderIDRegexp describes the format of a kubernetes providerID for an azure virtual machine scale set (VMSS) VM.
var azureVMSSProviderIDRegexp = regexp.MustCompile(`^azure:///subscriptions/([^/]+)/resourceGroups/([^/]+)/providers/Microsoft.Compute/virtualMachineScaleSets/([^/]+)/virtualMachines/([^/]+)$`)
// scaleSetInformationFromProviderID splits a provider's id belonging to an azure scale set into core components.
// A providerID for scale set VMs is build after the following schema:
// - 'azure:///subscriptions/<subscription-id>/resourceGroups/<resource-group>/providers/Microsoft.Compute/virtualMachineScaleSets/<scale-set-name>/virtualMachines/<instance-id>'
func scaleSetInformationFromProviderID(providerID string) (subscriptionID, resourceGroup, scaleSet, instanceID string, err error) {
matches := azureVMSSProviderIDRegexp.FindStringSubmatch(providerID)
if len(matches) != 5 {
return "", "", "", "", errors.New("error splitting providerID")
}
return matches[1], matches[2], matches[3], matches[4], nil
}

View file

@ -0,0 +1,58 @@
/*
Copyright (c) Edgeless Systems GmbH
SPDX-License-Identifier: AGPL-3.0-only
*/
package client
import (
"testing"
"github.com/stretchr/testify/assert"
)
func TestScaleSetInformationFromProviderID(t *testing.T) {
testCases := map[string]struct {
providerID string
wantSubscriptionID string
wantResourceGroup string
wantScaleSet string
wantInstanceID string
wantErr bool
}{
"providerID for scale set instance works": {
providerID: "azure:///subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/virtualMachineScaleSets/scale-set-name/virtualMachines/instance-id",
wantSubscriptionID: "subscription-id",
wantResourceGroup: "resource-group",
wantScaleSet: "scale-set-name",
wantInstanceID: "instance-id",
},
"providerID for individual instance must fail": {
providerID: "azure:///subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/virtualMachines/instance-name",
wantErr: true,
},
"providerID is malformed": {
providerID: "malformed-provider-id",
wantErr: true,
},
}
for name, tc := range testCases {
t.Run(name, func(t *testing.T) {
assert := assert.New(t)
subscriptionID, resourceGroup, scaleSet, instanceName, err := scaleSetInformationFromProviderID(tc.providerID)
if tc.wantErr {
assert.Error(err)
return
}
assert.NoError(err)
assert.Equal(tc.wantSubscriptionID, subscriptionID)
assert.Equal(tc.wantResourceGroup, resourceGroup)
assert.Equal(tc.wantScaleSet, scaleSet)
assert.Equal(tc.wantInstanceID, instanceName)
})
}
}

View file

@ -0,0 +1,122 @@
/*
Copyright (c) Edgeless Systems GmbH
SPDX-License-Identifier: AGPL-3.0-only
*/
package client
import (
"context"
"fmt"
"strings"
"github.com/Azure/azure-sdk-for-go/sdk/azcore/to"
"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v2"
)
// GetScalingGroupImage returns the image URI of the scaling group.
func (c *Client) GetScalingGroupImage(ctx context.Context, scalingGroupID string) (string, error) {
_, resourceGroup, scaleSet, err := splitVMSSID(scalingGroupID)
if err != nil {
return "", err
}
res, err := c.scaleSetsAPI.Get(ctx, resourceGroup, scaleSet, nil)
if err != nil {
return "", err
}
if res.Properties == nil ||
res.Properties.VirtualMachineProfile == nil ||
res.Properties.VirtualMachineProfile.StorageProfile == nil ||
res.Properties.VirtualMachineProfile.StorageProfile.ImageReference == nil ||
res.Properties.VirtualMachineProfile.StorageProfile.ImageReference.ID == nil && res.Properties.VirtualMachineProfile.StorageProfile.ImageReference.CommunityGalleryImageID == nil {
return "", fmt.Errorf("scalet set %q does not have valid image reference", scalingGroupID)
}
if res.Properties.VirtualMachineProfile.StorageProfile.ImageReference.ID != nil {
return *res.Properties.VirtualMachineProfile.StorageProfile.ImageReference.ID, nil
}
return *res.Properties.VirtualMachineProfile.StorageProfile.ImageReference.CommunityGalleryImageID, nil
}
// SetScalingGroupImage sets the image URI of the scaling group.
func (c *Client) SetScalingGroupImage(ctx context.Context, scalingGroupID, imageURI string) error {
_, resourceGroup, scaleSet, err := splitVMSSID(scalingGroupID)
if err != nil {
return err
}
poller, err := c.scaleSetsAPI.BeginUpdate(ctx, resourceGroup, scaleSet, armcompute.VirtualMachineScaleSetUpdate{
Properties: &armcompute.VirtualMachineScaleSetUpdateProperties{
VirtualMachineProfile: &armcompute.VirtualMachineScaleSetUpdateVMProfile{
StorageProfile: &armcompute.VirtualMachineScaleSetUpdateStorageProfile{
ImageReference: imageReferenceFromImage(imageURI),
},
},
},
}, nil)
if err != nil {
return err
}
if _, err := poller.PollUntilDone(ctx, nil); err != nil {
return err
}
return nil
}
// GetScalingGroupName retrieves the name of a scaling group, as expected by Kubernetes.
// This keeps the casing of the original name, but Kubernetes requires the name to be lowercase,
// so use strings.ToLower() on the result if using the name in a Kubernetes context.
func (c *Client) GetScalingGroupName(scalingGroupID string) (string, error) {
_, _, scaleSet, err := splitVMSSID(scalingGroupID)
if err != nil {
return "", fmt.Errorf("getting scaling group name: %w", err)
}
return scaleSet, nil
}
// GetAutoscalingGroupName retrieves the name of a scaling group as needed by the cluster-autoscaler.
func (c *Client) GetAutoscalingGroupName(scalingGroupID string) (string, error) {
return c.GetScalingGroupName(scalingGroupID)
}
// ListScalingGroups retrieves a list of scaling groups for the cluster.
func (c *Client) ListScalingGroups(ctx context.Context, uid string) (controlPlaneGroupIDs []string, workerGroupIDs []string, err error) {
pager := c.scaleSetsAPI.NewListPager(c.config.ResourceGroup, nil)
for pager.More() {
page, err := pager.NextPage(ctx)
if err != nil {
return nil, nil, fmt.Errorf("paging scale sets: %w", err)
}
for _, scaleSet := range page.Value {
if scaleSet == nil || scaleSet.ID == nil {
continue
}
if scaleSet.Tags == nil || scaleSet.Tags["constellation-uid"] == nil || *scaleSet.Tags["constellation-uid"] != uid {
continue
}
if err != nil {
return nil, nil, fmt.Errorf("getting scaling group name: %w", err)
}
switch *scaleSet.Tags["constellation-role"] {
case "control-plane", "controlplane":
controlPlaneGroupIDs = append(controlPlaneGroupIDs, *scaleSet.ID)
case "worker":
workerGroupIDs = append(workerGroupIDs, *scaleSet.ID)
}
}
}
return controlPlaneGroupIDs, workerGroupIDs, nil
}
func imageReferenceFromImage(img string) *armcompute.ImageReference {
ref := &armcompute.ImageReference{}
if strings.HasPrefix(img, "/CommunityGalleries") {
ref.CommunityGalleryImageID = to.Ptr(img)
} else {
ref.ID = to.Ptr(img)
}
return ref
}

View file

@ -0,0 +1,288 @@
/*
Copyright (c) Edgeless Systems GmbH
SPDX-License-Identifier: AGPL-3.0-only
*/
package client
import (
"context"
"errors"
"testing"
"github.com/Azure/azure-sdk-for-go/sdk/azcore/to"
armcomputev2 "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v2"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestGetScalingGroupImage(t *testing.T) {
testCases := map[string]struct {
scalingGroupID string
scaleSet armcomputev2.VirtualMachineScaleSet
getScaleSetErr error
wantImage string
wantErr bool
}{
"getting image works": {
scalingGroupID: "/subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/virtualMachineScaleSets/scale-set-name",
scaleSet: armcomputev2.VirtualMachineScaleSet{
Properties: &armcomputev2.VirtualMachineScaleSetProperties{
VirtualMachineProfile: &armcomputev2.VirtualMachineScaleSetVMProfile{
StorageProfile: &armcomputev2.VirtualMachineScaleSetStorageProfile{
ImageReference: &armcomputev2.ImageReference{
ID: to.Ptr("/subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/images/image-name"),
},
},
},
},
},
wantImage: "/subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/images/image-name",
},
"getting community image works": {
scalingGroupID: "/subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/virtualMachineScaleSets/scale-set-name",
scaleSet: armcomputev2.VirtualMachineScaleSet{
Properties: &armcomputev2.VirtualMachineScaleSetProperties{
VirtualMachineProfile: &armcomputev2.VirtualMachineScaleSetVMProfile{
StorageProfile: &armcomputev2.VirtualMachineScaleSetStorageProfile{
ImageReference: &armcomputev2.ImageReference{
CommunityGalleryImageID: to.Ptr("/CommunityGalleries/gallery-name/Images/image-name/Versions/1.2.3"),
},
},
},
},
},
wantImage: "/CommunityGalleries/gallery-name/Images/image-name/Versions/1.2.3",
},
"splitting scalingGroupID fails": {
scalingGroupID: "invalid",
wantErr: true,
},
"get scale set fails": {
scalingGroupID: "/subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/virtualMachineScaleSets/scale-set-name",
getScaleSetErr: errors.New("get scale set error"),
wantErr: true,
},
"scale set is invalid": {
scalingGroupID: "/subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/virtualMachineScaleSets/scale-set-name",
wantErr: true,
},
}
for name, tc := range testCases {
t.Run(name, func(t *testing.T) {
assert := assert.New(t)
require := require.New(t)
client := Client{
scaleSetsAPI: &stubScaleSetsAPI{
scaleSet: armcomputev2.VirtualMachineScaleSetsClientGetResponse{
VirtualMachineScaleSet: tc.scaleSet,
},
getErr: tc.getScaleSetErr,
},
}
gotImage, err := client.GetScalingGroupImage(context.Background(), tc.scalingGroupID)
if tc.wantErr {
assert.Error(err)
return
}
require.NoError(err)
assert.Equal(tc.wantImage, gotImage)
})
}
}
func TestSetScalingGroupImage(t *testing.T) {
testCases := map[string]struct {
scalingGroupID string
imageURI string
updateErr error
resultErr error
wantErr bool
}{
"setting image works": {
scalingGroupID: "/subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/virtualMachineScaleSets/scale-set-name",
imageURI: "/subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/images/image-name-2",
},
"splitting scalingGroupID fails": {
scalingGroupID: "invalid",
wantErr: true,
},
"beginning update fails": {
scalingGroupID: "/subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/virtualMachineScaleSets/scale-set-name",
imageURI: "/subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/images/image-name-2",
updateErr: errors.New("update error"),
wantErr: true,
},
"retrieving polling result fails": {
scalingGroupID: "/subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/virtualMachineScaleSets/scale-set-name",
imageURI: "/subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/images/image-name-2",
resultErr: errors.New("result error"),
wantErr: true,
},
}
for name, tc := range testCases {
t.Run(name, func(t *testing.T) {
assert := assert.New(t)
require := require.New(t)
client := Client{
scaleSetsAPI: &stubScaleSetsAPI{
updateErr: tc.updateErr,
resultErr: tc.resultErr,
},
}
err := client.SetScalingGroupImage(context.Background(), tc.scalingGroupID, tc.imageURI)
if tc.wantErr {
assert.Error(err)
return
}
require.NoError(err)
})
}
}
func TestGetScalingGroupName(t *testing.T) {
testCases := map[string]struct {
scalingGroupID string
wantName string
wantErr bool
}{
"getting name works": {
scalingGroupID: "/subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/virtualMachineScaleSets/scale-set-name",
wantName: "scale-set-name",
},
"uppercase name isn't lowercased": {
scalingGroupID: "/subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/virtualMachineScaleSets/SCALE-SET-NAME",
wantName: "SCALE-SET-NAME",
},
"splitting scalingGroupID fails": {
scalingGroupID: "invalid",
wantErr: true,
},
}
for name, tc := range testCases {
t.Run(name, func(t *testing.T) {
assert := assert.New(t)
require := require.New(t)
client := Client{}
gotName, err := client.GetScalingGroupName(tc.scalingGroupID)
if tc.wantErr {
assert.Error(err)
return
}
require.NoError(err)
assert.Equal(tc.wantName, gotName)
})
}
}
func TestListScalingGroups(t *testing.T) {
testCases := map[string]struct {
scaleSet armcomputev2.VirtualMachineScaleSet
fetchPageErr error
wantControlPlanes []string
wantWorkers []string
wantErr bool
}{
"listing control-plane works": {
scaleSet: armcomputev2.VirtualMachineScaleSet{
ID: to.Ptr("/subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/virtualMachineScaleSets/constellation-scale-set-control-planes-uid"),
Tags: map[string]*string{
"constellation-uid": to.Ptr("uid"),
"constellation-role": to.Ptr("control-plane"),
},
},
wantControlPlanes: []string{"/subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/virtualMachineScaleSets/constellation-scale-set-control-planes-uid"},
},
"listing worker works": {
scaleSet: armcomputev2.VirtualMachineScaleSet{
ID: to.Ptr("/subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/virtualMachineScaleSets/constellation-scale-set-workers-uid"),
Tags: map[string]*string{
"constellation-uid": to.Ptr("uid"),
"constellation-role": to.Ptr("worker"),
},
},
wantWorkers: []string{"/subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/virtualMachineScaleSets/constellation-scale-set-workers-uid"},
},
"listing is not dependent on resource name": {
scaleSet: armcomputev2.VirtualMachineScaleSet{
ID: to.Ptr("/subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/virtualMachineScaleSets/some-scale-set"),
Tags: map[string]*string{
"constellation-uid": to.Ptr("uid"),
"constellation-role": to.Ptr("control-plane"),
},
},
wantControlPlanes: []string{"/subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/virtualMachineScaleSets/some-scale-set"},
},
"listing other works": {
scaleSet: armcomputev2.VirtualMachineScaleSet{
ID: to.Ptr("/subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/virtualMachineScaleSets/other"),
},
},
"fetching scale sets fails": {
fetchPageErr: errors.New("fetch page error"),
wantErr: true,
},
"scale set is invalid": {},
}
for name, tc := range testCases {
t.Run(name, func(t *testing.T) {
assert := assert.New(t)
require := require.New(t)
client := Client{
scaleSetsAPI: &stubScaleSetsAPI{
pager: &stubVMSSPager{
list: []armcomputev2.VirtualMachineScaleSet{tc.scaleSet},
fetchErr: tc.fetchPageErr,
},
},
}
gotControlPlanes, gotWorkers, err := client.ListScalingGroups(context.Background(), "uid")
if tc.wantErr {
assert.Error(err)
return
}
require.NoError(err)
assert.ElementsMatch(tc.wantControlPlanes, gotControlPlanes)
assert.ElementsMatch(tc.wantWorkers, gotWorkers)
})
}
}
func TestImageReferenceFromImage(t *testing.T) {
testCases := map[string]struct {
img string
wantID *string
wantCommunityID *string
}{
"ID": {
img: "/subscriptions/0d202bbb-4fa7-4af8-8125-58c269a05435/resourceGroups/constellation-images/providers/Microsoft.Compute/galleries/Constellation/images/constellation/versions/1.5.0",
wantID: to.Ptr("/subscriptions/0d202bbb-4fa7-4af8-8125-58c269a05435/resourceGroups/constellation-images/providers/Microsoft.Compute/galleries/Constellation/images/constellation/versions/1.5.0"),
wantCommunityID: nil,
},
"Community": {
img: "/CommunityGalleries/ConstellationCVM-728bd310-e898-4450-a1ed-21cf2fb0d735/Images/feat-azure-cvm-sharing/Versions/2022.0826.084922",
wantID: nil,
wantCommunityID: to.Ptr("/CommunityGalleries/ConstellationCVM-728bd310-e898-4450-a1ed-21cf2fb0d735/Images/feat-azure-cvm-sharing/Versions/2022.0826.084922"),
},
}
for name, tc := range testCases {
t.Run(name, func(t *testing.T) {
assert := assert.New(t)
ref := imageReferenceFromImage(tc.img)
assert.Equal(tc.wantID, ref.ID)
assert.Equal(tc.wantCommunityID, ref.CommunityGalleryImageID)
})
}
}

View file

@ -0,0 +1,30 @@
/*
Copyright (c) Edgeless Systems GmbH
SPDX-License-Identifier: AGPL-3.0-only
*/
package client
import (
"fmt"
"regexp"
)
// vmssIDRegexp is describes the format of an azure virtual machine scale set (VMSS) id.
var vmssIDRegexp = regexp.MustCompile(`^/subscriptions/([^/]+)/resourceGroups/([^/]+)/providers/Microsoft.Compute/virtualMachineScaleSets/([^/]+)$`)
// joinVMSSID joins scale set parameters to generate a virtual machine scale set (VMSS) ID.
// Format: /subscriptions/<subscription>/resourceGroups/<resource-group>/providers/Microsoft.Compute/virtualMachineScaleSets/<scale-set> .
func joinVMSSID(subscriptionID, resourceGroup, scaleSet string) string {
return fmt.Sprintf("/subscriptions/%s/resourceGroups/%s/providers/Microsoft.Compute/virtualMachineScaleSets/%s", subscriptionID, resourceGroup, scaleSet)
}
// splitVMSSID splits a virtual machine scale set (VMSS) ID into its component.
func splitVMSSID(vmssID string) (subscriptionID, resourceGroup, scaleSet string, err error) {
matches := vmssIDRegexp.FindStringSubmatch(vmssID)
if len(matches) != 4 {
return "", "", "", fmt.Errorf("error splitting vmssID")
}
return matches[1], matches[2], matches[3], nil
}

View file

@ -0,0 +1,59 @@
/*
Copyright (c) Edgeless Systems GmbH
SPDX-License-Identifier: AGPL-3.0-only
*/
package client
import (
"testing"
"github.com/stretchr/testify/assert"
)
func TestJoinVMSSID(t *testing.T) {
assert.Equal(t,
"/subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/virtualMachineScaleSets/scale-set",
joinVMSSID("subscription-id", "resource-group", "scale-set"))
}
func TestSplitVMSSID(t *testing.T) {
testCases := map[string]struct {
vmssID string
wantSubscriptionID string
wantResourceGroup string
wantScaleSet string
wantInstanceID string
wantErr bool
}{
"vmssID can be splitted": {
vmssID: "/subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/virtualMachineScaleSets/scale-set-name",
wantSubscriptionID: "subscription-id",
wantResourceGroup: "resource-group",
wantScaleSet: "scale-set-name",
wantInstanceID: "instance-id",
},
"vmssID is malformed": {
vmssID: "malformed-vmss-id",
wantErr: true,
},
}
for name, tc := range testCases {
t.Run(name, func(t *testing.T) {
assert := assert.New(t)
subscriptionID, resourceGroup, scaleSet, err := splitVMSSID(tc.vmssID)
if tc.wantErr {
assert.Error(err)
return
}
assert.NoError(err)
assert.Equal(tc.wantSubscriptionID, subscriptionID)
assert.Equal(tc.wantResourceGroup, resourceGroup)
assert.Equal(tc.wantScaleSet, scaleSet)
})
}
}

View file

@ -0,0 +1,91 @@
/*
Copyright (c) Edgeless Systems GmbH
SPDX-License-Identifier: AGPL-3.0-only
*/
package client
import (
"context"
"fmt"
updatev1alpha1 "github.com/edgelesssys/constellation/operators/constellation-node-operator/v2/api/v1alpha1"
)
const (
controlPlanesID = "control-planes-id"
workersID = "workers-id"
)
// Client is a stub client providing the minimal implementation to set up the initial resources.
type Client struct{}
// GetNodeImage retrieves the image currently used by a node.
func (c *Client) GetNodeImage(ctx context.Context, providerID string) (string, error) {
panic("not implemented")
}
// GetScalingGroupID retrieves the scaling group that a node is part of.
func (c *Client) GetScalingGroupID(ctx context.Context, providerID string) (string, error) {
panic("not implemented")
}
// CreateNode creates a new node inside a specified scaling group at the CSP and returns its future name and provider id.
func (c *Client) CreateNode(ctx context.Context, scalingGroupID string) (nodeName, providerID string, err error) {
panic("not implemented")
}
// DeleteNode starts the termination of the node at the CSP.
func (c *Client) DeleteNode(ctx context.Context, providerID string) error {
panic("not implemented")
}
// GetNodeState retrieves the state of a pending node from a CSP.
func (c *Client) GetNodeState(ctx context.Context, providerID string) (updatev1alpha1.CSPNodeState, error) {
panic("not implemented")
}
// SetScalingGroupImage sets the image to be used by newly created nodes in a scaling group.
func (c *Client) SetScalingGroupImage(ctx context.Context, scalingGroupID, image string) error {
panic("not implemented")
}
// GetScalingGroupImage retrieves the image currently used by a scaling group.
func (c *Client) GetScalingGroupImage(ctx context.Context, scalingGroupID string) (string, error) {
return "unsupportedCSP", nil
}
// GetScalingGroupName retrieves the name of a scaling group.
func (c *Client) GetScalingGroupName(scalingGroupID string) (string, error) {
switch scalingGroupID {
case controlPlanesID:
return controlPlanesID, nil
case workersID:
return workersID, nil
default:
return "", fmt.Errorf("unknown scaling group id %s", scalingGroupID)
}
}
// GetAutoscalingGroupName retrieves the name of a scaling group as needed by the cluster-autoscaler.
func (c *Client) GetAutoscalingGroupName(scalingGroupID string) (string, error) {
switch scalingGroupID {
case controlPlanesID:
return controlPlanesID, nil
case workersID:
return workersID, nil
default:
return "", fmt.Errorf("unknown scaling group id %s", scalingGroupID)
}
}
// ListScalingGroups retrieves a list of scaling groups for the cluster.
func (c *Client) ListScalingGroups(ctx context.Context, uid string) (controlPlaneGroupIDs []string, workerGroupIDs []string, err error) {
return []string{controlPlanesID}, []string{workersID}, nil
}
// AutoscalingCloudProvider returns the cloud-provider name as used by k8s cluster-autoscaler.
func (c *Client) AutoscalingCloudProvider() string {
return "unsupportedCSP"
}

View file

@ -0,0 +1,74 @@
/*
Copyright (c) Edgeless Systems GmbH
SPDX-License-Identifier: AGPL-3.0-only
*/
package client
import (
"context"
compute "cloud.google.com/go/compute/apiv1"
"cloud.google.com/go/compute/apiv1/computepb"
"github.com/googleapis/gax-go/v2"
)
type projectAPI interface {
Close() error
Get(ctx context.Context, req *computepb.GetProjectRequest,
opts ...gax.CallOption) (*computepb.Project, error)
}
type instanceAPI interface {
Close() error
Get(ctx context.Context, req *computepb.GetInstanceRequest,
opts ...gax.CallOption) (*computepb.Instance, error)
}
type instanceTemplateAPI interface {
Close() error
Get(ctx context.Context, req *computepb.GetInstanceTemplateRequest,
opts ...gax.CallOption) (*computepb.InstanceTemplate, error)
Delete(ctx context.Context, req *computepb.DeleteInstanceTemplateRequest,
opts ...gax.CallOption) (Operation, error)
Insert(ctx context.Context, req *computepb.InsertInstanceTemplateRequest,
opts ...gax.CallOption) (Operation, error)
}
type instanceGroupManagersAPI interface {
Close() error
Get(ctx context.Context, req *computepb.GetInstanceGroupManagerRequest,
opts ...gax.CallOption) (*computepb.InstanceGroupManager, error)
AggregatedList(ctx context.Context, req *computepb.AggregatedListInstanceGroupManagersRequest,
opts ...gax.CallOption) InstanceGroupManagerScopedListIterator
SetInstanceTemplate(ctx context.Context, req *computepb.SetInstanceTemplateInstanceGroupManagerRequest,
opts ...gax.CallOption) (Operation, error)
CreateInstances(ctx context.Context, req *computepb.CreateInstancesInstanceGroupManagerRequest,
opts ...gax.CallOption) (Operation, error)
DeleteInstances(ctx context.Context, req *computepb.DeleteInstancesInstanceGroupManagerRequest,
opts ...gax.CallOption) (Operation, error)
}
type diskAPI interface {
Close() error
Get(ctx context.Context, req *computepb.GetDiskRequest,
opts ...gax.CallOption) (*computepb.Disk, error)
}
// Operation describes a generic protobuf operation that can be waited for.
type Operation interface {
Proto() *computepb.Operation
Done() bool
Wait(ctx context.Context, opts ...gax.CallOption) error
}
// InstanceGroupManagerScopedListIterator can list the Next InstanceGroupManagersScopedListPair.
type InstanceGroupManagerScopedListIterator interface {
Next() (compute.InstanceGroupManagersScopedListPair, error)
}
type prng interface {
// Intn returns, as an int, a non-negative pseudo-random number in the half-open interval [0,n). It panics if n <= 0.
Intn(n int) int
}

View file

@ -0,0 +1,12 @@
/*
Copyright (c) Edgeless Systems GmbH
SPDX-License-Identifier: AGPL-3.0-only
*/
package client
// AutoscalingCloudProvider returns the cloud-provider name as used by k8s cluster-autoscaler.
func (c *Client) AutoscalingCloudProvider() string {
return "gce"
}

View file

@ -0,0 +1,106 @@
/*
Copyright (c) Edgeless Systems GmbH
SPDX-License-Identifier: AGPL-3.0-only
*/
package client
import (
"context"
"math/rand"
"time"
compute "cloud.google.com/go/compute/apiv1"
"github.com/spf13/afero"
"go.uber.org/multierr"
)
// Client is a client for the Google Compute Engine.
type Client struct {
projectID string
projectAPI
instanceAPI
instanceTemplateAPI
instanceGroupManagersAPI
diskAPI
// prng is a pseudo-random number generator seeded with time. Not used for security.
prng
}
// New creates a new client for the Google Compute Engine.
func New(ctx context.Context, configPath string) (*Client, error) {
projectID, err := loadProjectID(afero.NewOsFs(), configPath)
if err != nil {
return nil, err
}
var closers []closer
projectAPI, err := compute.NewProjectsRESTClient(ctx)
if err != nil {
return nil, err
}
closers = append(closers, projectAPI)
insAPI, err := compute.NewInstancesRESTClient(ctx)
if err != nil {
_ = closeAll(closers)
return nil, err
}
closers = append(closers, insAPI)
templAPI, err := compute.NewInstanceTemplatesRESTClient(ctx)
if err != nil {
_ = closeAll(closers)
return nil, err
}
closers = append(closers, templAPI)
groupAPI, err := compute.NewInstanceGroupManagersRESTClient(ctx)
if err != nil {
_ = closeAll(closers)
return nil, err
}
closers = append(closers, groupAPI)
diskAPI, err := compute.NewDisksRESTClient(ctx)
if err != nil {
_ = closeAll(closers)
return nil, err
}
return &Client{
projectID: projectID,
projectAPI: projectAPI,
instanceAPI: insAPI,
instanceTemplateAPI: &instanceTemplateClient{templAPI},
instanceGroupManagersAPI: &instanceGroupManagersClient{groupAPI},
diskAPI: diskAPI,
prng: rand.New(rand.NewSource(int64(time.Now().Nanosecond()))),
}, nil
}
// Close closes the client's connection.
func (c *Client) Close() error {
closers := []closer{
c.projectAPI,
c.instanceAPI,
c.instanceTemplateAPI,
c.instanceGroupManagersAPI,
c.diskAPI,
}
return closeAll(closers)
}
type closer interface {
Close() error
}
// closeAll closes all closers, even if an error occurs.
//
// Errors are collected and a composed error is returned.
func closeAll(closers []closer) error {
// Since this function is intended to be deferred, it will always call all
// close operations, even if a previous operation failed.
var errs error
for _, closer := range closers {
errs = multierr.Append(errs, closer.Close())
}
return errs
}

View file

@ -0,0 +1,204 @@
/*
Copyright (c) Edgeless Systems GmbH
SPDX-License-Identifier: AGPL-3.0-only
*/
package client
import (
"context"
compute "cloud.google.com/go/compute/apiv1"
"cloud.google.com/go/compute/apiv1/computepb"
"github.com/googleapis/gax-go/v2"
"google.golang.org/api/iterator"
"google.golang.org/protobuf/proto"
)
type stubProjectAPI struct {
project *computepb.Project
getErr error
}
func (a stubProjectAPI) Close() error {
return nil
}
func (a stubProjectAPI) Get(ctx context.Context, req *computepb.GetProjectRequest,
opts ...gax.CallOption,
) (*computepb.Project, error) {
return a.project, a.getErr
}
type stubInstanceAPI struct {
instance *computepb.Instance
getErr error
}
func (a stubInstanceAPI) Close() error {
return nil
}
func (a stubInstanceAPI) Get(ctx context.Context, req *computepb.GetInstanceRequest,
opts ...gax.CallOption,
) (*computepb.Instance, error) {
return a.instance, a.getErr
}
type stubInstanceTemplateAPI struct {
template *computepb.InstanceTemplate
getErr error
deleteErr error
insertErr error
}
func (a stubInstanceTemplateAPI) Close() error {
return nil
}
func (a stubInstanceTemplateAPI) Get(ctx context.Context, req *computepb.GetInstanceTemplateRequest,
opts ...gax.CallOption,
) (*computepb.InstanceTemplate, error) {
return a.template, a.getErr
}
func (a stubInstanceTemplateAPI) Delete(ctx context.Context, req *computepb.DeleteInstanceTemplateRequest,
opts ...gax.CallOption,
) (Operation, error) {
return &stubOperation{
&computepb.Operation{
Name: proto.String("name"),
},
}, a.deleteErr
}
func (a stubInstanceTemplateAPI) Insert(ctx context.Context, req *computepb.InsertInstanceTemplateRequest,
opts ...gax.CallOption,
) (Operation, error) {
return &stubOperation{
&computepb.Operation{
Name: proto.String("name"),
},
}, a.insertErr
}
type stubInstanceGroupManagersAPI struct {
instanceGroupManager *computepb.InstanceGroupManager
getErr error
aggregatedListErr error
setInstanceTemplateErr error
createInstancesErr error
deleteInstancesErr error
}
func (a stubInstanceGroupManagersAPI) Close() error {
return nil
}
func (a stubInstanceGroupManagersAPI) Get(ctx context.Context, req *computepb.GetInstanceGroupManagerRequest,
opts ...gax.CallOption,
) (*computepb.InstanceGroupManager, error) {
return a.instanceGroupManager, a.getErr
}
func (a stubInstanceGroupManagersAPI) AggregatedList(ctx context.Context, req *computepb.AggregatedListInstanceGroupManagersRequest,
opts ...gax.CallOption,
) InstanceGroupManagerScopedListIterator {
return &stubInstanceGroupManagerScopedListIterator{
pairs: []compute.InstanceGroupManagersScopedListPair{
{
Key: "key",
Value: &computepb.InstanceGroupManagersScopedList{
InstanceGroupManagers: []*computepb.InstanceGroupManager{
a.instanceGroupManager,
},
},
},
},
nextErr: a.aggregatedListErr,
}
}
func (a stubInstanceGroupManagersAPI) SetInstanceTemplate(ctx context.Context, req *computepb.SetInstanceTemplateInstanceGroupManagerRequest,
opts ...gax.CallOption,
) (Operation, error) {
return &stubOperation{
&computepb.Operation{
Name: proto.String("name"),
},
}, a.setInstanceTemplateErr
}
func (a stubInstanceGroupManagersAPI) CreateInstances(ctx context.Context, req *computepb.CreateInstancesInstanceGroupManagerRequest,
opts ...gax.CallOption,
) (Operation, error) {
return &stubOperation{
&computepb.Operation{
Name: proto.String("name"),
},
}, a.createInstancesErr
}
func (a stubInstanceGroupManagersAPI) DeleteInstances(ctx context.Context, req *computepb.DeleteInstancesInstanceGroupManagerRequest,
opts ...gax.CallOption,
) (Operation, error) {
if a.deleteInstancesErr != nil {
return nil, a.deleteInstancesErr
}
return &stubOperation{
&computepb.Operation{
Name: proto.String("name"),
},
}, nil
}
type stubDiskAPI struct {
disk *computepb.Disk
getErr error
}
func (a stubDiskAPI) Close() error {
return nil
}
func (a stubDiskAPI) Get(ctx context.Context, req *computepb.GetDiskRequest,
opts ...gax.CallOption,
) (*computepb.Disk, error) {
return a.disk, a.getErr
}
type stubOperation struct {
*computepb.Operation
}
func (o *stubOperation) Proto() *computepb.Operation {
return o.Operation
}
func (o *stubOperation) Done() bool {
return true
}
func (o *stubOperation) Wait(ctx context.Context, opts ...gax.CallOption) error {
return nil
}
type stubInstanceGroupManagerScopedListIterator struct {
pairs []compute.InstanceGroupManagersScopedListPair
nextErr error
internalCounter int
}
func (i *stubInstanceGroupManagerScopedListIterator) Next() (compute.InstanceGroupManagersScopedListPair, error) {
if i.nextErr != nil {
return compute.InstanceGroupManagersScopedListPair{}, i.nextErr
}
if i.internalCounter >= len(i.pairs) {
return compute.InstanceGroupManagersScopedListPair{}, iterator.Done
}
pair := i.pairs[i.internalCounter]
i.internalCounter++
return pair, nil
}

View file

@ -0,0 +1,30 @@
/*
Copyright (c) Edgeless Systems GmbH
SPDX-License-Identifier: AGPL-3.0-only
*/
package client
import (
"errors"
"regexp"
"github.com/spf13/afero"
)
var projectIDRegex = regexp.MustCompile(`(?m)^project-id = (.*)$`)
// loadProjectID loads the project id from the gce config file.
func loadProjectID(fs afero.Fs, path string) (string, error) {
rawConfig, err := afero.ReadFile(fs, path)
if err != nil {
return "", err
}
// find project-id line
matches := projectIDRegex.FindStringSubmatch(string(rawConfig))
if len(matches) != 2 {
return "", errors.New("invalid config: project-id not found")
}
return matches[1], nil
}

View file

@ -0,0 +1,59 @@
/*
Copyright (c) Edgeless Systems GmbH
SPDX-License-Identifier: AGPL-3.0-only
*/
package client
import (
"testing"
"github.com/spf13/afero"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestLoadProjectID(t *testing.T) {
testCases := map[string]struct {
rawConfig string
skipWrite bool
wantProjectID string
wantErr bool
}{
"valid config": {
rawConfig: `project-id = project-id`,
wantProjectID: "project-id",
},
"invalid config": {
rawConfig: `x = y`,
wantErr: true,
},
"config is empty": {
wantErr: true,
},
"config does not exist": {
skipWrite: true,
wantErr: true,
},
}
for name, tc := range testCases {
t.Run(name, func(t *testing.T) {
assert := assert.New(t)
require := require.New(t)
fs := afero.NewMemMapFs()
if !tc.skipWrite {
require.NoError(afero.WriteFile(fs, "gce.conf", []byte(tc.rawConfig), 0o644))
}
gotProjectID, err := loadProjectID(fs, "gce.conf")
if tc.wantErr {
assert.Error(err)
return
}
require.NoError(err)
assert.Equal(tc.wantProjectID, gotProjectID)
})
}
}

View file

@ -0,0 +1,51 @@
/*
Copyright (c) Edgeless Systems GmbH
SPDX-License-Identifier: AGPL-3.0-only
*/
package client
import (
"fmt"
"regexp"
"cloud.google.com/go/compute/apiv1/computepb"
)
var (
diskSourceRegex = regexp.MustCompile(`^https://www.googleapis.com/compute/v1/projects/([^/]+)/zones/([^/]+)/disks/([^/]+)$`)
computeAPIBase = regexp.MustCompile(`^https://www.googleapis.com/compute/v1/(.+)$`)
)
// diskSourceToDiskReq converts a disk source URI to a disk request.
func diskSourceToDiskReq(diskSource string) (*computepb.GetDiskRequest, error) {
matches := diskSourceRegex.FindStringSubmatch(diskSource)
if len(matches) != 4 {
return nil, fmt.Errorf("error splitting diskSource: %v", diskSource)
}
return &computepb.GetDiskRequest{
Disk: matches[3],
Project: matches[1],
Zone: matches[2],
}, nil
}
// uriNormalize normalizes a compute API URI by removing the optional URI prefix.
// for normalization, the prefix 'https://www.googleapis.com/compute/v1/' is removed.
func uriNormalize(imageURI string) string {
matches := computeAPIBase.FindStringSubmatch(imageURI)
if len(matches) != 2 {
return imageURI
}
return matches[1]
}
// ensureURIPrefixed ensures that a compute API URI is prefixed with the optional URI prefix.
func ensureURIPrefixed(uri string) string {
matches := computeAPIBase.FindStringSubmatch(uri)
if len(matches) == 2 {
return uri
}
return "https://www.googleapis.com/compute/v1/" + uri
}

View file

@ -0,0 +1,79 @@
/*
Copyright (c) Edgeless Systems GmbH
SPDX-License-Identifier: AGPL-3.0-only
*/
package client
import (
"testing"
"cloud.google.com/go/compute/apiv1/computepb"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestDiskSourceToDiskReq(t *testing.T) {
testCases := map[string]struct {
diskSource string
wantRequest *computepb.GetDiskRequest
wantErr bool
}{
"valid request": {
diskSource: "https://www.googleapis.com/compute/v1/projects/project/zones/zone/disks/disk",
wantRequest: &computepb.GetDiskRequest{
Disk: "disk",
Project: "project",
Zone: "zone",
},
},
"invalid host": {
diskSource: "https://hostname/compute/v1/projects/project/zones/zone/disks/disk",
wantErr: true,
},
"invalid scheme": {
diskSource: "invalid://www.googleapis.com/compute/v1/projects/project/zones/zone/disks/disk",
wantErr: true,
},
}
for name, tc := range testCases {
t.Run(name, func(t *testing.T) {
assert := assert.New(t)
require := require.New(t)
request, err := diskSourceToDiskReq(tc.diskSource)
if tc.wantErr {
assert.Error(err)
return
}
require.NoError(err)
assert.Equal(tc.wantRequest, request)
})
}
}
func TestURINormalize(t *testing.T) {
testCases := map[string]struct {
imageURI string
wantNormalized string
}{
"URI with scheme and host": {
imageURI: "https://www.googleapis.com/compute/v1/projects/project/global/images/image",
wantNormalized: "projects/project/global/images/image",
},
"normalized": {
imageURI: "projects/project/global/images/image",
wantNormalized: "projects/project/global/images/image",
},
}
for name, tc := range testCases {
t.Run(name, func(t *testing.T) {
assert := assert.New(t)
normalized := uriNormalize(tc.imageURI)
assert.Equal(tc.wantNormalized, normalized)
})
}
}

View file

@ -0,0 +1,73 @@
/*
Copyright (c) Edgeless Systems GmbH
SPDX-License-Identifier: AGPL-3.0-only
*/
package client
import (
"context"
compute "cloud.google.com/go/compute/apiv1"
"cloud.google.com/go/compute/apiv1/computepb"
"github.com/googleapis/gax-go/v2"
)
type instanceTemplateClient struct {
*compute.InstanceTemplatesClient
}
func (c *instanceTemplateClient) Close() error {
return c.InstanceTemplatesClient.Close()
}
func (c *instanceTemplateClient) Delete(ctx context.Context, req *computepb.DeleteInstanceTemplateRequest,
opts ...gax.CallOption,
) (Operation, error) {
return c.InstanceTemplatesClient.Delete(ctx, req, opts...)
}
func (c *instanceTemplateClient) Insert(ctx context.Context, req *computepb.InsertInstanceTemplateRequest,
opts ...gax.CallOption,
) (Operation, error) {
return c.InstanceTemplatesClient.Insert(ctx, req, opts...)
}
type instanceGroupManagersClient struct {
*compute.InstanceGroupManagersClient
}
func (c *instanceGroupManagersClient) Close() error {
return c.InstanceGroupManagersClient.Close()
}
func (c *instanceGroupManagersClient) Get(ctx context.Context, req *computepb.GetInstanceGroupManagerRequest,
opts ...gax.CallOption,
) (*computepb.InstanceGroupManager, error) {
return c.InstanceGroupManagersClient.Get(ctx, req, opts...)
}
func (c *instanceGroupManagersClient) AggregatedList(ctx context.Context, req *computepb.AggregatedListInstanceGroupManagersRequest,
opts ...gax.CallOption,
) InstanceGroupManagerScopedListIterator {
return c.InstanceGroupManagersClient.AggregatedList(ctx, req, opts...)
}
func (c *instanceGroupManagersClient) SetInstanceTemplate(ctx context.Context, req *computepb.SetInstanceTemplateInstanceGroupManagerRequest,
opts ...gax.CallOption,
) (Operation, error) {
return c.InstanceGroupManagersClient.SetInstanceTemplate(ctx, req, opts...)
}
func (c *instanceGroupManagersClient) CreateInstances(ctx context.Context, req *computepb.CreateInstancesInstanceGroupManagerRequest,
opts ...gax.CallOption,
) (Operation, error) {
return c.InstanceGroupManagersClient.CreateInstances(ctx, req, opts...)
}
func (c *instanceGroupManagersClient) DeleteInstances(ctx context.Context, req *computepb.DeleteInstancesInstanceGroupManagerRequest,
opts ...gax.CallOption,
) (Operation, error) {
return c.InstanceGroupManagersClient.DeleteInstances(ctx, req, opts...)
}

View file

@ -0,0 +1,48 @@
/*
Copyright (c) Edgeless Systems GmbH
SPDX-License-Identifier: AGPL-3.0-only
*/
package client
import (
"context"
"fmt"
"regexp"
)
var instanceGroupIDRegex = regexp.MustCompile(`^projects/([^/]+)/zones/([^/]+)/instanceGroupManagers/([^/]+)$`)
func (c *Client) canonicalInstanceGroupID(ctx context.Context, instanceGroupID string) (string, error) {
project, zone, instanceGroup, err := splitInstanceGroupID(uriNormalize(instanceGroupID))
if err != nil {
return "", err
}
project, err = c.canonicalProjectID(ctx, project)
if err != nil {
return "", err
}
return fmt.Sprintf("projects/%s/zones/%s/instanceGroupManagers/%s", project, zone, instanceGroup), nil
}
// splitInstanceGroupID splits an instance group ID into core components.
func splitInstanceGroupID(instanceGroupID string) (project, zone, instanceGroup string, err error) {
matches := instanceGroupIDRegex.FindStringSubmatch(instanceGroupID)
if len(matches) != 4 {
return "", "", "", fmt.Errorf("error splitting instanceGroupID: %v", instanceGroupID)
}
return matches[1], matches[2], matches[3], nil
}
// generateInstanceName generates a random instance name.
func generateInstanceName(baseInstanceName string, random prng) string {
letters := []byte("abcdefghijklmnopqrstuvwxyz0123456789")
const uidLen = 4
uid := make([]byte, 0, uidLen)
for i := 0; i < uidLen; i++ {
n := random.Intn(len(letters))
uid = append(uid, letters[n])
}
return baseInstanceName + "-" + string(uid)
}

View file

@ -0,0 +1,78 @@
/*
Copyright (c) Edgeless Systems GmbH
SPDX-License-Identifier: AGPL-3.0-only
*/
package client
import (
"math/rand"
"regexp"
"testing"
"time"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestSplitInstanceGroupID(t *testing.T) {
testCases := map[string]struct {
instanceGroupID string
wantProject string
wantZone string
wantInstanceGroup string
wantErr bool
}{
"valid request": {
instanceGroupID: "projects/project/zones/zone/instanceGroupManagers/instanceGroup",
wantProject: "project",
wantZone: "zone",
wantInstanceGroup: "instanceGroup",
},
"wrong format": {
instanceGroupID: "wrong-format",
wantErr: true,
},
}
for name, tc := range testCases {
t.Run(name, func(t *testing.T) {
assert := assert.New(t)
require := require.New(t)
gotProject, gotZone, gotInstanceGroup, err := splitInstanceGroupID(tc.instanceGroupID)
if tc.wantErr {
assert.Error(err)
return
}
require.NoError(err)
assert.Equal(tc.wantProject, gotProject)
assert.Equal(tc.wantZone, gotZone)
assert.Equal(tc.wantInstanceGroup, gotInstanceGroup)
})
}
}
func TestGenerateInstanceName(t *testing.T) {
assert := assert.New(t)
baseInstanceName := "base"
gotInstanceName := generateInstanceName(baseInstanceName, &stubRng{result: 0})
assert.Equal("base-aaaa", gotInstanceName)
}
func TestGenerateInstanceNameRandomTest(t *testing.T) {
assert := assert.New(t)
instanceNameRegexp := regexp.MustCompile(`^base-[0-9a-z]{4}$`)
baseInstanceName := "base"
random := rand.New(rand.NewSource(int64(time.Now().Nanosecond())))
gotInstanceName := generateInstanceName(baseInstanceName, random)
assert.Regexp(instanceNameRegexp, gotInstanceName)
}
type stubRng struct {
result int
}
func (r *stubRng) Intn(n int) int {
return r.result
}

View file

@ -0,0 +1,53 @@
/*
Copyright (c) Edgeless Systems GmbH
SPDX-License-Identifier: AGPL-3.0-only
*/
package client
import (
"fmt"
"math"
"regexp"
"strconv"
)
var (
numberedNameRegex = regexp.MustCompile(`^(.+)-(\d+)$`)
instanceTemplateIDRegex = regexp.MustCompile(`projects/([^/]+)/global/instanceTemplates/([^/]+)`)
)
// generateInstanceTemplateName generates a unique name for an instance template by incrementing a counter.
// The name is in the format <prefix>-<counter>.
func generateInstanceTemplateName(last string) (string, error) {
if len(last) > 0 && last[len(last)-1] == '-' {
return last + "1", nil
}
matches := numberedNameRegex.FindStringSubmatch(last)
if len(matches) != 3 {
return last + "-1", nil
}
n, err := strconv.Atoi(matches[2])
if err != nil {
return "", err
}
if n < 1 || n == math.MaxInt {
return "", fmt.Errorf("invalid counter: %v", n)
}
return matches[1] + "-" + strconv.Itoa(n+1), nil
}
// splitInstanceTemplateID splits an instance template ID into its project and name components.
func splitInstanceTemplateID(instanceTemplateID string) (project, templateName string, err error) {
matches := instanceTemplateIDRegex.FindStringSubmatch(instanceTemplateID)
if len(matches) != 3 {
return "", "", fmt.Errorf("error splitting instanceTemplateID: %v", instanceTemplateID)
}
return matches[1], matches[2], nil
}
// joinInstanceTemplateURI joins a project and template name into an instance template URI.
func joinInstanceTemplateURI(project, templateName string) string {
return fmt.Sprintf("https://www.googleapis.com/compute/v1/projects/%v/global/instanceTemplates/%v", project, templateName)
}

View file

@ -0,0 +1,105 @@
/*
Copyright (c) Edgeless Systems GmbH
SPDX-License-Identifier: AGPL-3.0-only
*/
package client
import (
"fmt"
"math"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestGenerateInstanceTemplateName(t *testing.T) {
testCases := map[string]struct {
last string
wantNext string
wantErr bool
}{
"no numbering yet": {
last: "prefix",
wantNext: "prefix-1",
},
"ends in -": {
last: "prefix-",
wantNext: "prefix-1",
},
"has number": {
last: "prefix-1",
wantNext: "prefix-2",
},
"last number too small": {
last: "prefix-0",
wantErr: true,
},
"last number would overflow": {
last: fmt.Sprintf("prefix-%d", math.MaxInt),
wantErr: true,
},
"integer out of range": {
last: "prefix-999999999999999999999999999999999999999999",
wantErr: true,
},
}
for name, tc := range testCases {
t.Run(name, func(t *testing.T) {
assert := assert.New(t)
require := require.New(t)
gotNext, err := generateInstanceTemplateName(tc.last)
if tc.wantErr {
assert.Error(err)
return
}
require.NoError(err)
assert.Equal(tc.wantNext, gotNext)
})
}
}
func TestSplitInstanceTemplateID(t *testing.T) {
testCases := map[string]struct {
instanceTemplateID string
wantProject string
wantTemplateName string
wantErr bool
}{
"valid request": {
instanceTemplateID: "projects/project/global/instanceTemplates/template",
wantProject: "project",
wantTemplateName: "template",
},
"wrong format": {
instanceTemplateID: "wrong-format",
wantErr: true,
},
}
for name, tc := range testCases {
t.Run(name, func(t *testing.T) {
assert := assert.New(t)
require := require.New(t)
gotProject, gotTemplateName, err := splitInstanceTemplateID(tc.instanceTemplateID)
if tc.wantErr {
assert.Error(err)
return
}
require.NoError(err)
assert.Equal(tc.wantProject, gotProject)
assert.Equal(tc.wantTemplateName, gotTemplateName)
})
}
}
func TestJoinInstanceTemplateID(t *testing.T) {
assert := assert.New(t)
project := "project"
templateName := "template"
wantInstanceTemplateURI := "https://www.googleapis.com/compute/v1/projects/project/global/instanceTemplates/template"
gotInstancetemplateURI := joinInstanceTemplateURI(project, templateName)
assert.Equal(wantInstanceTemplateURI, gotInstancetemplateURI)
}

View file

@ -0,0 +1,27 @@
/*
Copyright (c) Edgeless Systems GmbH
SPDX-License-Identifier: AGPL-3.0-only
*/
package client
import (
"cloud.google.com/go/compute/apiv1/computepb"
)
// getMetadataByKey returns the value of the metadata key in the given metadata.
func getMetadataByKey(metadata *computepb.Metadata, key string) string {
if metadata == nil {
return ""
}
for _, item := range metadata.Items {
if item.Key == nil || item.Value == nil {
continue
}
if *item.Key == key {
return *item.Value
}
}
return ""
}

View file

@ -0,0 +1,63 @@
/*
Copyright (c) Edgeless Systems GmbH
SPDX-License-Identifier: AGPL-3.0-only
*/
package client
import (
"testing"
"cloud.google.com/go/compute/apiv1/computepb"
"github.com/stretchr/testify/assert"
"google.golang.org/protobuf/proto"
)
func TestGetMetadataByKey(t *testing.T) {
testCases := map[string]struct {
metadata *computepb.Metadata
key string
wantValue string
}{
"metadata has key": {
metadata: &computepb.Metadata{
Items: []*computepb.Items{
{Key: proto.String("key"), Value: proto.String("value")},
},
},
key: "key",
wantValue: "value",
},
"metadata does not have key": {
metadata: &computepb.Metadata{
Items: []*computepb.Items{
{Key: proto.String("otherkey"), Value: proto.String("value")},
},
},
key: "key",
wantValue: "",
},
"metadata contains invalid item": {
metadata: &computepb.Metadata{
Items: []*computepb.Items{
{},
{Key: proto.String("key"), Value: proto.String("value")},
},
},
key: "key",
wantValue: "value",
},
"metadata is nil": {
key: "key",
wantValue: "",
},
}
for name, tc := range testCases {
t.Run(name, func(t *testing.T) {
assert := assert.New(t)
assert.Equal(tc.wantValue, getMetadataByKey(tc.metadata, tc.key))
})
}
}

View file

@ -0,0 +1,149 @@
/*
Copyright (c) Edgeless Systems GmbH
SPDX-License-Identifier: AGPL-3.0-only
*/
package client
import (
"context"
"fmt"
"cloud.google.com/go/compute/apiv1/computepb"
"google.golang.org/protobuf/proto"
)
// GetNodeImage returns the image name of the node.
func (c *Client) GetNodeImage(ctx context.Context, providerID string) (string, error) {
project, zone, instanceName, err := splitProviderID(providerID)
if err != nil {
return "", err
}
project, err = c.canonicalProjectID(ctx, project)
if err != nil {
return "", err
}
instance, err := c.instanceAPI.Get(ctx, &computepb.GetInstanceRequest{
Instance: instanceName,
Project: project,
Zone: zone,
})
if err != nil {
return "", err
}
// first disk is always the boot disk
if len(instance.Disks) < 1 {
return "", fmt.Errorf("instance %v has no disks", instanceName)
}
if instance.Disks[0] == nil || instance.Disks[0].Source == nil {
return "", fmt.Errorf("instance %q has invalid disk", instanceName)
}
diskReq, err := diskSourceToDiskReq(*instance.Disks[0].Source)
if err != nil {
return "", err
}
disk, err := c.diskAPI.Get(ctx, diskReq)
if err != nil {
return "", err
}
if disk.SourceImage == nil {
return "", fmt.Errorf("disk %q has no source image", diskReq.Disk)
}
return uriNormalize(*disk.SourceImage), nil
}
// GetScalingGroupID returns the scaling group ID of the node.
func (c *Client) GetScalingGroupID(ctx context.Context, providerID string) (string, error) {
project, zone, instanceName, err := splitProviderID(providerID)
if err != nil {
return "", err
}
instance, err := c.instanceAPI.Get(ctx, &computepb.GetInstanceRequest{
Instance: instanceName,
Project: project,
Zone: zone,
})
if err != nil {
return "", fmt.Errorf("getting instance %q: %w", instanceName, err)
}
scalingGroupID := getMetadataByKey(instance.Metadata, "created-by")
if scalingGroupID == "" {
return "", fmt.Errorf("instance %q has no created-by metadata", instanceName)
}
scalingGroupID, err = c.canonicalInstanceGroupID(ctx, scalingGroupID)
if err != nil {
return "", err
}
return scalingGroupID, nil
}
// CreateNode creates a node in the specified scaling group.
func (c *Client) CreateNode(ctx context.Context, scalingGroupID string) (nodeName, providerID string, err error) {
project, zone, instanceGroupName, err := splitInstanceGroupID(scalingGroupID)
if err != nil {
return "", "", err
}
project, err = c.canonicalProjectID(ctx, project)
if err != nil {
return "", "", err
}
instanceGroupManager, err := c.instanceGroupManagersAPI.Get(ctx, &computepb.GetInstanceGroupManagerRequest{
InstanceGroupManager: instanceGroupName,
Project: project,
Zone: zone,
})
if err != nil {
return "", "", err
}
if instanceGroupManager.BaseInstanceName == nil {
return "", "", fmt.Errorf("instance group manager %q has no base instance name", instanceGroupName)
}
instanceName := generateInstanceName(*instanceGroupManager.BaseInstanceName, c.prng)
op, err := c.instanceGroupManagersAPI.CreateInstances(ctx, &computepb.CreateInstancesInstanceGroupManagerRequest{
InstanceGroupManager: instanceGroupName,
Project: project,
Zone: zone,
InstanceGroupManagersCreateInstancesRequestResource: &computepb.InstanceGroupManagersCreateInstancesRequest{
Instances: []*computepb.PerInstanceConfig{
{Name: proto.String(instanceName)},
},
},
})
if err != nil {
return "", "", err
}
if err := op.Wait(ctx); err != nil {
return "", "", err
}
return instanceName, joinProviderID(project, zone, instanceName), nil
}
// DeleteNode deletes a node specified by its provider ID.
func (c *Client) DeleteNode(ctx context.Context, providerID string) error {
_, zone, instanceName, err := splitProviderID(providerID)
if err != nil {
return err
}
scalingGroupID, err := c.GetScalingGroupID(ctx, providerID)
if err != nil {
return err
}
instanceGroupProject, instanceGroupZone, instanceGroupName, err := splitInstanceGroupID(scalingGroupID)
if err != nil {
return err
}
instanceID := joinInstanceID(zone, instanceName)
op, err := c.instanceGroupManagersAPI.DeleteInstances(ctx, &computepb.DeleteInstancesInstanceGroupManagerRequest{
InstanceGroupManager: instanceGroupName,
Project: instanceGroupProject,
Zone: instanceGroupZone,
InstanceGroupManagersDeleteInstancesRequestResource: &computepb.InstanceGroupManagersDeleteInstancesRequest{
Instances: []string{instanceID},
},
})
if err != nil {
return fmt.Errorf("deleting instance %q from instance group manager %q: %w", instanceID, scalingGroupID, err)
}
return op.Wait(ctx)
}

View file

@ -0,0 +1,298 @@
/*
Copyright (c) Edgeless Systems GmbH
SPDX-License-Identifier: AGPL-3.0-only
*/
package client
import (
"context"
"errors"
"math/rand"
"testing"
"time"
"cloud.google.com/go/compute/apiv1/computepb"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"google.golang.org/protobuf/proto"
)
func TestGetNodeImage(t *testing.T) {
testCases := map[string]struct {
providerID string
attachedDisks []*computepb.AttachedDisk
disk *computepb.Disk
getInstanceErr error
getDiskErr error
wantImage string
wantErr bool
}{
"boot disk is found": {
providerID: "gce://project/zone/instance-name",
attachedDisks: []*computepb.AttachedDisk{
{
Source: proto.String("https://www.googleapis.com/compute/v1/projects/project/zones/zone/disks/disk"),
},
},
disk: &computepb.Disk{
SourceImage: proto.String("https://www.googleapis.com/compute/v1/projects/project/global/images/image"),
},
wantImage: "projects/project/global/images/image",
},
"splitting providerID fails": {
providerID: "invalid",
wantErr: true,
},
"get instance fails": {
providerID: "gce://project/zone/instance-name",
getInstanceErr: errors.New("get instance error"),
wantErr: true,
},
"instance has no disks": {
providerID: "gce://project/zone/instance-name",
wantErr: true,
},
"attached disk is invalid": {
providerID: "gce://project/zone/instance-name",
attachedDisks: []*computepb.AttachedDisk{{}},
wantErr: true,
},
"boot disk reference is invalid": {
providerID: "gce://project/zone/instance-name",
attachedDisks: []*computepb.AttachedDisk{{
Source: proto.String("invalid"),
}},
wantErr: true,
},
"get disk fails": {
providerID: "gce://project/zone/instance-name",
attachedDisks: []*computepb.AttachedDisk{{
Source: proto.String("https://www.googleapis.com/compute/v1/projects/project/zones/zone/disks/disk"),
}},
getDiskErr: errors.New("get disk error"),
wantErr: true,
},
"disk has no source image": {
providerID: "gce://project/zone/instance-name",
attachedDisks: []*computepb.AttachedDisk{{
Source: proto.String("https://www.googleapis.com/compute/v1/projects/project/zones/zone/disks/disk"),
}},
disk: &computepb.Disk{},
wantErr: true,
},
}
for name, tc := range testCases {
t.Run(name, func(t *testing.T) {
assert := assert.New(t)
require := require.New(t)
client := Client{
instanceAPI: &stubInstanceAPI{
getErr: tc.getInstanceErr,
instance: &computepb.Instance{
Disks: tc.attachedDisks,
},
},
diskAPI: &stubDiskAPI{
getErr: tc.getDiskErr,
disk: tc.disk,
},
}
gotImage, err := client.GetNodeImage(context.Background(), tc.providerID)
if tc.wantErr {
assert.Error(err)
return
}
require.NoError(err)
assert.Equal(tc.wantImage, gotImage)
})
}
}
func TestGetScalingGroupID(t *testing.T) {
testCases := map[string]struct {
providerID string
createdBy string
getInstanceErr error
wantScalingGroupID string
wantErr bool
}{
"scaling group is found": {
providerID: "gce://project/zone/instance-name",
createdBy: "projects/project/zones/zone/instanceGroupManagers/instance-group",
wantScalingGroupID: "projects/project/zones/zone/instanceGroupManagers/instance-group",
},
"splitting providerID fails": {
providerID: "invalid",
wantErr: true,
},
"get instance fails": {
providerID: "gce://project/zone/instance-name",
getInstanceErr: errors.New("get instance error"),
wantErr: true,
},
"instance has no created-by": {
providerID: "gce://project/zone/instance-name",
wantErr: true,
},
}
for name, tc := range testCases {
t.Run(name, func(t *testing.T) {
assert := assert.New(t)
require := require.New(t)
instance := computepb.Instance{}
if tc.createdBy != "" {
instance.Metadata = &computepb.Metadata{
Items: []*computepb.Items{
{
Key: proto.String("created-by"),
Value: proto.String(tc.createdBy),
},
},
}
}
client := Client{
instanceAPI: &stubInstanceAPI{
getErr: tc.getInstanceErr,
instance: &instance,
},
}
gotScalingGroupID, err := client.GetScalingGroupID(context.Background(), tc.providerID)
if tc.wantErr {
assert.Error(err)
return
}
require.NoError(err)
assert.Equal(tc.wantScalingGroupID, gotScalingGroupID)
})
}
}
func TestCreateNode(t *testing.T) {
testCases := map[string]struct {
scalingGroupID string
baseInstanceName *string
getInstanceGroupManagerErr error
createInstanceErr error
wantErr bool
}{
"scaling group is found": {
scalingGroupID: "projects/project/zones/zone/instanceGroupManagers/instance-group",
baseInstanceName: proto.String("base-name"),
},
"splitting scalingGroupID fails": {
scalingGroupID: "invalid",
wantErr: true,
},
"get instance group manager fails": {
scalingGroupID: "projects/project/zones/zone/instanceGroupManagers/instance-group",
getInstanceGroupManagerErr: errors.New("get instance group manager error"),
wantErr: true,
},
"instance group manager has no base instance name": {
scalingGroupID: "projects/project/zones/zone/instanceGroupManagers/instance-group",
wantErr: true,
},
"create instance fails": {
scalingGroupID: "projects/project/zones/zone/instanceGroupManagers/instance-group",
baseInstanceName: proto.String("base-name"),
createInstanceErr: errors.New("create instance error"),
wantErr: true,
},
}
for name, tc := range testCases {
t.Run(name, func(t *testing.T) {
assert := assert.New(t)
require := require.New(t)
client := Client{
instanceGroupManagersAPI: &stubInstanceGroupManagersAPI{
getErr: tc.getInstanceGroupManagerErr,
createInstancesErr: tc.createInstanceErr,
instanceGroupManager: &computepb.InstanceGroupManager{
BaseInstanceName: tc.baseInstanceName,
},
},
prng: rand.New(rand.NewSource(int64(time.Now().Nanosecond()))),
}
instanceName, providerID, err := client.CreateNode(context.Background(), tc.scalingGroupID)
if tc.wantErr {
assert.Error(err)
return
}
require.NoError(err)
assert.Contains(instanceName, "base-name")
assert.Contains(providerID, "base-name")
})
}
}
func TestDeleteNode(t *testing.T) {
testCases := map[string]struct {
providerID string
scalingGroupID string
getInstanceErr error
deleteInstanceErr error
wantErr bool
}{
"node is deleted": {
providerID: "gce://project/zone/instance-name",
scalingGroupID: "projects/project/zones/zone/instanceGroupManagers/instance-group",
},
"splitting providerID fails": {
providerID: "invalid",
wantErr: true,
},
"get instance fails": {
providerID: "gce://project/zone/instance-name",
getInstanceErr: errors.New("get instance error"),
wantErr: true,
},
"splitting scalingGroupID fails": {
providerID: "gce://project/zone/instance-name",
scalingGroupID: "invalid",
wantErr: true,
},
"delete instance fails": {
providerID: "gce://project/zone/instance-name",
scalingGroupID: "projects/project/zones/zone/instanceGroupManagers/instance-group",
deleteInstanceErr: errors.New("delete instance error"),
wantErr: true,
},
}
for name, tc := range testCases {
t.Run(name, func(t *testing.T) {
assert := assert.New(t)
require := require.New(t)
client := Client{
instanceGroupManagersAPI: &stubInstanceGroupManagersAPI{
deleteInstancesErr: tc.deleteInstanceErr,
},
instanceAPI: &stubInstanceAPI{
getErr: tc.getInstanceErr,
instance: &computepb.Instance{
Metadata: &computepb.Metadata{
Items: []*computepb.Items{
{Key: proto.String("created-by"), Value: &tc.scalingGroupID},
},
},
},
},
}
err := client.DeleteNode(context.Background(), tc.providerID)
if tc.wantErr {
assert.Error(err)
return
}
require.NoError(err)
})
}
}

View file

@ -0,0 +1,65 @@
/*
Copyright (c) Edgeless Systems GmbH
SPDX-License-Identifier: AGPL-3.0-only
*/
package client
import (
"context"
"errors"
"net/http"
"cloud.google.com/go/compute/apiv1/computepb"
"github.com/edgelesssys/constellation/operators/constellation-node-operator/v2/api/v1alpha1"
updatev1alpha1 "github.com/edgelesssys/constellation/operators/constellation-node-operator/v2/api/v1alpha1"
"google.golang.org/api/googleapi"
)
// GetNodeState returns the state of the node.
func (c *Client) GetNodeState(ctx context.Context, providerID string) (updatev1alpha1.CSPNodeState, error) {
project, zone, instanceName, err := splitProviderID(providerID)
if err != nil {
return "", err
}
instance, err := c.instanceAPI.Get(ctx, &computepb.GetInstanceRequest{
Instance: instanceName,
Project: project,
Zone: zone,
})
if err != nil {
var apiErr *googleapi.Error
if errors.As(err, &apiErr) {
if apiErr.Code == http.StatusNotFound {
return v1alpha1.NodeStateTerminated, nil
}
}
return "", err
}
if instance.Status == nil {
return v1alpha1.NodeStateUnknown, nil
}
// reference: https://cloud.google.com/compute/docs/instances/instance-life-cycle
switch *instance.Status {
case computepb.Instance_PROVISIONING.String():
fallthrough
case computepb.Instance_STAGING.String():
return v1alpha1.NodeStateCreating, nil
case computepb.Instance_RUNNING.String():
return v1alpha1.NodeStateReady, nil
case computepb.Instance_STOPPING.String():
fallthrough
case computepb.Instance_SUSPENDING.String():
fallthrough
case computepb.Instance_SUSPENDED.String():
fallthrough
case computepb.Instance_REPAIRING.String():
fallthrough
case computepb.Instance_TERMINATED.String(): // this is stopped in GCP terms
return v1alpha1.NodeStateStopped, nil
}
return v1alpha1.NodeStateUnknown, nil
}

View file

@ -0,0 +1,120 @@
/*
Copyright (c) Edgeless Systems GmbH
SPDX-License-Identifier: AGPL-3.0-only
*/
package client
import (
"context"
"errors"
"net/http"
"testing"
"cloud.google.com/go/compute/apiv1/computepb"
updatev1alpha1 "github.com/edgelesssys/constellation/operators/constellation-node-operator/v2/api/v1alpha1"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"google.golang.org/api/googleapi"
"google.golang.org/protobuf/proto"
)
func TestGetNodeState(t *testing.T) {
testCases := map[string]struct {
providerID string
getInstanceErr error
instanceStatus *string
wantNodeState updatev1alpha1.CSPNodeState
wantErr bool
}{
"node is deleted and API returns 404": {
providerID: "gce://project/zone/instance-name",
getInstanceErr: &googleapi.Error{
Code: http.StatusNotFound,
},
wantNodeState: updatev1alpha1.NodeStateTerminated,
},
"splitting providerID fails": {
providerID: "invalid",
wantErr: true,
},
"node is deleted and API returns other error": {
providerID: "gce://project/zone/instance-name",
getInstanceErr: errors.New("get instance error"),
wantErr: true,
},
"instance has no status": {
providerID: "gce://project/zone/instance-name",
wantNodeState: updatev1alpha1.NodeStateUnknown,
},
"instance is provisioning": {
providerID: "gce://project/zone/instance-name",
instanceStatus: proto.String("PROVISIONING"),
wantNodeState: updatev1alpha1.NodeStateCreating,
},
"instance is staging": {
providerID: "gce://project/zone/instance-name",
instanceStatus: proto.String("STAGING"),
wantNodeState: updatev1alpha1.NodeStateCreating,
},
"instance is running": {
providerID: "gce://project/zone/instance-name",
instanceStatus: proto.String("RUNNING"),
wantNodeState: updatev1alpha1.NodeStateReady,
},
"instance is stopping": {
providerID: "gce://project/zone/instance-name",
instanceStatus: proto.String("STOPPING"),
wantNodeState: updatev1alpha1.NodeStateStopped,
},
"instance is suspending": {
providerID: "gce://project/zone/instance-name",
instanceStatus: proto.String("SUSPENDING"),
wantNodeState: updatev1alpha1.NodeStateStopped,
},
"instance is suspended": {
providerID: "gce://project/zone/instance-name",
instanceStatus: proto.String("SUSPENDED"),
wantNodeState: updatev1alpha1.NodeStateStopped,
},
"instance is repairing": {
providerID: "gce://project/zone/instance-name",
instanceStatus: proto.String("REPAIRING"),
wantNodeState: updatev1alpha1.NodeStateStopped,
},
"instance terminated": {
providerID: "gce://project/zone/instance-name",
instanceStatus: proto.String("TERMINATED"),
wantNodeState: updatev1alpha1.NodeStateStopped,
},
"instance state unknown": {
providerID: "gce://project/zone/instance-name",
instanceStatus: proto.String("unknown"),
wantNodeState: updatev1alpha1.NodeStateUnknown,
},
}
for name, tc := range testCases {
t.Run(name, func(t *testing.T) {
assert := assert.New(t)
require := require.New(t)
client := Client{
instanceAPI: &stubInstanceAPI{
getErr: tc.getInstanceErr,
instance: &computepb.Instance{
Status: tc.instanceStatus,
},
},
}
nodeState, err := client.GetNodeState(context.Background(), tc.providerID)
if tc.wantErr {
assert.Error(err)
return
}
require.NoError(err)
assert.Equal(tc.wantNodeState, nodeState)
})
}
}

View file

@ -0,0 +1,32 @@
/*
Copyright (c) Edgeless Systems GmbH
SPDX-License-Identifier: AGPL-3.0-only
*/
package client
import (
"context"
"errors"
"regexp"
"cloud.google.com/go/compute/apiv1/computepb"
)
var numericProjectIDRegex = regexp.MustCompile(`^\d+$`)
// canonicalProjectID returns the project id for a given project id or project number.
func (c *Client) canonicalProjectID(ctx context.Context, project string) (string, error) {
if !numericProjectIDRegex.MatchString(project) {
return project, nil
}
computeProject, err := c.projectAPI.Get(ctx, &computepb.GetProjectRequest{Project: project})
if err != nil {
return "", err
}
if computeProject == nil || computeProject.Name == nil {
return "", errors.New("invalid project")
}
return *computeProject.Name, nil
}

View file

@ -0,0 +1,69 @@
/*
Copyright (c) Edgeless Systems GmbH
SPDX-License-Identifier: AGPL-3.0-only
*/
package client
import (
"context"
"errors"
"testing"
"cloud.google.com/go/compute/apiv1/computepb"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"google.golang.org/protobuf/proto"
)
func TestCanonicalProjectID(t *testing.T) {
testCases := map[string]struct {
projectID string
project *computepb.Project
getProjectErr error
wantProjectID string
wantErr bool
}{
"already canonical": {
projectID: "project-12345",
wantProjectID: "project-12345",
},
"numeric project id": {
projectID: "12345",
wantProjectID: "project-12345",
project: &computepb.Project{Name: proto.String("project-12345")},
},
"numeric project id with error": {
projectID: "12345",
wantProjectID: "project-12345",
getProjectErr: errors.New("get error"),
wantErr: true,
},
"numeric project id with nil project": {
projectID: "12345",
wantErr: true,
},
}
for name, tc := range testCases {
t.Run(name, func(t *testing.T) {
assert := assert.New(t)
require := require.New(t)
client := Client{
projectAPI: &stubProjectAPI{
project: tc.project,
getErr: tc.getProjectErr,
},
}
gotID, err := client.canonicalProjectID(context.Background(), tc.projectID)
if tc.wantErr {
assert.Error(err)
return
}
require.NoError(err)
assert.Equal(tc.wantProjectID, gotID)
})
}
}

View file

@ -0,0 +1,36 @@
/*
Copyright (c) Edgeless Systems GmbH
SPDX-License-Identifier: AGPL-3.0-only
*/
package client
import (
"fmt"
"regexp"
)
var providerIDRegex = regexp.MustCompile(`^gce://([^/]+)/([^/]+)/([^/]+)$`)
// splitProviderID splits a provider's id into core components.
// A providerID is build after the schema 'gce://<project-id>/<zone>/<instance-name>'
func splitProviderID(providerID string) (project, zone, instance string, err error) {
matches := providerIDRegex.FindStringSubmatch(providerID)
if len(matches) != 4 {
return "", "", "", fmt.Errorf("splitting providerID: %q. matches: %v", providerID, matches)
}
return matches[1], matches[2], matches[3], nil
}
// joinProviderID builds a k8s provider ID for GCP instances.
// A providerID is build after the schema 'gce://<project-id>/<zone>/<instance-name>'
func joinProviderID(project, zone, instanceName string) string {
return fmt.Sprintf("gce://%v/%v/%v", project, zone, instanceName)
}
// joinInstanceID builds a gcp instance ID from the zone and instance name.
func joinInstanceID(zone, instanceName string) string {
return fmt.Sprintf("zones/%v/instances/%v", zone, instanceName)
}

View file

@ -0,0 +1,105 @@
/*
Copyright (c) Edgeless Systems GmbH
SPDX-License-Identifier: AGPL-3.0-only
*/
package client
import (
"testing"
"github.com/stretchr/testify/assert"
)
func TestSplitProviderID(t *testing.T) {
testCases := map[string]struct {
providerID string
wantProjectID string
wantZone string
wantInstance string
wantErr bool
}{
"simple id": {
providerID: "gce://someProject/someZone/someInstance",
wantProjectID: "someProject",
wantZone: "someZone",
wantInstance: "someInstance",
},
"incomplete id": {
providerID: "gce://someProject/someZone",
wantErr: true,
},
"wrong provider": {
providerID: "azure://someProject/someZone/someInstance",
wantErr: true,
},
}
for name, tc := range testCases {
t.Run(name, func(t *testing.T) {
assert := assert.New(t)
projectID, zone, instance, err := splitProviderID(tc.providerID)
if tc.wantErr {
assert.Error(err)
return
}
assert.NoError(err)
assert.Equal(tc.wantProjectID, projectID)
assert.Equal(tc.wantZone, zone)
assert.Equal(tc.wantInstance, instance)
})
}
}
func TestJoinProviderID(t *testing.T) {
testCases := map[string]struct {
projectID string
zone string
instance string
wantProviderID string
}{
"simple id": {
projectID: "someProject",
zone: "someZone",
instance: "someInstance",
wantProviderID: "gce://someProject/someZone/someInstance",
},
}
for name, tc := range testCases {
t.Run(name, func(t *testing.T) {
assert := assert.New(t)
providerID := joinProviderID(tc.projectID, tc.zone, tc.instance)
assert.Equal(tc.wantProviderID, providerID)
})
}
}
func TestJoinnstanceID(t *testing.T) {
testCases := map[string]struct {
zone string
instanceName string
wantInstanceID string
}{
"simple id": {
zone: "someZone",
instanceName: "someInstance",
wantInstanceID: "zones/someZone/instances/someInstance",
},
}
for name, tc := range testCases {
t.Run(name, func(t *testing.T) {
assert := assert.New(t)
providerID := joinInstanceID(tc.zone, tc.instanceName)
assert.Equal(tc.wantInstanceID, providerID)
})
}
}

View file

@ -0,0 +1,200 @@
/*
Copyright (c) Edgeless Systems GmbH
SPDX-License-Identifier: AGPL-3.0-only
*/
package client
import (
"context"
"errors"
"fmt"
"strings"
"cloud.google.com/go/compute/apiv1/computepb"
"google.golang.org/api/iterator"
)
// GetScalingGroupImage returns the image URI of the scaling group.
func (c *Client) GetScalingGroupImage(ctx context.Context, scalingGroupID string) (string, error) {
instanceTemplate, err := c.getScalingGroupTemplate(ctx, scalingGroupID)
if err != nil {
return "", err
}
return instanceTemplateSourceImage(instanceTemplate)
}
// SetScalingGroupImage sets the image URI of the scaling group.
func (c *Client) SetScalingGroupImage(ctx context.Context, scalingGroupID, imageURI string) error {
project, zone, instanceGroupName, err := splitInstanceGroupID(scalingGroupID)
if err != nil {
return err
}
// get current template
instanceTemplate, err := c.getScalingGroupTemplate(ctx, scalingGroupID)
if err != nil {
return err
}
// check if template already uses the same image
oldImageURI, err := instanceTemplateSourceImage(instanceTemplate)
if err != nil {
return err
}
if oldImageURI == imageURI {
return nil
}
// clone template with desired image
if instanceTemplate.Name == nil {
return fmt.Errorf("instance template of scaling group %q has no name", scalingGroupID)
}
instanceTemplate.Properties.Disks[0].InitializeParams.SourceImage = &imageURI
newTemplateName, err := generateInstanceTemplateName(*instanceTemplate.Name)
if err != nil {
return err
}
instanceTemplate.Name = &newTemplateName
op, err := c.instanceTemplateAPI.Insert(ctx, &computepb.InsertInstanceTemplateRequest{
Project: project,
InstanceTemplateResource: instanceTemplate,
})
if err != nil {
return fmt.Errorf("cloning instance template: %w", err)
}
if err := op.Wait(ctx); err != nil {
return fmt.Errorf("waiting for cloned instance template: %w", err)
}
newTemplateURI := joinInstanceTemplateURI(project, newTemplateName)
// update instance group manager to use new template
op, err = c.instanceGroupManagersAPI.SetInstanceTemplate(ctx, &computepb.SetInstanceTemplateInstanceGroupManagerRequest{
InstanceGroupManager: instanceGroupName,
Project: project,
Zone: zone,
InstanceGroupManagersSetInstanceTemplateRequestResource: &computepb.InstanceGroupManagersSetInstanceTemplateRequest{
InstanceTemplate: &newTemplateURI,
},
})
if err != nil {
return fmt.Errorf("setting instance template: %w", err)
}
if err := op.Wait(ctx); err != nil {
return fmt.Errorf("waiting for setting instance template: %w", err)
}
return nil
}
// GetScalingGroupName retrieves the name of a scaling group.
// This keeps the casing of the original name, but Kubernetes requires the name to be lowercase,
// so use strings.ToLower() on the result if using the name in a Kubernetes context.
func (c *Client) GetScalingGroupName(scalingGroupID string) (string, error) {
_, _, instanceGroupName, err := splitInstanceGroupID(scalingGroupID)
if err != nil {
return "", fmt.Errorf("getting scaling group name: %w", err)
}
return instanceGroupName, nil
}
// GetAutoscalingGroupName retrieves the name of a scaling group as needed by the cluster-autoscaler.
func (c *Client) GetAutoscalingGroupName(scalingGroupID string) (string, error) {
project, zone, instanceGroupName, err := splitInstanceGroupID(scalingGroupID)
if err != nil {
return "", fmt.Errorf("getting autoscaling scaling group name: %w", err)
}
return ensureURIPrefixed(fmt.Sprintf("projects/%s/zones/%s/instanceGroups/%s", project, zone, instanceGroupName)), nil
}
// ListScalingGroups retrieves a list of scaling groups for the cluster.
func (c *Client) ListScalingGroups(ctx context.Context, uid string) (controlPlaneGroupIDs []string, workerGroupIDs []string, err error) {
iter := c.instanceGroupManagersAPI.AggregatedList(ctx, &computepb.AggregatedListInstanceGroupManagersRequest{
Project: c.projectID,
})
for instanceGroupManagerScopedListPair, err := iter.Next(); ; instanceGroupManagerScopedListPair, err = iter.Next() {
if err == iterator.Done {
break
}
if err != nil {
return nil, nil, fmt.Errorf("listing instance group managers: %w", err)
}
if instanceGroupManagerScopedListPair.Value == nil {
continue
}
for _, grpManager := range instanceGroupManagerScopedListPair.Value.InstanceGroupManagers {
if grpManager == nil || grpManager.Name == nil || grpManager.SelfLink == nil || grpManager.InstanceTemplate == nil {
continue
}
templateURI := strings.Split(*grpManager.InstanceTemplate, "/")
if len(templateURI) < 1 {
continue // invalid template URI
}
template, err := c.instanceTemplateAPI.Get(ctx, &computepb.GetInstanceTemplateRequest{
Project: c.projectID,
InstanceTemplate: templateURI[len(templateURI)-1],
})
if err != nil {
return nil, nil, fmt.Errorf("getting instance template: %w", err)
}
if template.Properties == nil || template.Properties.Labels == nil {
continue
}
if template.Properties.Labels["constellation-uid"] != uid {
continue
}
groupID, err := c.canonicalInstanceGroupID(ctx, *grpManager.SelfLink)
if err != nil {
return nil, nil, fmt.Errorf("normalizing instance group ID: %w", err)
}
switch strings.ToLower(template.Properties.Labels["constellation-role"]) {
case "control-plane", "controlplane":
controlPlaneGroupIDs = append(controlPlaneGroupIDs, groupID)
case "worker":
workerGroupIDs = append(workerGroupIDs, groupID)
}
}
}
return controlPlaneGroupIDs, workerGroupIDs, nil
}
func (c *Client) getScalingGroupTemplate(ctx context.Context, scalingGroupID string) (*computepb.InstanceTemplate, error) {
project, zone, instanceGroupName, err := splitInstanceGroupID(scalingGroupID)
if err != nil {
return nil, err
}
instanceGroupManager, err := c.instanceGroupManagersAPI.Get(ctx, &computepb.GetInstanceGroupManagerRequest{
InstanceGroupManager: instanceGroupName,
Project: project,
Zone: zone,
})
if err != nil {
return nil, fmt.Errorf("getting instance group manager %q: %w", instanceGroupName, err)
}
if instanceGroupManager.InstanceTemplate == nil {
return nil, fmt.Errorf("instance group manager %q has no instance template", instanceGroupName)
}
instanceTemplateProject, instanceTemplateName, err := splitInstanceTemplateID(uriNormalize(*instanceGroupManager.InstanceTemplate))
if err != nil {
return nil, fmt.Errorf("splitting instance template name: %w", err)
}
instanceTemplate, err := c.instanceTemplateAPI.Get(ctx, &computepb.GetInstanceTemplateRequest{
InstanceTemplate: instanceTemplateName,
Project: instanceTemplateProject,
})
if err != nil {
return nil, fmt.Errorf("getting instance template %q: %w", instanceTemplateName, err)
}
return instanceTemplate, nil
}
func instanceTemplateSourceImage(instanceTemplate *computepb.InstanceTemplate) (string, error) {
if instanceTemplate.Properties == nil ||
len(instanceTemplate.Properties.Disks) == 0 ||
instanceTemplate.Properties.Disks[0].InitializeParams == nil ||
instanceTemplate.Properties.Disks[0].InitializeParams.SourceImage == nil {
return "", errors.New("instance template has no source image")
}
return uriNormalize(*instanceTemplate.Properties.Disks[0].InitializeParams.SourceImage), nil
}

View file

@ -0,0 +1,428 @@
/*
Copyright (c) Edgeless Systems GmbH
SPDX-License-Identifier: AGPL-3.0-only
*/
package client
import (
"context"
"errors"
"testing"
"cloud.google.com/go/compute/apiv1/computepb"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"google.golang.org/protobuf/proto"
)
func TestGetScalingGroupImage(t *testing.T) {
testCases := map[string]struct {
scalingGroupID string
instanceGroupManagerTemplateID *string
instanceTemplate *computepb.InstanceTemplate
getInstanceGroupManagerErr error
getInstanceTemplateErr error
wantImage string
wantErr bool
}{
"getting image works": {
scalingGroupID: "projects/project/zones/zone/instanceGroupManagers/instance-group",
instanceGroupManagerTemplateID: proto.String("projects/project/global/instanceTemplates/instance-template"),
instanceTemplate: &computepb.InstanceTemplate{
Properties: &computepb.InstanceProperties{
Disks: []*computepb.AttachedDisk{
{
InitializeParams: &computepb.AttachedDiskInitializeParams{
SourceImage: proto.String("https://www.googleapis.com/compute/v1/projects/project/global/images/image"),
},
},
},
},
},
wantImage: "projects/project/global/images/image",
},
"splitting scalingGroupID fails": {
scalingGroupID: "invalid",
wantErr: true,
},
"get instance fails": {
scalingGroupID: "projects/project/zones/zone/instanceGroupManagers/instance-group",
getInstanceGroupManagerErr: errors.New("get instance error"),
wantErr: true,
},
"instance group manager has no template": {
scalingGroupID: "projects/project/zones/zone/instanceGroupManagers/instance-group",
wantErr: true,
},
"instance group manager template id is invalid": {
scalingGroupID: "projects/project/zones/zone/instanceGroupManagers/instance-group",
instanceGroupManagerTemplateID: proto.String("invalid"),
wantErr: true,
},
"get instance template fails": {
scalingGroupID: "projects/project/zones/zone/instanceGroupManagers/instance-group",
instanceGroupManagerTemplateID: proto.String("projects/project/global/instanceTemplates/instance-template"),
getInstanceTemplateErr: errors.New("get instance template error"),
wantErr: true,
},
"instance template has no disks": {
scalingGroupID: "projects/project/zones/zone/instanceGroupManagers/instance-group",
instanceGroupManagerTemplateID: proto.String("projects/project/global/instanceTemplates/instance-template"),
instanceTemplate: &computepb.InstanceTemplate{
Properties: &computepb.InstanceProperties{},
},
wantErr: true,
},
}
for name, tc := range testCases {
t.Run(name, func(t *testing.T) {
assert := assert.New(t)
require := require.New(t)
client := Client{
instanceGroupManagersAPI: &stubInstanceGroupManagersAPI{
getErr: tc.getInstanceGroupManagerErr,
instanceGroupManager: &computepb.InstanceGroupManager{
InstanceTemplate: tc.instanceGroupManagerTemplateID,
},
},
instanceTemplateAPI: &stubInstanceTemplateAPI{
getErr: tc.getInstanceTemplateErr,
template: tc.instanceTemplate,
},
}
gotImage, err := client.GetScalingGroupImage(context.Background(), tc.scalingGroupID)
if tc.wantErr {
assert.Error(err)
return
}
require.NoError(err)
assert.Equal(tc.wantImage, gotImage)
})
}
}
func TestSetScalingGroupImage(t *testing.T) {
testCases := map[string]struct {
scalingGroupID string
imageURI string
instanceGroupManagerTemplateID *string
instanceTemplate *computepb.InstanceTemplate
getInstanceGroupManagerErr error
getInstanceTemplateErr error
setInstanceTemplateErr error
insertInstanceTemplateErr error
wantErr bool
}{
"setting image works": {
scalingGroupID: "projects/project/zones/zone/instanceGroupManagers/instance-group",
imageURI: "projects/project/global/images/image-2",
instanceGroupManagerTemplateID: proto.String("projects/project/global/instanceTemplates/instance-template"),
instanceTemplate: &computepb.InstanceTemplate{
Name: proto.String("instance-template"),
Properties: &computepb.InstanceProperties{
Disks: []*computepb.AttachedDisk{
{
InitializeParams: &computepb.AttachedDiskInitializeParams{
SourceImage: proto.String("https://www.googleapis.com/compute/v1/projects/project/global/images/image-1"),
},
},
},
},
},
},
"same image already in use": {
scalingGroupID: "projects/project/zones/zone/instanceGroupManagers/instance-group",
imageURI: "projects/project/global/images/image",
instanceGroupManagerTemplateID: proto.String("projects/project/global/instanceTemplates/instance-template"),
instanceTemplate: &computepb.InstanceTemplate{
Name: proto.String("instance-template"),
Properties: &computepb.InstanceProperties{
Disks: []*computepb.AttachedDisk{
{
InitializeParams: &computepb.AttachedDiskInitializeParams{
SourceImage: proto.String("https://www.googleapis.com/compute/v1/projects/project/global/images/image"),
},
},
},
},
},
// will not be triggered
insertInstanceTemplateErr: errors.New("insert instance template error"),
},
"splitting scalingGroupID fails": {
scalingGroupID: "invalid",
wantErr: true,
},
"get instance fails": {
scalingGroupID: "projects/project/zones/zone/instanceGroupManagers/instance-group",
getInstanceGroupManagerErr: errors.New("get instance error"),
wantErr: true,
},
"instance group manager has no template": {
scalingGroupID: "projects/project/zones/zone/instanceGroupManagers/instance-group",
wantErr: true,
},
"instance group manager template id is invalid": {
scalingGroupID: "projects/project/zones/zone/instanceGroupManagers/instance-group",
instanceGroupManagerTemplateID: proto.String("invalid"),
wantErr: true,
},
"get instance template fails": {
scalingGroupID: "projects/project/zones/zone/instanceGroupManagers/instance-group",
instanceGroupManagerTemplateID: proto.String("projects/project/global/instanceTemplates/instance-template"),
getInstanceTemplateErr: errors.New("get instance template error"),
wantErr: true,
},
"instance template has no disks": {
scalingGroupID: "projects/project/zones/zone/instanceGroupManagers/instance-group",
instanceGroupManagerTemplateID: proto.String("projects/project/global/instanceTemplates/instance-template"),
instanceTemplate: &computepb.InstanceTemplate{
Properties: &computepb.InstanceProperties{},
},
wantErr: true,
},
"instance template has no name": {
scalingGroupID: "projects/project/zones/zone/instanceGroupManagers/instance-group",
imageURI: "projects/project/global/images/image-2",
instanceGroupManagerTemplateID: proto.String("projects/project/global/instanceTemplates/instance-template"),
instanceTemplate: &computepb.InstanceTemplate{
Properties: &computepb.InstanceProperties{
Disks: []*computepb.AttachedDisk{
{
InitializeParams: &computepb.AttachedDiskInitializeParams{
SourceImage: proto.String("https://www.googleapis.com/compute/v1/projects/project/global/images/image-1"),
},
},
},
},
},
wantErr: true,
},
"instance template name generation fails": {
scalingGroupID: "projects/project/zones/zone/instanceGroupManagers/instance-group",
imageURI: "projects/project/global/images/image-2",
instanceGroupManagerTemplateID: proto.String("projects/project/global/instanceTemplates/instance-template"),
instanceTemplate: &computepb.InstanceTemplate{
Name: proto.String("instance-template-999999999999999999999"),
Properties: &computepb.InstanceProperties{
Disks: []*computepb.AttachedDisk{
{
InitializeParams: &computepb.AttachedDiskInitializeParams{
SourceImage: proto.String("https://www.googleapis.com/compute/v1/projects/project/global/images/image-1"),
},
},
},
},
},
wantErr: true,
},
"instance template insert fails": {
scalingGroupID: "projects/project/zones/zone/instanceGroupManagers/instance-group",
imageURI: "projects/project/global/images/image-2",
instanceGroupManagerTemplateID: proto.String("projects/project/global/instanceTemplates/instance-template"),
instanceTemplate: &computepb.InstanceTemplate{
Name: proto.String("instance-template"),
Properties: &computepb.InstanceProperties{
Disks: []*computepb.AttachedDisk{
{
InitializeParams: &computepb.AttachedDiskInitializeParams{
SourceImage: proto.String("https://www.googleapis.com/compute/v1/projects/project/global/images/image-1"),
},
},
},
},
},
insertInstanceTemplateErr: errors.New("insert instance template error"),
wantErr: true,
},
"setting instance template fails": {
scalingGroupID: "projects/project/zones/zone/instanceGroupManagers/instance-group",
imageURI: "projects/project/global/images/image-2",
instanceGroupManagerTemplateID: proto.String("projects/project/global/instanceTemplates/instance-template"),
instanceTemplate: &computepb.InstanceTemplate{
Name: proto.String("instance-template"),
Properties: &computepb.InstanceProperties{
Disks: []*computepb.AttachedDisk{
{
InitializeParams: &computepb.AttachedDiskInitializeParams{
SourceImage: proto.String("https://www.googleapis.com/compute/v1/projects/project/global/images/image-1"),
},
},
},
},
},
setInstanceTemplateErr: errors.New("setting instance template error"),
wantErr: true,
},
}
for name, tc := range testCases {
t.Run(name, func(t *testing.T) {
assert := assert.New(t)
require := require.New(t)
client := Client{
instanceGroupManagersAPI: &stubInstanceGroupManagersAPI{
getErr: tc.getInstanceGroupManagerErr,
setInstanceTemplateErr: tc.setInstanceTemplateErr,
instanceGroupManager: &computepb.InstanceGroupManager{
InstanceTemplate: tc.instanceGroupManagerTemplateID,
},
},
instanceTemplateAPI: &stubInstanceTemplateAPI{
getErr: tc.getInstanceTemplateErr,
insertErr: tc.insertInstanceTemplateErr,
template: tc.instanceTemplate,
},
}
err := client.SetScalingGroupImage(context.Background(), tc.scalingGroupID, tc.imageURI)
if tc.wantErr {
assert.Error(err)
return
}
require.NoError(err)
})
}
}
func TestGetScalingGroupName(t *testing.T) {
testCases := map[string]struct {
scalingGroupID string
wantName string
wantErr bool
}{
"valid scaling group ID": {
scalingGroupID: "projects/project/zones/zone/instanceGroupManagers/instance-group",
wantName: "instance-group",
},
"invalid scaling group ID": {
scalingGroupID: "invalid",
wantErr: true,
},
}
for name, tc := range testCases {
t.Run(name, func(t *testing.T) {
assert := assert.New(t)
require := require.New(t)
client := Client{}
gotName, err := client.GetScalingGroupName(tc.scalingGroupID)
if tc.wantErr {
assert.Error(err)
return
}
require.NoError(err)
assert.Equal(tc.wantName, gotName)
})
}
}
func TestListScalingGroups(t *testing.T) {
testCases := map[string]struct {
name *string
groupID *string
templateRef *string
templateLabels map[string]string
listInstanceGroupManagersErr error
templateGetErr error
wantControlPlanes []string
wantWorkers []string
wantErr bool
}{
"list instance group managers fails": {
listInstanceGroupManagersErr: errors.New("list instance group managers error"),
wantErr: true,
},
"get instance template fails": {
name: proto.String("test-control-plane-uid"),
groupID: proto.String("projects/project/zones/zone/instanceGroupManagers/test-control-plane-uid"),
templateRef: proto.String("projects/project/global/instanceTemplates/test-control-plane-uid"),
templateGetErr: errors.New("get instance template error"),
wantErr: true,
},
"list instance group managers for control plane": {
name: proto.String("test-control-plane-uid"),
groupID: proto.String("projects/project/zones/zone/instanceGroupManagers/test-control-plane-uid"),
templateRef: proto.String("projects/project/global/instanceTemplates/test-control-plane-uid"),
templateLabels: map[string]string{
"constellation-uid": "uid",
"constellation-role": "control-plane",
},
wantControlPlanes: []string{
"projects/project/zones/zone/instanceGroupManagers/test-control-plane-uid",
},
},
"list instance group managers for worker": {
name: proto.String("test-worker-uid"),
groupID: proto.String("projects/project/zones/zone/instanceGroupManagers/test-worker-uid"),
templateRef: proto.String("projects/project/global/instanceTemplates/test-control-plane-uid"),
templateLabels: map[string]string{
"constellation-uid": "uid",
"constellation-role": "worker",
},
wantWorkers: []string{
"projects/project/zones/zone/instanceGroupManagers/test-worker-uid",
},
},
"listing instance group managers is not dependant on resource name": {
name: proto.String("some-instance-group-manager"),
groupID: proto.String("projects/project/zones/zone/instanceGroupManagers/some-instance-group-manager"),
templateRef: proto.String("projects/project/global/instanceTemplates/some-instance-group-template"),
templateLabels: map[string]string{
"constellation-uid": "uid",
"constellation-role": "control-plane",
},
wantControlPlanes: []string{
"projects/project/zones/zone/instanceGroupManagers/some-instance-group-manager",
},
},
"unrelated instance group manager": {
name: proto.String("test-control-plane-uid"),
groupID: proto.String("projects/project/zones/zone/instanceGroupManagers/test-unrelated-uid"),
templateRef: proto.String("projects/project/global/instanceTemplates/test-control-plane-uid"),
templateLabels: map[string]string{
"label": "value",
},
},
"invalid instance group manager": {},
}
for name, tc := range testCases {
t.Run(name, func(t *testing.T) {
assert := assert.New(t)
require := require.New(t)
client := Client{
instanceGroupManagersAPI: &stubInstanceGroupManagersAPI{
aggregatedListErr: tc.listInstanceGroupManagersErr,
instanceGroupManager: &computepb.InstanceGroupManager{
Name: tc.name,
SelfLink: tc.groupID,
InstanceTemplate: tc.templateRef,
},
},
instanceTemplateAPI: &stubInstanceTemplateAPI{
template: &computepb.InstanceTemplate{
Properties: &computepb.InstanceProperties{
Labels: tc.templateLabels,
},
},
getErr: tc.templateGetErr,
},
}
gotControlPlanes, gotWorkers, err := client.ListScalingGroups(context.Background(), "uid")
if tc.wantErr {
assert.Error(err)
return
}
require.NoError(err)
assert.ElementsMatch(tc.wantControlPlanes, gotControlPlanes)
assert.ElementsMatch(tc.wantWorkers, gotWorkers)
})
}
}