From f04765dab5186a7dfab1ff709a58511a4775ce2e Mon Sep 17 00:00:00 2001 From: Malte Poll Date: Tue, 29 Mar 2022 13:30:50 +0200 Subject: [PATCH] re-enable azure node groups in statefile and send azure scaleset as autoscaling group --- cli/azure/autoscaling_node_group.go | 8 ++ cli/azure/autoscaling_node_group_test.go | 14 +++ cli/azure/client/client.go | 34 ++++--- cli/azure/client/client_test.go | 114 +++++++++++------------ cli/cmd/init.go | 7 +- 5 files changed, 98 insertions(+), 79 deletions(-) create mode 100644 cli/azure/autoscaling_node_group.go create mode 100644 cli/azure/autoscaling_node_group_test.go diff --git a/cli/azure/autoscaling_node_group.go b/cli/azure/autoscaling_node_group.go new file mode 100644 index 000000000..1b009c80f --- /dev/null +++ b/cli/azure/autoscaling_node_group.go @@ -0,0 +1,8 @@ +package azure + +import "fmt" + +// AutoscalingNodeGroup converts an azure scale set into a node group used by the k8s cluster-autoscaler. +func AutoscalingNodeGroup(scaleSet string, min int, max int) string { + return fmt.Sprintf("%d:%d:%s", min, max, scaleSet) +} diff --git a/cli/azure/autoscaling_node_group_test.go b/cli/azure/autoscaling_node_group_test.go new file mode 100644 index 000000000..ac976f780 --- /dev/null +++ b/cli/azure/autoscaling_node_group_test.go @@ -0,0 +1,14 @@ +package azure + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestAutoscalingNodeGroup(t *testing.T) { + assert := assert.New(t) + nodeGroups := AutoscalingNodeGroup("scale-set", 0, 100) + expectedNodeGroups := "0:100:scale-set" + assert.Equal(expectedNodeGroups, nodeGroups) +} diff --git a/cli/azure/client/client.go b/cli/azure/client/client.go index d45479a34..df06f2c6a 100644 --- a/cli/azure/client/client.go +++ b/cli/azure/client/client.go @@ -165,15 +165,14 @@ func (c *Client) GetState() (state.ConstellationState, error) { return state.ConstellationState{}, errors.New("client has no network security group") } stat.AzureNetworkSecurityGroup = c.networkSecurityGroup - // TODO: un-deprecate as soon as scale sets are available - // if len(c.nodesScaleSet) == 0 { - // return state.ConstellationState{}, errors.New("client has no nodes scale set") - // } - // stat.AzureNodesScaleSet = c.nodesScaleSet - // if len(c.coordinatorsScaleSet) == 0 { - // return state.ConstellationState{}, errors.New("client has no coordinators scale set") - // } - // stat.AzureCoordinatorsScaleSet = c.coordinatorsScaleSet + if len(c.nodesScaleSet) == 0 { + return state.ConstellationState{}, errors.New("client has no nodes scale set") + } + stat.AzureNodesScaleSet = c.nodesScaleSet + if len(c.coordinatorsScaleSet) == 0 { + return state.ConstellationState{}, errors.New("client has no coordinators scale set") + } + stat.AzureCoordinatorsScaleSet = c.coordinatorsScaleSet if len(c.nodes) == 0 { return state.ConstellationState{}, errors.New("client has no nodes") } @@ -225,15 +224,14 @@ func (c *Client) SetState(stat state.ConstellationState) error { return errors.New("state has no subnet") } c.networkSecurityGroup = stat.AzureNetworkSecurityGroup - // TODO: un-deprecate as soon as scale sets are available - //if len(stat.AzureNodesScaleSet) == 0 { - // return errors.New("state has no nodes scale set") - //} - //c.nodesScaleSet = stat.AzureNodesScaleSet - //if len(stat.AzureCoordinatorsScaleSet) == 0 { - // return errors.New("state has no nodes scale set") - //} - //c.coordinatorsScaleSet = stat.AzureCoordinatorsScaleSet + if len(stat.AzureNodesScaleSet) == 0 { + return errors.New("state has no nodes scale set") + } + c.nodesScaleSet = stat.AzureNodesScaleSet + if len(stat.AzureCoordinatorsScaleSet) == 0 { + return errors.New("state has no nodes scale set") + } + c.coordinatorsScaleSet = stat.AzureCoordinatorsScaleSet if len(stat.AzureNodes) == 0 { return errors.New("state has no coordinator scale set") } diff --git a/cli/azure/client/client_test.go b/cli/azure/client/client_test.go index 6aa794876..59f39a17b 100644 --- a/cli/azure/client/client_test.go +++ b/cli/azure/client/client_test.go @@ -38,9 +38,8 @@ func TestSetGetState(t *testing.T) { AzureTenant: "tenant", AzureSubnet: "azure-subnet", AzureNetworkSecurityGroup: "network-security-group", - // TODO: un-deprecate as soon as scale sets are available - // AzureNodesScaleSet: "node-scale-set", - // AzureCoordinatorsScaleSet: "coordinator-scale-set", + AzureNodesScaleSet: "node-scale-set", + AzureCoordinatorsScaleSet: "coordinator-scale-set", }, }, "missing nodes": { @@ -303,61 +302,60 @@ func TestSetGetState(t *testing.T) { }, errExpected: true, }, - // TODO: un-deprecate as soon as scale sets are available - // "missing node scale set": { - // state: state.ConstellationState{ - // CloudProvider: cloudprovider.Azure.String(), - // AzureNodes: azure.Instances{ - // "0": { - // PublicIP: "ip1", - // PrivateIP: "ip2", - // }, - // }, - // AzureCoordinators: azure.Instances{ - // "0": { - // PublicIP: "ip3", - // PrivateIP: "ip4", - // }, - // }, - // Name: "name", - // UID: "uid", - // AzureResourceGroup: "resource-group", - // AzureLocation: "location", - // AzureSubscription: "subscription", - // AzureTenant: "tenant", - // AzureSubnet: "azure-subnet", - // AzureNetworkSecurityGroup: "network-security-group", - // AzureCoordinatorsScaleSet: "coordinator-scale-set", - // }, - // errExpected: true, - // }, - // "missing coordinator scale set": { - // state: state.ConstellationState{ - // CloudProvider: cloudprovider.Azure.String(), - // AzureNodes: azure.Instances{ - // "0": { - // PublicIP: "ip1", - // PrivateIP: "ip2", - // }, - // }, - // AzureCoordinators: azure.Instances{ - // "0": { - // PublicIP: "ip3", - // PrivateIP: "ip4", - // }, - // }, - // Name: "name", - // UID: "uid", - // AzureResourceGroup: "resource-group", - // AzureLocation: "location", - // AzureSubscription: "subscription", - // AzureTenant: "tenant", - // AzureSubnet: "azure-subnet", - // AzureNetworkSecurityGroup: "network-security-group", - // AzureNodesScaleSet: "node-scale-set", - // }, - // errExpected: true, - // }, + "missing node scale set": { + state: state.ConstellationState{ + CloudProvider: cloudprovider.Azure.String(), + AzureNodes: azure.Instances{ + "0": { + PublicIP: "ip1", + PrivateIP: "ip2", + }, + }, + AzureCoordinators: azure.Instances{ + "0": { + PublicIP: "ip3", + PrivateIP: "ip4", + }, + }, + Name: "name", + UID: "uid", + AzureResourceGroup: "resource-group", + AzureLocation: "location", + AzureSubscription: "subscription", + AzureTenant: "tenant", + AzureSubnet: "azure-subnet", + AzureNetworkSecurityGroup: "network-security-group", + AzureCoordinatorsScaleSet: "coordinator-scale-set", + }, + errExpected: true, + }, + "missing coordinator scale set": { + state: state.ConstellationState{ + CloudProvider: cloudprovider.Azure.String(), + AzureNodes: azure.Instances{ + "0": { + PublicIP: "ip1", + PrivateIP: "ip2", + }, + }, + AzureCoordinators: azure.Instances{ + "0": { + PublicIP: "ip3", + PrivateIP: "ip4", + }, + }, + Name: "name", + UID: "uid", + AzureResourceGroup: "resource-group", + AzureLocation: "location", + AzureSubscription: "subscription", + AzureTenant: "tenant", + AzureSubnet: "azure-subnet", + AzureNetworkSecurityGroup: "network-security-group", + AzureNodesScaleSet: "node-scale-set", + }, + errExpected: true, + }, } t.Run("SetState", func(t *testing.T) { diff --git a/cli/cmd/init.go b/cli/cmd/init.go index b9ea35db4..979a490a1 100644 --- a/cli/cmd/init.go +++ b/cli/cmd/init.go @@ -13,6 +13,7 @@ import ( "github.com/spf13/cobra" "golang.zx2c4.com/wireguard/wgctrl/wgtypes" + "github.com/edgelesssys/constellation/cli/azure" "github.com/edgelesssys/constellation/cli/file" "github.com/edgelesssys/constellation/cli/gcp" "github.com/edgelesssys/constellation/cli/proto" @@ -355,7 +356,7 @@ func getScalingGroupsFromConfig(stat state.ConstellationState, config *config.Co case len(stat.GCPCoordinators) != 0: return getGCPInstances(stat, config) case len(stat.AzureCoordinators) != 0: - return getAzureInstances(stat) + return getAzureInstances(stat, config) default: return ScalingGroup{}, ScalingGroup{}, errors.New("no instances to init") } @@ -413,7 +414,7 @@ func getGCPInstances(stat state.ConstellationState, config *config.Config) (coor return } -func getAzureInstances(stat state.ConstellationState) (coordinators, nodes ScalingGroup, err error) { +func getAzureInstances(stat state.ConstellationState, config *config.Config) (coordinators, nodes ScalingGroup, err error) { _, coordinator, err := stat.AzureCoordinators.GetOne() if err != nil { return @@ -434,7 +435,7 @@ func getAzureInstances(stat state.ConstellationState) (coordinators, nodes Scali // TODO: make min / max configurable and abstract autoscaling for different cloud providers nodes = ScalingGroup{ Instances: nodeInstances, - GroupID: "", + GroupID: azure.AutoscalingNodeGroup(stat.AzureNodesScaleSet, *config.AutoscalingNodeGroupsMin, *config.AutoscalingNodeGroupsMax), } return }