replace flannel with cilium

cli/azure/client/api.go

@@ -34,6 +34,18 @@ type networkSecurityGroupsAPI interface {
 		networkSecurityGroupsCreateOrUpdatePollerResponse, error)
 }
 
+type loadBalancersClientCreateOrUpdatePollerResponse interface {
+	PollUntilDone(ctx context.Context, freq time.Duration) (armnetwork.LoadBalancersClientCreateOrUpdateResponse, error)
+}
+
+type loadBalancersAPI interface {
+	BeginCreateOrUpdate(ctx context.Context, resourceGroupName string,
+		loadBalancerName string, parameters armnetwork.LoadBalancer,
+		options *armnetwork.LoadBalancersClientBeginCreateOrUpdateOptions) (
+		loadBalancersClientCreateOrUpdatePollerResponse, error,
+	)
+}
+
 type virtualMachineScaleSetsCreateOrUpdatePollerResponse interface {
 	PollUntilDone(ctx context.Context, freq time.Duration) (armcompute.VirtualMachineScaleSetsClientCreateOrUpdateResponse, error)
 }

cli/azure/client/api_test.go

@@ -48,6 +48,29 @@ func (a stubNetworksAPI) BeginCreateOrUpdate(ctx context.Context, resourceGroupN
 	return a.stubResponse, a.createErr
 }
 
+type stubLoadBalancersAPI struct {
+	createErr    error
+	stubResponse stubLoadBalancersClientCreateOrUpdatePollerResponse
+}
+
+type stubLoadBalancersClientCreateOrUpdatePollerResponse struct {
+	pollResponse armnetwork.LoadBalancersClientCreateOrUpdateResponse
+	pollErr      error
+}
+
+func (r stubLoadBalancersClientCreateOrUpdatePollerResponse) PollUntilDone(ctx context.Context, freq time.Duration,
+) (armnetwork.LoadBalancersClientCreateOrUpdateResponse, error) {
+	return r.pollResponse, r.pollErr
+}
+
+func (a stubLoadBalancersAPI) BeginCreateOrUpdate(ctx context.Context, resourceGroupName string,
+	loadBalancerName string, parameters armnetwork.LoadBalancer,
+	options *armnetwork.LoadBalancersClientBeginCreateOrUpdateOptions) (
+	loadBalancersClientCreateOrUpdatePollerResponse, error,
+) {
+	return a.stubResponse, a.createErr
+}
+
 type stubNetworkSecurityGroupsCreateOrUpdatePollerResponse struct {
 	armnetwork.SecurityGroupsClientCreateOrUpdatePollerResponse
 	pollerErr error
@@ -143,23 +166,17 @@ func (a stubScaleSetsAPI) BeginCreateOrUpdate(ctx context.Context, resourceGroup
 	return a.stubResponse, a.createErr
 }
 
-// TODO: deprecate as soon as scale sets are available.
 type stubPublicIPAddressesAPI struct {
-	// TODO: deprecate as soon as scale sets are available.
-	createErr error
-	// TODO: deprecate as soon as scale sets are available.
-	getErr error
-	// TODO: deprecate as soon as scale sets are available.
+	createErr          error
+	getErr             error
 	stubCreateResponse stubPublicIPAddressesClientCreateOrUpdatePollerResponse
 }
 
-// TODO: deprecate as soon as scale sets are available.
 type stubPublicIPAddressesClientCreateOrUpdatePollerResponse struct {
 	armnetwork.PublicIPAddressesClientCreateOrUpdatePollerResponse
 	pollErr error
 }
 
-// TODO: deprecate as soon as scale sets are available.
 func (r stubPublicIPAddressesClientCreateOrUpdatePollerResponse) PollUntilDone(ctx context.Context, freq time.Duration) (
 	armnetwork.PublicIPAddressesClientCreateOrUpdateResponse, error,
 ) {
@@ -167,6 +184,9 @@ func (r stubPublicIPAddressesClientCreateOrUpdatePollerResponse) PollUntilDone(c
 		PublicIPAddressesClientCreateOrUpdateResult: armnetwork.PublicIPAddressesClientCreateOrUpdateResult{
 			PublicIPAddress: armnetwork.PublicIPAddress{
 				ID: to.StringPtr("pubIP-id"),
+				Properties: &armnetwork.PublicIPAddressPropertiesFormat{
+					IPAddress: to.StringPtr("192.0.2.1"),
+				},
 			},
 		},
 	}, r.pollErr
@@ -206,7 +226,6 @@ func (a stubPublicIPAddressesAPI) ListVirtualMachineScaleSetVMPublicIPAddresses(
 	return &stubPublicIPAddressesListVirtualMachineScaleSetVMPublicIPAddressesPager{pagesCounter: 0, PagesMax: 1}
 }
 
-// TODO: deprecate as soon as scale sets are available.
 func (a stubPublicIPAddressesAPI) BeginCreateOrUpdate(ctx context.Context, resourceGroupName string, publicIPAddressName string,
 	parameters armnetwork.PublicIPAddress, options *armnetwork.PublicIPAddressesClientBeginCreateOrUpdateOptions) (
 	publicIPAddressesClientCreateOrUpdatePollerResponse, error,
@@ -214,7 +233,6 @@ func (a stubPublicIPAddressesAPI) BeginCreateOrUpdate(ctx context.Context, resou
 	return a.stubCreateResponse, a.createErr
 }
 
-// TODO: deprecate as soon as scale sets are available.
 func (a stubPublicIPAddressesAPI) Get(ctx context.Context, resourceGroupName string, publicIPAddressName string, options *armnetwork.PublicIPAddressesClientGetOptions) (
 	armnetwork.PublicIPAddressesClientGetResponse, error,
 ) {
@@ -230,11 +248,9 @@ func (a stubPublicIPAddressesAPI) Get(ctx context.Context, resourceGroupName str
 }
 
 type stubNetworkInterfacesAPI struct {
-	getErr error
-	// TODO: deprecate as soon as scale sets are available
+	getErr    error
 	createErr error
-	// TODO: deprecate as soon as scale sets are available
-	stubResp stubInterfacesClientCreateOrUpdatePollerResponse
+	stubResp  stubInterfacesClientCreateOrUpdatePollerResponse
 }
 
 func (a stubNetworkInterfacesAPI) GetVirtualMachineScaleSetNetworkInterface(ctx context.Context, resourceGroupName string,

cli/azure/client/azurewrappers.go

@@ -35,6 +35,17 @@ func (c *networkInterfacesClient) BeginCreateOrUpdate(ctx context.Context, resou
 	return c.InterfacesClient.BeginCreateOrUpdate(ctx, resourceGroupName, networkInterfaceName, parameters, options)
 }
 
+type loadBalancersClient struct {
+	*armnetwork.LoadBalancersClient
+}
+
+func (c *loadBalancersClient) BeginCreateOrUpdate(ctx context.Context, resourceGroupName string, loadBalancerName string,
+	parameters armnetwork.LoadBalancer, options *armnetwork.LoadBalancersClientBeginCreateOrUpdateOptions) (
+	loadBalancersClientCreateOrUpdatePollerResponse, error,
+) {
+	return c.LoadBalancersClient.BeginCreateOrUpdate(ctx, resourceGroupName, loadBalancerName, parameters, options)
+}
+
 type networkSecurityGroupsClient struct {
 	*armnetwork.SecurityGroupsClient
 }

cli/azure/client/client.go

@@ -33,6 +33,7 @@ type Client struct {
 	scaleSetsAPI
 	publicIPAddressesAPI
 	networkInterfacesAPI
+	loadBalancersAPI
 	virtualMachinesAPI
 	applicationsAPI
 	servicePrincipalsAPI
@@ -53,6 +54,8 @@ type Client struct {
 	subnetID             string
 	coordinatorsScaleSet string
 	nodesScaleSet        string
+	loadBalancerName     string
+	loadBalancerPubIP    string
 	networkSecurityGroup string
 	adAppObjectID        string
 }
@@ -77,6 +80,7 @@ func NewFromDefault(subscriptionID, tenantID string) (*Client, error) {
 	scaleSetAPI := armcompute.NewVirtualMachineScaleSetsClient(subscriptionID, cred, nil)
 	publicIPAddressesAPI := armnetwork.NewPublicIPAddressesClient(subscriptionID, cred, nil)
 	networkInterfacesAPI := armnetwork.NewInterfacesClient(subscriptionID, cred, nil)
+	loadBalancersAPI := armnetwork.NewLoadBalancersClient(subscriptionID, cred, nil)
 	virtualMachinesAPI := armcompute.NewVirtualMachinesClient(subscriptionID, cred, nil)
 	applicationsAPI := graphrbac.NewApplicationsClient(tenantID)
 	applicationsAPI.Authorizer = graphAuthorizer
@@ -92,6 +96,7 @@ func NewFromDefault(subscriptionID, tenantID string) (*Client, error) {
 		scaleSetsAPI:                    &virtualMachineScaleSetsClient{scaleSetAPI},
 		publicIPAddressesAPI:            &publicIPAddressesClient{publicIPAddressesAPI},
 		networkInterfacesAPI:            &networkInterfacesClient{networkInterfacesAPI},
+		loadBalancersAPI:                &loadBalancersClient{loadBalancersAPI},
 		applicationsAPI:                 &applicationsClient{&applicationsAPI},
 		servicePrincipalsAPI:            &servicePrincipalsClient{&servicePrincipalsAPI},
 		roleAssignmentsAPI:              &roleAssignmentsClient{&roleAssignmentsAPI},
@@ -233,7 +238,7 @@ func (c *Client) SetState(stat state.ConstellationState) error {
 	}
 	c.coordinatorsScaleSet = stat.AzureCoordinatorsScaleSet
 	if len(stat.AzureNodes) == 0 {
-		return errors.New("state has no coordinator scale set")
+		return errors.New("state has no nodes")
 	}
 	c.nodes = stat.AzureNodes
 	if len(stat.AzureCoordinators) == 0 {

cli/azure/client/compute.go

@@ -13,13 +13,14 @@ import (
 func (c *Client) CreateInstances(ctx context.Context, input CreateInstancesInput) error {
 	// Create nodes scale set
 	createNodesInput := CreateScaleSetInput{
-		Name:                 "constellation-scale-set-nodes-" + c.uid,
-		NamePrefix:           c.name + "-worker-" + c.uid + "-",
-		Count:                input.CountNodes,
-		InstanceType:         input.InstanceType,
-		StateDiskSizeGB:      int32(input.StateDiskSizeGB),
-		Image:                input.Image,
-		UserAssingedIdentity: input.UserAssingedIdentity,
+		Name:                           "constellation-scale-set-nodes-" + c.uid,
+		NamePrefix:                     c.name + "-worker-" + c.uid + "-",
+		Count:                          input.CountNodes,
+		InstanceType:                   input.InstanceType,
+		StateDiskSizeGB:                int32(input.StateDiskSizeGB),
+		Image:                          input.Image,
+		UserAssingedIdentity:           input.UserAssingedIdentity,
+		LoadBalancerBackendAddressPool: azure.BackendAddressPoolWorkerName + "-" + c.uid,
 	}
 
 	if err := c.createScaleSet(ctx, createNodesInput); err != nil {
@@ -30,13 +31,14 @@ func (c *Client) CreateInstances(ctx context.Context, input CreateInstancesInput
 
 	// Create coordinator scale set
 	createCoordinatorsInput := CreateScaleSetInput{
-		Name:                 "constellation-scale-set-coordinators-" + c.uid,
-		NamePrefix:           c.name + "-control-plane-" + c.uid + "-",
-		Count:                input.CountCoordinators,
-		InstanceType:         input.InstanceType,
-		StateDiskSizeGB:      int32(input.StateDiskSizeGB),
-		Image:                input.Image,
-		UserAssingedIdentity: input.UserAssingedIdentity,
+		Name:                           "constellation-scale-set-coordinators-" + c.uid,
+		NamePrefix:                     c.name + "-control-plane-" + c.uid + "-",
+		Count:                          input.CountCoordinators,
+		InstanceType:                   input.InstanceType,
+		StateDiskSizeGB:                int32(input.StateDiskSizeGB),
+		Image:                          input.Image,
+		UserAssingedIdentity:           input.UserAssingedIdentity,
+		LoadBalancerBackendAddressPool: azure.BackendAddressPoolControlPlaneName + "-" + c.uid,
 	}
 
 	if err := c.createScaleSet(ctx, createCoordinatorsInput); err != nil {
@@ -58,6 +60,14 @@ func (c *Client) CreateInstances(ctx context.Context, input CreateInstancesInput
 	}
 	c.coordinators = instances
 
+	// Set the load balancer public IP in the first coordinator
+	coord, ok := c.coordinators["0"]
+	if !ok {
+		return errors.New("coordinator 0 not found")
+	}
+	coord.PublicIP = c.loadBalancerPubIP
+	c.coordinators["0"] = coord
+
 	return nil
 }
 
@@ -119,13 +129,13 @@ func (c *Client) CreateInstancesVMs(ctx context.Context, input CreateInstancesIn
 // TODO: deprecate as soon as scale sets are available.
 func (c *Client) createInstanceVM(ctx context.Context, input azure.VMInstance) (azure.Instance, error) {
 	pubIPName := input.Name + "-pubIP"
-	pubIPID, err := c.createPublicIPAddress(ctx, pubIPName)
+	pubIP, err := c.createPublicIPAddress(ctx, pubIPName)
 	if err != nil {
 		return azure.Instance{}, err
 	}
 
 	nicName := input.Name + "-NIC"
-	privIP, nicID, err := c.createNIC(ctx, nicName, pubIPID)
+	privIP, nicID, err := c.createNIC(ctx, nicName, *pubIP.ID)
 	if err != nil {
 		return azure.Instance{}, err
 	}
@@ -167,18 +177,22 @@ func (c *Client) createScaleSet(ctx context.Context, input CreateScaleSetInput)
 		return err
 	}
 	scaleSet := azure.ScaleSet{
-		Name:                 input.Name,
-		NamePrefix:           input.NamePrefix,
-		Location:             c.location,
-		InstanceType:         input.InstanceType,
-		StateDiskSizeGB:      input.StateDiskSizeGB,
-		Count:                int64(input.Count),
-		Username:             "constellation",
-		SubnetID:             c.subnetID,
-		NetworkSecurityGroup: c.networkSecurityGroup,
-		Image:                input.Image,
-		Password:             pw,
-		UserAssignedIdentity: input.UserAssingedIdentity,
+		Name:                           input.Name,
+		NamePrefix:                     input.NamePrefix,
+		Location:                       c.location,
+		InstanceType:                   input.InstanceType,
+		StateDiskSizeGB:                input.StateDiskSizeGB,
+		Count:                          int64(input.Count),
+		Username:                       "constellation",
+		SubnetID:                       c.subnetID,
+		NetworkSecurityGroup:           c.networkSecurityGroup,
+		Image:                          input.Image,
+		Password:                       pw,
+		UserAssignedIdentity:           input.UserAssingedIdentity,
+		Subscription:                   c.subscriptionID,
+		ResourceGroup:                  c.resourceGroup,
+		LoadBalancerName:               c.loadBalancerName,
+		LoadBalancerBackendAddressPool: input.LoadBalancerBackendAddressPool,
 	}.Azure()
 
 	poller, err := c.scaleSetsAPI.BeginCreateOrUpdate(
@@ -242,13 +256,14 @@ func (c *Client) getInstanceIPs(ctx context.Context, scaleSet string, count int)
 
 // CreateScaleSetInput is the input for a CreateScaleSet operation.
 type CreateScaleSetInput struct {
-	Name                 string
-	NamePrefix           string
-	Count                int
-	InstanceType         string
-	StateDiskSizeGB      int32
-	Image                string
-	UserAssingedIdentity string
+	Name                           string
+	NamePrefix                     string
+	Count                          int
+	InstanceType                   string
+	StateDiskSizeGB                int32
+	Image                          string
+	UserAssingedIdentity           string
+	LoadBalancerBackendAddressPool string
 }
 
 // CreateResourceGroup creates a resource group.

cli/azure/client/compute_test.go

@@ -216,6 +216,7 @@ func TestCreateInstances(t *testing.T) {
 				roleAssignmentsAPI:   tc.roleAssignmentsAPI,
 				nodes:                make(azure.Instances),
 				coordinators:         make(azure.Instances),
+				loadBalancerPubIP:    "lbip",
 			}
 
 			if tc.wantErr {
@@ -227,7 +228,7 @@ func TestCreateInstances(t *testing.T) {
 				assert.NotEmpty(client.nodes["0"].PrivateIP)
 				assert.NotEmpty(client.nodes["0"].PublicIP)
 				assert.NotEmpty(client.coordinators["0"].PrivateIP)
-				assert.NotEmpty(client.coordinators["0"].PublicIP)
+				assert.Equal("lbip", client.coordinators["0"].PublicIP)
 			}
 		})
 	}

cli/azure/client/network.go

@@ -6,21 +6,34 @@ import (
 
 	"github.com/Azure/azure-sdk-for-go/sdk/azcore/to"
 	"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork"
+	"github.com/edgelesssys/constellation/cli/azure"
 	"github.com/edgelesssys/constellation/cli/cloud/cloudtypes"
 )
 
 type createNetworkInput struct {
-	name         string
-	location     string
-	addressSpace string
+	name             string
+	location         string
+	addressSpace     string
+	nodeAddressSpace string
+	podAddressSpace  string
 }
 
+const (
+	nodeNetworkName     = "nodeNetwork"
+	podNetworkName      = "podNetwork"
+	networkAddressSpace = "10.0.0.0/8"
+	nodeAddressSpace    = "10.9.0.0/16"
+	podAddressSpace     = "10.10.0.0/16"
+)
+
 // CreateVirtualNetwork creates a virtual network.
 func (c *Client) CreateVirtualNetwork(ctx context.Context) error {
 	createNetworkInput := createNetworkInput{
-		name:         "constellation-" + c.uid,
-		location:     c.location,
-		addressSpace: "172.20.0.0/16",
+		name:             "constellation-" + c.uid,
+		location:         c.location,
+		addressSpace:     networkAddressSpace,
+		nodeAddressSpace: nodeAddressSpace,
+		podAddressSpace:  podAddressSpace,
 	}
 
 	poller, err := c.networksAPI.BeginCreateOrUpdate(
@@ -36,9 +49,15 @@ func (c *Client) CreateVirtualNetwork(ctx context.Context) error {
 				},
 				Subnets: []*armnetwork.Subnet{
 					{
-						Name: to.StringPtr("default"),
+						Name: to.StringPtr(nodeNetworkName),
 						Properties: &armnetwork.SubnetPropertiesFormat{
-							AddressPrefix: to.StringPtr(createNetworkInput.addressSpace),
+							AddressPrefix: to.StringPtr(createNetworkInput.nodeAddressSpace),
+						},
+					},
+					{
+						Name: to.StringPtr(podNetworkName),
+						Properties: &armnetwork.SubnetPropertiesFormat{
+							AddressPrefix: to.StringPtr(createNetworkInput.podAddressSpace),
 						},
 					},
 				},
@@ -141,25 +160,29 @@ func (c *Client) createNIC(ctx context.Context, name, publicIPAddressID string)
 		nil
 }
 
-// createPublicIPAddress creates a public IP address.
-// TODO: deprecate as soon as scale sets are available.
-func (c *Client) createPublicIPAddress(ctx context.Context, name string) (string, error) {
+func (c *Client) createPublicIPAddress(ctx context.Context, name string) (*armnetwork.PublicIPAddress, error) {
 	poller, err := c.publicIPAddressesAPI.BeginCreateOrUpdate(
 		ctx, c.resourceGroup, name,
 		armnetwork.PublicIPAddress{
 			Location: to.StringPtr(c.location),
+			SKU: &armnetwork.PublicIPAddressSKU{
+				Name: armnetwork.PublicIPAddressSKUNameStandard.ToPtr(),
+			},
+			Properties: &armnetwork.PublicIPAddressPropertiesFormat{
+				PublicIPAllocationMethod: armnetwork.IPAllocationMethodStatic.ToPtr(),
+			},
 		},
 		nil,
 	)
 	if err != nil {
-		return "", err
+		return nil, err
 	}
 	pollerResp, err := poller.PollUntilDone(ctx, 30*time.Second)
 	if err != nil {
-		return "", err
+		return nil, err
 	}
 
-	return *pollerResp.PublicIPAddressesClientCreateOrUpdateResult.PublicIPAddress.ID, nil
+	return &pollerResp.PublicIPAddressesClientCreateOrUpdateResult.PublicIPAddress, nil
 }
 
 // NetworkSecurityGroupInput defines firewall rules to be set.
@@ -167,3 +190,42 @@ type NetworkSecurityGroupInput struct {
 	Ingress cloudtypes.Firewall
 	Egress  cloudtypes.Firewall
 }
+
+// CreateExternalLoadBalancer creates an external load balancer.
+func (c *Client) CreateExternalLoadBalancer(ctx context.Context) error {
+	// First, create a public IP address for the load balancer.
+	publicIPAddress, err := c.createPublicIPAddress(ctx, "loadbalancer-public-ip-"+c.uid)
+	if err != nil {
+		return err
+	}
+
+	// Then, create the load balancer.
+	loadBalancerName := "constellation-load-balancer-" + c.uid
+	loadBalancer := azure.LoadBalancer{
+		Name:          loadBalancerName,
+		Location:      c.location,
+		ResourceGroup: c.resourceGroup,
+		Subscription:  c.subscriptionID,
+		PublicIPID:    *publicIPAddress.ID,
+		UID:           c.uid,
+	}
+	azureLoadBalancer := loadBalancer.Azure()
+
+	poller, err := c.loadBalancersAPI.BeginCreateOrUpdate(
+		ctx, c.resourceGroup, loadBalancerName,
+		azureLoadBalancer,
+		nil,
+	)
+	if err != nil {
+		return err
+	}
+
+	_, err = poller.PollUntilDone(ctx, 30*time.Second)
+	if err != nil {
+		return err
+	}
+	c.loadBalancerName = loadBalancerName
+
+	c.loadBalancerPubIP = *publicIPAddress.Properties.IPAddress
+	return nil
+}

cli/azure/client/network_test.go

@@ -170,7 +170,6 @@ func TestCreateNIC(t *testing.T) {
 	}
 }
 
-// TODO: deprecate as soon as scale sets are available.
 func TestCreatePublicIPAddress(t *testing.T) {
 	someErr := errors.New("failed")
 
@@ -218,3 +217,58 @@ func TestCreatePublicIPAddress(t *testing.T) {
 		})
 	}
 }
+
+func TestCreateExternalLoadBalancer(t *testing.T) {
+	someErr := errors.New("failed")
+
+	testCases := map[string]struct {
+		publicIPAddressesAPI publicIPAddressesAPI
+		loadBalancersAPI     loadBalancersAPI
+		wantErr              bool
+	}{
+		"successful create": {
+			publicIPAddressesAPI: stubPublicIPAddressesAPI{stubCreateResponse: stubPublicIPAddressesClientCreateOrUpdatePollerResponse{}},
+			loadBalancersAPI:     stubLoadBalancersAPI{},
+		},
+		"failed to get response from successful create": {
+			loadBalancersAPI:     stubLoadBalancersAPI{stubResponse: stubLoadBalancersClientCreateOrUpdatePollerResponse{pollErr: someErr}},
+			publicIPAddressesAPI: stubPublicIPAddressesAPI{},
+			wantErr:              true,
+		},
+		"failed create": {
+			loadBalancersAPI:     stubLoadBalancersAPI{createErr: someErr},
+			publicIPAddressesAPI: stubPublicIPAddressesAPI{},
+			wantErr:              true,
+		},
+		"cannot create public IP": {
+			publicIPAddressesAPI: stubPublicIPAddressesAPI{createErr: someErr},
+			loadBalancersAPI:     stubLoadBalancersAPI{},
+			wantErr:              true,
+		},
+	}
+
+	for name, tc := range testCases {
+		t.Run(name, func(t *testing.T) {
+			assert := assert.New(t)
+
+			ctx := context.Background()
+			client := Client{
+				resourceGroup:        "resource-group",
+				location:             "location",
+				name:                 "name",
+				uid:                  "uid",
+				nodes:                make(azure.Instances),
+				coordinators:         make(azure.Instances),
+				loadBalancersAPI:     tc.loadBalancersAPI,
+				publicIPAddressesAPI: tc.publicIPAddressesAPI,
+			}
+
+			err := client.CreateExternalLoadBalancer(ctx)
+			if tc.wantErr {
+				assert.Error(err)
+			} else {
+				assert.NoError(err)
+			}
+		})
+	}
+}

cli/azure/loadbalancer.go

@@ -0,0 +1,162 @@
+package azure
+
+import (
+	"github.com/Azure/azure-sdk-for-go/sdk/azcore/to"
+	"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork"
+	"github.com/edgelesssys/constellation/internal/constants"
+)
+
+// LoadBalancer defines a Azure load balancer.
+type LoadBalancer struct {
+	Name          string
+	Subscription  string
+	ResourceGroup string
+	Location      string
+	PublicIPID    string
+	UID           string
+}
+
+const (
+	BackendAddressPoolWorkerName       = "backendAddressWorkerPool"
+	BackendAddressPoolControlPlaneName = "backendAddressControlPlanePool"
+)
+
+// Azure returns a Azure representation of LoadBalancer.
+func (l LoadBalancer) Azure() armnetwork.LoadBalancer {
+	frontEndIPConfigName := "frontEndIPConfig"
+	kubeHealthProbeName := "kubeHealthProbe"
+	coordHealthProbeName := "coordHealthProbe"
+	debugdHealthProbeName := "debugdHealthProbe"
+	backEndAddressPoolNodeName := BackendAddressPoolWorkerName + "-" + l.UID
+	backEndAddressPoolControlPlaneName := BackendAddressPoolControlPlaneName + "-" + l.UID
+
+	return armnetwork.LoadBalancer{
+		Name:     to.StringPtr(l.Name),
+		Location: to.StringPtr(l.Location),
+		SKU:      &armnetwork.LoadBalancerSKU{Name: armnetwork.LoadBalancerSKUNameStandard.ToPtr()},
+		Properties: &armnetwork.LoadBalancerPropertiesFormat{
+			FrontendIPConfigurations: []*armnetwork.FrontendIPConfiguration{
+				{
+					Name: to.StringPtr(frontEndIPConfigName),
+					Properties: &armnetwork.FrontendIPConfigurationPropertiesFormat{
+						PublicIPAddress: &armnetwork.PublicIPAddress{
+							ID: to.StringPtr(l.PublicIPID),
+						},
+					},
+				},
+			},
+			BackendAddressPools: []*armnetwork.BackendAddressPool{
+				{
+					Name: to.StringPtr(backEndAddressPoolNodeName),
+				},
+				{
+					Name: to.StringPtr(backEndAddressPoolControlPlaneName),
+				},
+				{
+					Name: to.StringPtr("all"),
+				},
+			},
+			Probes: []*armnetwork.Probe{
+				{
+					Name: to.StringPtr(kubeHealthProbeName),
+					Properties: &armnetwork.ProbePropertiesFormat{
+						Protocol: armnetwork.ProbeProtocolTCP.ToPtr(),
+						Port:     to.Int32Ptr(int32(6443)),
+					},
+				},
+				{
+					Name: to.StringPtr(coordHealthProbeName),
+					Properties: &armnetwork.ProbePropertiesFormat{
+						Protocol: armnetwork.ProbeProtocolTCP.ToPtr(),
+						Port:     to.Int32Ptr(int32(constants.CoordinatorPort)),
+					},
+				},
+				{
+					Name: to.StringPtr(debugdHealthProbeName),
+					Properties: &armnetwork.ProbePropertiesFormat{
+						Protocol: armnetwork.ProbeProtocolTCP.ToPtr(),
+						Port:     to.Int32Ptr(int32(4000)),
+					},
+				},
+			},
+			LoadBalancingRules: []*armnetwork.LoadBalancingRule{
+				{
+					Name: to.StringPtr("kubeLoadBalancerRule"),
+					Properties: &armnetwork.LoadBalancingRulePropertiesFormat{
+						FrontendIPConfiguration: &armnetwork.SubResource{
+							ID: to.StringPtr("/subscriptions/" + l.Subscription + "/resourceGroups/" + l.ResourceGroup + "/providers/Microsoft.Network/loadBalancers/" + l.Name + "/frontendIPConfigurations/" + frontEndIPConfigName),
+						},
+						FrontendPort: to.Int32Ptr(int32(6443)),
+						BackendPort:  to.Int32Ptr(int32(6443)),
+						Protocol:     armnetwork.TransportProtocolTCP.ToPtr(),
+						Probe: &armnetwork.SubResource{
+							ID: to.StringPtr("/subscriptions/" + l.Subscription + "/resourceGroups/" + l.ResourceGroup + "/providers/Microsoft.Network/loadBalancers/" + l.Name + "/probes/" + kubeHealthProbeName),
+						},
+						DisableOutboundSnat: to.BoolPtr(true),
+						BackendAddressPools: []*armnetwork.SubResource{
+							{
+								ID: to.StringPtr("/subscriptions/" + l.Subscription + "/resourceGroups/" + l.ResourceGroup + "/providers/Microsoft.Network/loadBalancers/" + l.Name + "/backendAddressPools/" + backEndAddressPoolControlPlaneName),
+							},
+						},
+					},
+				},
+				{
+					Name: to.StringPtr("coordLoadBalancerRule"),
+					Properties: &armnetwork.LoadBalancingRulePropertiesFormat{
+						FrontendIPConfiguration: &armnetwork.SubResource{
+							ID: to.StringPtr("/subscriptions/" + l.Subscription + "/resourceGroups/" + l.ResourceGroup + "/providers/Microsoft.Network/loadBalancers/" + l.Name + "/frontendIPConfigurations/" + frontEndIPConfigName),
+						},
+						FrontendPort: to.Int32Ptr(int32(constants.CoordinatorPort)),
+						BackendPort:  to.Int32Ptr(int32(constants.CoordinatorPort)),
+						Protocol:     armnetwork.TransportProtocolTCP.ToPtr(),
+						Probe: &armnetwork.SubResource{
+							ID: to.StringPtr("/subscriptions/" + l.Subscription + "/resourceGroups/" + l.ResourceGroup + "/providers/Microsoft.Network/loadBalancers/" + l.Name + "/probes/" + coordHealthProbeName),
+						},
+						DisableOutboundSnat: to.BoolPtr(true),
+						BackendAddressPools: []*armnetwork.SubResource{
+							{
+								ID: to.StringPtr("/subscriptions/" + l.Subscription + "/resourceGroups/" + l.ResourceGroup + "/providers/Microsoft.Network/loadBalancers/" + l.Name + "/backendAddressPools/" + backEndAddressPoolControlPlaneName),
+							},
+						},
+					},
+				},
+				{
+					Name: to.StringPtr("debudLoadBalancerRule"),
+					Properties: &armnetwork.LoadBalancingRulePropertiesFormat{
+						FrontendIPConfiguration: &armnetwork.SubResource{
+							ID: to.StringPtr("/subscriptions/" + l.Subscription + "/resourceGroups/" + l.ResourceGroup + "/providers/Microsoft.Network/loadBalancers/" + l.Name + "/frontendIPConfigurations/" + frontEndIPConfigName),
+						},
+						FrontendPort: to.Int32Ptr(int32(4000)),
+						BackendPort:  to.Int32Ptr(int32(4000)),
+						Protocol:     armnetwork.TransportProtocolTCP.ToPtr(),
+						Probe: &armnetwork.SubResource{
+							ID: to.StringPtr("/subscriptions/" + l.Subscription + "/resourceGroups/" + l.ResourceGroup + "/providers/Microsoft.Network/loadBalancers/" + l.Name + "/probes/" + debugdHealthProbeName),
+						},
+						DisableOutboundSnat: to.BoolPtr(true),
+						BackendAddressPools: []*armnetwork.SubResource{
+							{
+								ID: to.StringPtr("/subscriptions/" + l.Subscription + "/resourceGroups/" + l.ResourceGroup + "/providers/Microsoft.Network/loadBalancers/" + l.Name + "/backendAddressPools/" + backEndAddressPoolControlPlaneName),
+							},
+						},
+					},
+				},
+			},
+			OutboundRules: []*armnetwork.OutboundRule{
+				{
+					Name: to.StringPtr("outboundRuleControlPlane"),
+					Properties: &armnetwork.OutboundRulePropertiesFormat{
+						FrontendIPConfigurations: []*armnetwork.SubResource{
+							{
+								ID: to.StringPtr("/subscriptions/" + l.Subscription + "/resourceGroups/" + l.ResourceGroup + "/providers/Microsoft.Network/loadBalancers/" + l.Name + "/frontendIPConfigurations/" + frontEndIPConfigName),
+							},
+						},
+						BackendAddressPool: &armnetwork.SubResource{
+							ID: to.StringPtr("/subscriptions/" + l.Subscription + "/resourceGroups/" + l.ResourceGroup + "/providers/Microsoft.Network/loadBalancers/" + l.Name + "/backendAddressPools/all"),
+						},
+						Protocol: armnetwork.LoadBalancerOutboundRuleProtocolAll.ToPtr(),
+					},
+				},
+			},
+		},
+	}
+}

cli/azure/scaleset.go

@@ -10,18 +10,22 @@ import (
 
 // ScaleSet defines a Azure scale set.
 type ScaleSet struct {
-	Name                 string
-	NamePrefix           string
-	Location             string
-	InstanceType         string
-	StateDiskSizeGB      int32
-	Count                int64
-	Username             string
-	SubnetID             string
-	NetworkSecurityGroup string
-	Password             string
-	Image                string
-	UserAssignedIdentity string
+	Name                           string
+	NamePrefix                     string
+	Subscription                   string
+	ResourceGroup                  string
+	Location                       string
+	InstanceType                   string
+	StateDiskSizeGB                int32
+	Count                          int64
+	Username                       string
+	SubnetID                       string
+	NetworkSecurityGroup           string
+	Password                       string
+	Image                          string
+	UserAssignedIdentity           string
+	LoadBalancerName               string
+	LoadBalancerBackendAddressPool string
 }
 
 // Azure returns the Azure representation of ScaleSet.
@@ -72,13 +76,16 @@ func (s ScaleSet) Azure() armcompute.VirtualMachineScaleSet {
 									{
 										Name: to.StringPtr(s.Name),
 										Properties: &armcompute.VirtualMachineScaleSetIPConfigurationProperties{
+											Primary: to.BoolPtr(true),
 											Subnet: &armcompute.APIEntityReference{
 												ID: to.StringPtr(s.SubnetID),
 											},
-											PublicIPAddressConfiguration: &armcompute.VirtualMachineScaleSetPublicIPAddressConfiguration{
-												Name: to.StringPtr(s.Name),
-												Properties: &armcompute.VirtualMachineScaleSetPublicIPAddressConfigurationProperties{
-													IdleTimeoutInMinutes: to.Int32Ptr(15), // default per https://docs.microsoft.com/en-us/azure/virtual-machine-scale-sets/virtual-machine-scale-sets-networking#creating-a-scale-set-with-public-ip-per-virtual-machine
+											LoadBalancerBackendAddressPools: []*armcompute.SubResource{
+												{
+													ID: to.StringPtr("/subscriptions/" + s.Subscription + "/resourcegroups/" + s.ResourceGroup + "/providers/Microsoft.Network/loadBalancers/" + s.LoadBalancerName + "/backendAddressPools/" + s.LoadBalancerBackendAddressPool),
+												},
+												{
+													ID: to.StringPtr("/subscriptions/" + s.Subscription + "/resourcegroups/" + s.ResourceGroup + "/providers/Microsoft.Network/loadBalancers/" + s.LoadBalancerName + "/backendAddressPools/all"),
 												},
 											},
 										},

cli/cloud/cloudcmd/clients.go

@@ -26,6 +26,7 @@ type azureclient interface {
 	GetState() (state.ConstellationState, error)
 	SetState(state.ConstellationState) error
 	CreateResourceGroup(ctx context.Context) error
+	CreateExternalLoadBalancer(ctx context.Context) error
 	CreateVirtualNetwork(ctx context.Context) error
 	CreateSecurityGroup(ctx context.Context, input azurecl.NetworkSecurityGroupInput) error
 	CreateInstances(ctx context.Context, input azurecl.CreateInstancesInput) error

cli/cloud/cloudcmd/clients_test.go

@@ -24,6 +24,7 @@ type fakeAzureClient struct {
 	subscriptionID       string
 	tenantID             string
 	subnetID             string
+	loadBalancerName     string
 	coordinatorsScaleSet string
 	nodesScaleSet        string
 	networkSecurityGroup string
@@ -77,6 +78,11 @@ func (c *fakeAzureClient) CreateVirtualNetwork(ctx context.Context) error {
 	return nil
 }
 
+func (c *fakeAzureClient) CreateExternalLoadBalancer(ctx context.Context) error {
+	c.loadBalancerName = "loadBalancer"
+	return nil
+}
+
 func (c *fakeAzureClient) CreateSecurityGroup(ctx context.Context, input azurecl.NetworkSecurityGroupInput) error {
 	c.networkSecurityGroup = "network-security-group"
 	return nil
@@ -152,6 +158,7 @@ type stubAzureClient struct {
 	createResourceGroupErr       error
 	createVirtualNetworkErr      error
 	createSecurityGroupErr       error
+	createLoadBalancerErr        error
 	createInstancesErr           error
 	createServicePrincipalErr    error
 	terminateResourceGroupErr    error
@@ -166,6 +173,10 @@ func (c *stubAzureClient) SetState(state.ConstellationState) error {
 	return c.setStateErr
 }
 
+func (c *stubAzureClient) CreateExternalLoadBalancer(ctx context.Context) error {
+	return c.createLoadBalancerErr
+}
+
 func (c *stubAzureClient) CreateResourceGroup(ctx context.Context) error {
 	return c.createResourceGroupErr
 }
@@ -271,11 +282,9 @@ func (c *fakeGcpClient) CreateFirewall(ctx context.Context, input gcpcl.Firewall
 	if c.network == "" {
 		return errors.New("client has not network")
 	}
-	var firewalls []string
 	for _, rule := range input.Ingress {
-		firewalls = append(firewalls, rule.Name)
+		c.firewalls = append(c.firewalls, rule.Name)
 	}
-	c.firewalls = firewalls
 	return nil
 }
 

cli/cloud/cloudcmd/create.go

@@ -82,6 +82,45 @@ func (c *Creator) createGCP(ctx context.Context, cl gcpclient, config *config.Co
 		return state.ConstellationState{}, err
 	}
 
+	// additionally create allow-internal rules
+	internalFirewallInput := gcpcl.FirewallInput{
+		Ingress: cloudtypes.Firewall{
+			{
+				Name:     "allow-cluster-internal-tcp",
+				Protocol: "tcp",
+				IPRange:  gcpcl.SubnetExtCIDR,
+			},
+			{
+				Name:     "allow-cluster-internal-udp",
+				Protocol: "udp",
+				IPRange:  gcpcl.SubnetExtCIDR,
+			},
+			{
+				Name:     "allow-cluster-internal-icmp",
+				Protocol: "icmp",
+				IPRange:  gcpcl.SubnetExtCIDR,
+			},
+			{
+				Name:     "allow-node-internal-tcp",
+				Protocol: "tcp",
+				IPRange:  gcpcl.SubnetCIDR,
+			},
+			{
+				Name:     "allow-node-internal-udp",
+				Protocol: "udp",
+				IPRange:  gcpcl.SubnetCIDR,
+			},
+			{
+				Name:     "allow-node-internal-icmp",
+				Protocol: "icmp",
+				IPRange:  gcpcl.SubnetCIDR,
+			},
+		},
+	}
+	if err := cl.CreateFirewall(ctx, internalFirewallInput); err != nil {
+		return state.ConstellationState{}, err
+	}
+
 	createInput := client.CreateInstancesInput{
 		CountCoordinators: coordCount,
 		CountNodes:        nodeCount,
@@ -104,6 +143,9 @@ func (c *Creator) createAzure(ctx context.Context, cl azureclient, config *confi
 	if err := cl.CreateResourceGroup(ctx); err != nil {
 		return state.ConstellationState{}, err
 	}
+	if err := cl.CreateExternalLoadBalancer(ctx); err != nil {
+		return state.ConstellationState{}, err
+	}
 	if err := cl.CreateVirtualNetwork(ctx); err != nil {
 		return state.ConstellationState{}, err
 	}

cli/cloud/cloudcmd/create_test.go

@@ -33,7 +33,11 @@ func TestCreator(t *testing.T) {
 		GCPCoordinatorInstanceTemplate: "coordinator-template",
 		GCPNetwork:                     "network",
 		GCPSubnetwork:                  "subnetwork",
-		GCPFirewalls:                   []string{"coordinator", "wireguard", "ssh", "nodeport"},
+		GCPFirewalls: []string{
+			"coordinator", "wireguard", "ssh", "nodeport", "kubernetes",
+			"allow-cluster-internal-tcp", "allow-cluster-internal-udp", "allow-cluster-internal-icmp",
+			"allow-node-internal-tcp", "allow-node-internal-udp", "allow-node-internal-icmp",
+		},
 	}
 
 	wantAzureState := state.ConstellationState{

cli/cloud/cloudtypes/firewall.go

@@ -19,25 +19,30 @@ type Firewall config.Firewall
 func (f Firewall) GCP() ([]*computepb.Firewall, error) {
 	var fw []*computepb.Firewall
 	for _, rule := range f {
-		var destRange []string = nil
+		var srcRange []string
 		if rule.IPRange != "" {
-			destRange = append(destRange, rule.IPRange)
+			srcRange = []string{rule.IPRange}
 		}
 
-		ports, err := portOrRange(rule.FromPort, rule.ToPort)
-		if err != nil {
-			return nil, err
+		var ports []string
+		if rule.FromPort != 0 || rule.ToPort != 0 {
+			port, err := portOrRange(rule.FromPort, rule.ToPort)
+			if err != nil {
+				return nil, err
+			}
+			ports = []string{port}
 		}
+
 		fw = append(fw, &computepb.Firewall{
 			Allowed: []*computepb.Allowed{
 				{
 					IPProtocol: proto.String(rule.Protocol),
-					Ports:      []string{ports},
+					Ports:      ports,
 				},
 			},
-			Description:       proto.String(rule.Description),
-			DestinationRanges: destRange,
-			Name:              proto.String(rule.Name),
+			Description:  proto.String(rule.Description),
+			SourceRanges: srcRange,
+			Name:         proto.String(rule.Name),
 		})
 	}
 	return fw, nil

cli/cloud/cloudtypes/firewall_test.go

@@ -29,11 +29,18 @@ func TestFirewallGCP(t *testing.T) {
 			IPRange:     "",
 			FromPort:    51820,
 		},
+		{
+			Name:        "test-3",
+			Description: "This is the Test-3 Permission",
+			Protocol:    "tcp",
+			IPRange:     "192.0.2.0/24",
+			FromPort:    4000,
+		},
 	}
 
 	firewalls, err := testFw.GCP()
 	assert.NoError(err)
-	assert.Equal(2, len(firewalls))
+	assert.Equal(len(testFw), len(firewalls))
 
 	// Check permissions
 	for i := 0; i < len(testFw); i++ {
@@ -47,6 +54,11 @@ func TestFirewallGCP(t *testing.T) {
 
 		assert.Equal(testFw[i].Name, firewall1.GetName())
 		assert.Equal(testFw[i].Description, firewall1.GetDescription())
+
+		if testFw[i].IPRange != "" {
+			require.Len(firewall1.GetSourceRanges(), 1)
+			assert.Equal(testFw[i].IPRange, firewall1.GetSourceRanges()[0])
+		}
 	}
 }
 

cli/cmd/init.go

@@ -357,6 +357,9 @@ func readOrGenerateVPNKey(fileHandler file.Handler, privKeyPath string) (privKey
 func ipsToEndpoints(ips []string, port string) []string {
 	var endpoints []string
 	for _, ip := range ips {
+		if ip == "" {
+			continue
+		}
 		endpoints = append(endpoints, net.JoinHostPort(ip, port))
 	}
 	return endpoints

cli/cmd/init_test.go

@@ -355,7 +355,7 @@ func TestWriteOutput(t *testing.T) {
 func TestIpsToEndpoints(t *testing.T) {
 	assert := assert.New(t)
 
-	ips := []string{"192.0.2.1", "192.0.2.2", "192.0.2.3"}
+	ips := []string{"192.0.2.1", "192.0.2.2", "", "192.0.2.3"}
 	port := "8080"
 	endpoints := ipsToEndpoints(ips, port)
 	assert.Equal([]string{"192.0.2.1:8080", "192.0.2.2:8080", "192.0.2.3:8080"}, endpoints)

cli/gcp/client/instances.go

@@ -48,18 +48,19 @@ func (c *Client) CreateInstances(ctx context.Context, input CreateInstancesInput
 	c.nodeTemplate = nodeTemplateInput.Name
 
 	coordinatorTemplateInput := insertInstanceTemplateInput{
-		Name:            c.name + "-control-plane-" + c.uid,
-		Network:         c.network,
-		Subnetwork:      c.subnetwork,
-		ImageId:         input.ImageId,
-		InstanceType:    input.InstanceType,
-		StateDiskSizeGB: int64(input.StateDiskSizeGB),
-		Role:            role.Coordinator.String(),
-		KubeEnv:         input.KubeEnv,
-		Project:         c.project,
-		Zone:            c.zone,
-		Region:          c.region,
-		UID:             c.uid,
+		Name:                         c.name + "-control-plane-" + c.uid,
+		Network:                      c.network,
+		Subnetwork:                   c.subnetwork,
+		SecondarySubnetworkRangeName: c.secondarySubnetworkRange,
+		ImageId:                      input.ImageId,
+		InstanceType:                 input.InstanceType,
+		StateDiskSizeGB:              int64(input.StateDiskSizeGB),
+		Role:                         role.Coordinator.String(),
+		KubeEnv:                      input.KubeEnv,
+		Project:                      c.project,
+		Zone:                         c.zone,
+		Region:                       c.region,
+		UID:                          c.uid,
 	}
 	op, err = c.insertInstanceTemplate(ctx, coordinatorTemplateInput)
 	if err != nil {
@@ -72,6 +73,21 @@ func (c *Client) CreateInstances(ctx context.Context, input CreateInstancesInput
 	}
 	ops = []Operation{}
 
+	coordinatorGroupInput := instanceGroupManagerInput{
+		Count:    input.CountCoordinators,
+		Name:     strings.Join([]string{c.name, "control-plane", c.uid}, "-"),
+		Template: c.coordinatorTemplate,
+		UID:      c.uid,
+		Project:  c.project,
+		Zone:     c.zone,
+	}
+	op, err = c.insertInstanceGroupManger(ctx, coordinatorGroupInput)
+	if err != nil {
+		return fmt.Errorf("inserting instanceGroupManager failed: %w", err)
+	}
+	ops = append(ops, op)
+	c.coordinatorInstanceGroup = coordinatorGroupInput.Name
+
 	nodeGroupInput := instanceGroupManagerInput{
 		Count:    input.CountNodes,
 		Name:     strings.Join([]string{c.name, "worker", c.uid}, "-"),
@@ -87,20 +103,6 @@ func (c *Client) CreateInstances(ctx context.Context, input CreateInstancesInput
 	ops = append(ops, op)
 	c.nodesInstanceGroup = nodeGroupInput.Name
 
-	coordinatorGroupInput := instanceGroupManagerInput{
-		Count:    input.CountCoordinators,
-		Name:     strings.Join([]string{c.name, "control-plane", c.uid}, "-"),
-		Template: c.coordinatorTemplate,
-		UID:      c.uid,
-		Project:  c.project,
-		Zone:     c.zone,
-	}
-	op, err = c.insertInstanceGroupManger(ctx, coordinatorGroupInput)
-	if err != nil {
-		return fmt.Errorf("inserting instanceGroupManager failed: %w", err)
-	}
-	ops = append(ops, op)
-	c.coordinatorInstanceGroup = coordinatorGroupInput.Name
 	if err := c.waitForOperations(ctx, ops); err != nil {
 		return err
 	}
@@ -390,7 +392,7 @@ func (i insertInstanceTemplateInput) insertInstanceTemplateRequest() *computepb.
 					EnableVtpm:                proto.Bool(true),
 				},
 				Tags: &computepb.Tags{
-					Items: []string{"constellation"},
+					Items: []string{"constellation-" + i.UID},
 				},
 			},
 		},

cli/gcp/client/network.go

@@ -9,6 +9,11 @@ import (
 	"google.golang.org/protobuf/proto"
 )
 
+const (
+	SubnetCIDR    = "192.168.178.0/24"
+	SubnetExtCIDR = "10.10.0.0/16"
+)
+
 // CreateFirewall creates a set of firewall rules for the client's network.
 //
 // The client must have a VPC network to set firewall rules.
@@ -163,13 +168,13 @@ func (c *Client) createSubnet(ctx context.Context, name, network, secondaryRange
 		Project: c.project,
 		Region:  c.region,
 		SubnetworkResource: &computepb.Subnetwork{
-			IpCidrRange: proto.String("192.168.178.0/24"),
+			IpCidrRange: proto.String(SubnetCIDR),
 			Name:        proto.String(name),
 			Network:     proto.String("projects/" + c.project + "/global/networks/" + network),
 			SecondaryIpRanges: []*computepb.SubnetworkSecondaryRange{
 				{
 					RangeName:   proto.String(secondaryRangeName),
-					IpCidrRange: proto.String("10.10.0.0/16"),
+					IpCidrRange: proto.String(SubnetExtCIDR),
 				},
 			},
 		},