Cloud provider Azure: adopt changes to CCM / CNM for Azure

cli/azure/client/activedirectory.go

@@ -48,6 +48,7 @@ func (c *Client) CreateServicePrincipal(ctx context.Context) (string, error) {
 	}
 
 	return ApplicationCredentials{
+		TenantID:     c.tenantID,
 		ClientID:     createAppRes.AppID,
 		ClientSecret: clientSecret,
 	}.ConvertToCloudServiceAccountURI(), nil
@@ -165,6 +166,7 @@ func (c *Client) assignResourceGroupRole(ctx context.Context, principalID, roleD
 // ApplicationCredentials is a set of Azure AD application credentials.
 // It is the equivalent of a service account key in other cloud providers.
 type ApplicationCredentials struct {
+	TenantID     string
 	ClientID     string
 	ClientSecret string
 }
@@ -172,6 +174,7 @@ type ApplicationCredentials struct {
 // ConvertToCloudServiceAccountURI converts the ApplicationCredentials into a cloud service account URI.
 func (c ApplicationCredentials) ConvertToCloudServiceAccountURI() string {
 	query := url.Values{}
+	query.Add("tenant_id", c.TenantID)
 	query.Add("client_id", c.ClientID)
 	query.Add("client_secret", c.ClientSecret)
 	uri := url.URL{

cli/azure/client/activedirectory_test.go

@@ -363,6 +363,7 @@ func TestConvertToCloudServiceAccountURI(t *testing.T) {
 	assert := assert.New(t)
 	require := require.New(t)
 	key := ApplicationCredentials{
+		TenantID:     "tenant-id",
 		ClientID:     "client-id",
 		ClientSecret: "client-secret",
 	}
@@ -374,6 +375,7 @@ func TestConvertToCloudServiceAccountURI(t *testing.T) {
 	assert.Equal("serviceaccount", uri.Scheme)
 	assert.Equal("azure", uri.Host)
 	assert.Equal(url.Values{
+		"tenant_id":     []string{"tenant-id"},
 		"client_id":     []string{"client-id"},
 		"client_secret": []string{"client-secret"},
 	}, query)

coordinator/cloudprovider/azure/autoscaler.go

@@ -4,11 +4,11 @@ package azure
 type Autoscaler struct{}
 
 // Name returns the cloud-provider name as used by k8s cluster-autoscaler.
-func (a Autoscaler) Name() string {
+func (a *Autoscaler) Name() string {
 	return "azure"
 }
 
 // Supported is used to determine if we support autoscaling for the cloud provider.
-func (a Autoscaler) Supported() bool {
+func (a *Autoscaler) Supported() bool {
 	return false
 }

coordinator/cloudprovider/azure/ccm.go

@@ -1,33 +1,150 @@
 package azure
 
-import "github.com/edgelesssys/constellation/coordinator/core"
+import (
+	"encoding/json"
+
+	"github.com/edgelesssys/constellation/coordinator/cloudprovider"
+	"github.com/edgelesssys/constellation/coordinator/core"
+	"github.com/edgelesssys/constellation/coordinator/kubernetes/k8sapi/resources"
+	k8s "k8s.io/api/core/v1"
+	meta "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
 
 // CloudControllerManager holds the Azure cloud-controller-manager configuration.
 type CloudControllerManager struct{}
 
 // Image returns the container image used to provide cloud-controller-manager for the cloud-provider.
-func (c CloudControllerManager) Image() string {
+func (c *CloudControllerManager) Image() string {
-	return ""
+	return cloudprovider.CloudControllerManagerImageAzure
 }
 
 // Path returns the path used by cloud-controller-manager executable within the container image.
-func (c CloudControllerManager) Path() string {
+func (c *CloudControllerManager) Path() string {
-	return ""
+	return "cloud-controller-manager"
 }
 
 // Name returns the cloud-provider name as used by k8s cloud-controller-manager (k8s.gcr.io/cloud-controller-manager).
-func (c CloudControllerManager) Name() string {
+func (c *CloudControllerManager) Name() string {
-	return ""
+	return "azure"
+}
+
+// ExtraArgs returns a list of arguments to append to the cloud-controller-manager command.
+func (c *CloudControllerManager) ExtraArgs() []string {
+	return []string{
+		"--controllers=*,-cloud-node",
+		"--cloud-config=/etc/azure/azure.json",
+	}
+}
+
+// ConfigMaps returns a list of ConfigMaps to deploy together with the k8s cloud-controller-manager
+// Reference: https://kubernetes.io/docs/concepts/configuration/configmap/ .
+func (c *CloudControllerManager) ConfigMaps(instance core.Instance) (resources.ConfigMaps, error) {
+	return resources.ConfigMaps{}, nil
+}
+
+// Secrets returns a list of secrets to deploy together with the k8s cloud-controller-manager.
+// Reference: https://kubernetes.io/docs/concepts/configuration/secret/ .
+func (c *CloudControllerManager) Secrets(instance core.Instance, cloudServiceAccountURI string) (resources.Secrets, error) {
+	// Azure CCM expects cloud provider config to contain cluster configuration and service principal client secrets
+	// reference: https://kubernetes-sigs.github.io/cloud-provider-azure/install/configs/
+
+	subscriptionID, resourceGroup, err := extractBasicsFromProviderID(instance.ProviderID)
+	if err != nil {
+		return resources.Secrets{}, err
+	}
+	creds, err := getApplicationCredentials(cloudServiceAccountURI)
+	if err != nil {
+		return resources.Secrets{}, err
+	}
+
+	config := cloudConfig{
+		Cloud:               "AzurePublicCloud",
+		TenantID:            creds.TenantID,
+		SubscriptionID:      subscriptionID,
+		ResourceGroup:       resourceGroup,
+		UseInstanceMetadata: true,
+		AADClientID:         creds.ClientID,
+		AADClientSecret:     creds.ClientSecret,
+	}
+
+	rawConfig, err := json.Marshal(config)
+	if err != nil {
+		return resources.Secrets{}, err
+	}
+
+	return resources.Secrets{
+		&k8s.Secret{
+			TypeMeta: meta.TypeMeta{
+				Kind:       "Secret",
+				APIVersion: "v1",
+			},
+			ObjectMeta: meta.ObjectMeta{
+				Name:      "azureconfig",
+				Namespace: "kube-system",
+			},
+			Data: map[string][]byte{
+				"azure.json": rawConfig,
+			},
+		},
+	}, nil
+}
+
+// Volumes returns a list of volumes to deploy together with the k8s cloud-controller-manager.
+// Reference: https://kubernetes.io/docs/concepts/storage/volumes/ .
+func (c *CloudControllerManager) Volumes() []k8s.Volume {
+	return []k8s.Volume{
+		{
+			Name: "azureconfig",
+			VolumeSource: k8s.VolumeSource{
+				Secret: &k8s.SecretVolumeSource{
+					SecretName: "azureconfig",
+				},
+			},
+		},
+	}
+}
+
+// VolumeMounts a list of of volume mounts to deploy together with the k8s cloud-controller-manager.
+func (c *CloudControllerManager) VolumeMounts() []k8s.VolumeMount {
+	return []k8s.VolumeMount{
+		{
+			Name:      "azureconfig",
+			ReadOnly:  true,
+			MountPath: "/etc/azure",
+		},
+	}
+}
+
+// Env returns a list of k8s environment key-value pairs to deploy together with the k8s cloud-controller-manager.
+func (c *CloudControllerManager) Env() []k8s.EnvVar {
+	return []k8s.EnvVar{}
 }
 
 // PrepareInstance is called on every instance before deploying the cloud-controller-manager.
 // Allows for cloud-provider specific hooks.
-func (c CloudControllerManager) PrepareInstance(instance core.Instance, vpnIP string) error {
+func (c *CloudControllerManager) PrepareInstance(instance core.Instance, vpnIP string) error {
 	// no specific hook required.
 	return nil
 }
 
 // Supported is used to determine if cloud controller manager is implemented for this cloud provider.
-func (c CloudControllerManager) Supported() bool {
+func (c *CloudControllerManager) Supported() bool {
-	return false
+	return true
+}
+
+type cloudConfig struct {
+	Cloud                      string `json:"cloud,omitempty"`
+	TenantID                   string `json:"tenantId,omitempty"`
+	SubscriptionID             string `json:"subscriptionId,omitempty"`
+	ResourceGroup              string `json:"resourceGroup,omitempty"`
+	Location                   string `json:"location,omitempty"`
+	SubnetName                 string `json:"subnetName,omitempty"`
+	SecurityGroupName          string `json:"securityGroupName,omitempty"`
+	SecurityGroupResourceGroup string `json:"securityGroupResourceGroup,omitempty"`
+	VNetName                   string `json:"vnetName,omitempty"`
+	VNetResourceGroup          string `json:"vnetResourceGroup,omitempty"`
+	CloudProviderBackoff       bool   `json:"cloudProviderBackoff,omitempty"`
+	UseInstanceMetadata        bool   `json:"useInstanceMetadata,omitempty"`
+	AADClientID                string `json:"aadClientId,omitempty"`
+	AADClientSecret            string `json:"aadClientSecret,omitempty"`
 }

coordinator/cloudprovider/azure/ccm_test.go

@@ -0,0 +1,82 @@
+package azure
+
+import (
+	"testing"
+
+	"github.com/edgelesssys/constellation/coordinator/core"
+	"github.com/edgelesssys/constellation/coordinator/kubernetes/k8sapi/resources"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	k8s "k8s.io/api/core/v1"
+	meta "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+func TestSecrets(t *testing.T) {
+	testCases := map[string]struct {
+		instance               core.Instance
+		cloudServiceAccountURI string
+		expectedSecrets        resources.Secrets
+		expectErr              bool
+	}{
+		"Secrets works": {
+			instance:               core.Instance{ProviderID: "azure:///subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/virtualMachines/instance-name"},
+			cloudServiceAccountURI: "serviceaccount://azure?tenant_id=tenant-id&client_id=client-id&client_secret=client-secret",
+			expectedSecrets: resources.Secrets{
+				&k8s.Secret{
+					TypeMeta: meta.TypeMeta{
+						Kind:       "Secret",
+						APIVersion: "v1",
+					},
+					ObjectMeta: meta.ObjectMeta{
+						Name:      "azureconfig",
+						Namespace: "kube-system",
+					},
+					Data: map[string][]byte{
+						"azure.json": []byte(`{"cloud":"AzurePublicCloud","tenantId":"tenant-id","subscriptionId":"subscription-id","resourceGroup":"resource-group","useInstanceMetadata":true,"aadClientId":"client-id","aadClientSecret":"client-secret"}`),
+					},
+				},
+			},
+		},
+		"invalid providerID fails": {
+			instance:  core.Instance{ProviderID: "invalid"},
+			expectErr: true,
+		},
+		"invalid cloudServiceAccountURI fails": {
+			instance:               core.Instance{ProviderID: "azure:///subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/virtualMachines/instance-name"},
+			cloudServiceAccountURI: "invalid",
+			expectErr:              true,
+		},
+	}
+
+	for name, tc := range testCases {
+		t.Run(name, func(t *testing.T) {
+			assert := assert.New(t)
+			require := require.New(t)
+
+			cloud := CloudControllerManager{}
+			secrets, err := cloud.Secrets(tc.instance, tc.cloudServiceAccountURI)
+			if tc.expectErr {
+				assert.Error(err)
+				return
+			}
+			require.NoError(err)
+			assert.Equal(tc.expectedSecrets, secrets)
+		})
+	}
+}
+
+func TestTrivialCCMFunctions(t *testing.T) {
+	assert := assert.New(t)
+	cloud := CloudControllerManager{}
+
+	assert.NotEmpty(cloud.Image())
+	assert.NotEmpty(cloud.Path())
+	assert.NotEmpty(cloud.Name())
+	assert.NotEmpty(cloud.ExtraArgs())
+	assert.Empty(cloud.ConfigMaps(core.Instance{}))
+	assert.NotEmpty(cloud.Volumes())
+	assert.NotEmpty(cloud.VolumeMounts())
+	assert.Empty(cloud.Env())
+	assert.NoError(cloud.PrepareInstance(core.Instance{}, "192.0.2.0"))
+	assert.True(cloud.Supported())
+}

coordinator/cloudprovider/azure/cloudnodemanager.go

@@ -0,0 +1,29 @@
+package azure
+
+import "github.com/edgelesssys/constellation/coordinator/cloudprovider"
+
+// CloudNodeManager holds the Azure cloud-node-manager configuration.
+// reference: https://raw.githubusercontent.com/kubernetes-sigs/cloud-provider-azure/master/examples/out-of-tree/cloud-node-manager.yaml .
+type CloudNodeManager struct{}
+
+// Image returns the container image used to provide cloud-node-manager for the cloud-provider.
+func (c *CloudNodeManager) Image() string {
+	return cloudprovider.CloudNodeManagerImageAzure
+}
+
+// Path returns the path used by cloud-node-manager executable within the container image.
+func (c *CloudNodeManager) Path() string {
+	return "cloud-node-manager"
+}
+
+// ExtraArgs returns a list of arguments to append to the cloud-node-manager command.
+func (c *CloudNodeManager) ExtraArgs() []string {
+	return []string{
+		"--wait-routes=false",
+	}
+}
+
+// Supported is used to determine if cloud node manager is implemented for this cloud provider.
+func (c *CloudNodeManager) Supported() bool {
+	return true
+}

coordinator/cloudprovider/azure/cloudnodemanager_test.go

@@ -0,0 +1,17 @@
+package azure
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestTrivialCNMFunctions(t *testing.T) {
+	assert := assert.New(t)
+	cloud := CloudNodeManager{}
+
+	assert.NotEmpty(cloud.Image())
+	assert.NotEmpty(cloud.Path())
+	assert.NotEmpty(cloud.ExtraArgs())
+	assert.True(cloud.Supported())
+}

coordinator/cloudprovider/azure/serviceaccounturi.go

@@ -0,0 +1,28 @@
+package azure
+
+import (
+	"fmt"
+	"net/url"
+
+	"github.com/edgelesssys/constellation/cli/azure/client"
+)
+
+// getApplicationCredentials converts a cloudServiceAccountURI into Azure ApplicationCredentials.
+func getApplicationCredentials(cloudServiceAccountURI string) (client.ApplicationCredentials, error) {
+	uri, err := url.Parse(cloudServiceAccountURI)
+	if err != nil {
+		return client.ApplicationCredentials{}, err
+	}
+	if uri.Scheme != "serviceaccount" {
+		return client.ApplicationCredentials{}, fmt.Errorf("invalid service account URI: invalid scheme: %s", uri.Scheme)
+	}
+	if uri.Host != "azure" {
+		return client.ApplicationCredentials{}, fmt.Errorf("invalid service account URI: invalid host: %s", uri.Host)
+	}
+	query := uri.Query()
+	return client.ApplicationCredentials{
+		TenantID:     query.Get("tenant_id"),
+		ClientID:     query.Get("client_id"),
+		ClientSecret: query.Get("client_secret"),
+	}, nil
+}

coordinator/cloudprovider/azure/serviceaccounturi_test.go

@@ -0,0 +1,54 @@
+package azure
+
+import (
+	"testing"
+
+	"github.com/edgelesssys/constellation/cli/azure/client"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestGetApplicationCredentials(t *testing.T) {
+	creds := client.ApplicationCredentials{
+		TenantID:     "tenant-id",
+		ClientID:     "client-id",
+		ClientSecret: "client-secret",
+	}
+	testCases := map[string]struct {
+		cloudServiceAccountURI string
+		expectedCreds          client.ApplicationCredentials
+		expectErr              bool
+	}{
+		"getApplicationCredentials works": {
+			cloudServiceAccountURI: "serviceaccount://azure?tenant_id=tenant-id&client_id=client-id&client_secret=client-secret",
+			expectedCreds:          creds,
+		},
+		"invalid URI fails": {
+			cloudServiceAccountURI: "\x00",
+			expectErr:              true,
+		},
+		"incorrect URI scheme fails": {
+			cloudServiceAccountURI: "invalid",
+			expectErr:              true,
+		},
+		"incorrect URI host fails": {
+			cloudServiceAccountURI: "serviceaccount://incorrect",
+			expectErr:              true,
+		},
+	}
+
+	for name, tc := range testCases {
+		t.Run(name, func(t *testing.T) {
+			assert := assert.New(t)
+			require := require.New(t)
+
+			creds, err := getApplicationCredentials(tc.cloudServiceAccountURI)
+			if tc.expectErr {
+				assert.Error(err)
+				return
+			}
+			require.NoError(err)
+			assert.Equal(tc.expectedCreds, creds)
+		})
+	}
+}

coordinator/cloudprovider/images.go

@@ -6,4 +6,8 @@ const (
 	// CloudControllerManagerImageGCP is the CCM image used on GCP.
 	// TODO: use newer "cloud-provider-gcp" from https://github.com/kubernetes/cloud-provider-gcp when newer releases are available.
 	CloudControllerManagerImageGCP = "ghcr.io/malt3/cloud-provider-gcp:latest"
+	// CloudControllerManagerImageAzure is the CCM image used on Azure.
+	CloudControllerManagerImageAzure = "mcr.microsoft.com/oss/kubernetes/azure-cloud-controller-manager:v1.23.5"
+	// CloudNodeManagerImageAzure is the cloud-node-manager image used on Azure.
+	CloudNodeManagerImageAzure = "mcr.microsoft.com/oss/kubernetes/azure-cloud-node-manager:v1.23.5"
 )

coordinator/cmd/coordinator/main.go

@@ -128,8 +128,9 @@ func main() {
 		if err != nil {
 			log.Fatal(err)
 		}
-		cloudControllerManager = azurecloud.CloudControllerManager{}
+		cloudControllerManager = &azurecloud.CloudControllerManager{}
-		autoscaler = azurecloud.Autoscaler{}
+		cloudNodeManager = &azurecloud.CloudNodeManager{}
+		autoscaler = &azurecloud.Autoscaler{}
 		bindIP = defaultIP
 		bindPort = defaultPort
 		etcdEndpoint = defaultEtcdEndpoint