Rename coordinator to bootstrapper and rename roles

This commit is contained in:
katexochen 2022-06-29 15:26:29 +02:00 committed by Paul Meyer
parent 3280ed200c
commit 916e5d6b55
191 changed files with 1763 additions and 2030 deletions

View file

@ -0,0 +1,88 @@
package azure
import (
"context"
"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/applicationinsights/armapplicationinsights"
"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute"
"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork"
"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources"
)
type imdsAPI interface {
Retrieve(ctx context.Context) (metadataResponse, error)
}
type virtualNetworksClientListPager interface {
NextPage(ctx context.Context) bool
PageResponse() armnetwork.VirtualNetworksClientListResponse
}
type virtualNetworksAPI interface {
List(resourceGroupName string, options *armnetwork.VirtualNetworksClientListOptions) virtualNetworksClientListPager
}
type securityGroupsClientListPager interface {
NextPage(ctx context.Context) bool
PageResponse() armnetwork.SecurityGroupsClientListResponse
}
type securityGroupsAPI interface {
List(resourceGroupName string, options *armnetwork.SecurityGroupsClientListOptions) securityGroupsClientListPager
}
type networkInterfacesAPI interface {
GetVirtualMachineScaleSetNetworkInterface(ctx context.Context, resourceGroupName string,
virtualMachineScaleSetName string, virtualmachineIndex string, networkInterfaceName string,
options *armnetwork.InterfacesClientGetVirtualMachineScaleSetNetworkInterfaceOptions,
) (armnetwork.InterfacesClientGetVirtualMachineScaleSetNetworkInterfaceResponse, error)
Get(ctx context.Context, resourceGroupName string, networkInterfaceName string,
options *armnetwork.InterfacesClientGetOptions) (armnetwork.InterfacesClientGetResponse, error)
}
type publicIPAddressesAPI interface {
GetVirtualMachineScaleSetPublicIPAddress(ctx context.Context, resourceGroupName string,
virtualMachineScaleSetName string, virtualmachineIndex string, networkInterfaceName string,
ipConfigurationName string, publicIPAddressName string,
options *armnetwork.PublicIPAddressesClientGetVirtualMachineScaleSetPublicIPAddressOptions,
) (armnetwork.PublicIPAddressesClientGetVirtualMachineScaleSetPublicIPAddressResponse, error)
Get(ctx context.Context, resourceGroupName string, publicIPAddressName string,
options *armnetwork.PublicIPAddressesClientGetOptions) (armnetwork.PublicIPAddressesClientGetResponse, error)
}
type virtualMachineScaleSetVMsAPI interface {
Get(ctx context.Context, resourceGroupName string, vmScaleSetName string, instanceID string, options *armcompute.VirtualMachineScaleSetVMsClientGetOptions) (armcompute.VirtualMachineScaleSetVMsClientGetResponse, error)
List(resourceGroupName string, virtualMachineScaleSetName string, options *armcompute.VirtualMachineScaleSetVMsClientListOptions) virtualMachineScaleSetVMsClientListPager
}
type virtualMachineScaleSetVMsClientListPager interface {
NextPage(ctx context.Context) bool
PageResponse() armcompute.VirtualMachineScaleSetVMsClientListResponse
}
type scaleSetsAPI interface {
List(resourceGroupName string, options *armcompute.VirtualMachineScaleSetsClientListOptions) virtualMachineScaleSetsClientListPager
}
type loadBalancersClientListPager interface {
NextPage(ctx context.Context) bool
PageResponse() armnetwork.LoadBalancersClientListResponse
}
type loadBalancerAPI interface {
List(resourceGroupName string, options *armnetwork.LoadBalancersClientListOptions) loadBalancersClientListPager
}
type virtualMachineScaleSetsClientListPager interface {
NextPage(ctx context.Context) bool
PageResponse() armcompute.VirtualMachineScaleSetsClientListResponse
}
type tagsAPI interface {
CreateOrUpdateAtScope(ctx context.Context, scope string, parameters armresources.TagsResource, options *armresources.TagsClientCreateOrUpdateAtScopeOptions) (armresources.TagsClientCreateOrUpdateAtScopeResponse, error)
UpdateAtScope(ctx context.Context, scope string, parameters armresources.TagsPatchResource, options *armresources.TagsClientUpdateAtScopeOptions) (armresources.TagsClientUpdateAtScopeResponse, error)
}
type applicationInsightsAPI interface {
Get(ctx context.Context, resourceGroupName string, resourceName string, options *armapplicationinsights.ComponentsClientGetOptions) (armapplicationinsights.ComponentsClientGetResponse, error)
}

View file

@ -0,0 +1,260 @@
package azure
import (
"context"
"testing"
"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute"
"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork"
"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources"
"go.uber.org/goleak"
)
func TestMain(m *testing.M) {
goleak.VerifyTestMain(m,
// https://github.com/census-instrumentation/opencensus-go/issues/1262
goleak.IgnoreTopFunction("go.opencensus.io/stats/view.(*worker).start"),
)
}
type stubIMDSAPI struct {
res metadataResponse
retrieveErr error
}
func (a *stubIMDSAPI) Retrieve(ctx context.Context) (metadataResponse, error) {
return a.res, a.retrieveErr
}
type stubNetworkInterfacesAPI struct {
getInterface armnetwork.Interface
getErr error
}
func (a *stubNetworkInterfacesAPI) GetVirtualMachineScaleSetNetworkInterface(ctx context.Context, resourceGroupName string,
virtualMachineScaleSetName string, virtualmachineIndex string, networkInterfaceName string,
options *armnetwork.InterfacesClientGetVirtualMachineScaleSetNetworkInterfaceOptions,
) (armnetwork.InterfacesClientGetVirtualMachineScaleSetNetworkInterfaceResponse, error) {
return armnetwork.InterfacesClientGetVirtualMachineScaleSetNetworkInterfaceResponse{
InterfacesClientGetVirtualMachineScaleSetNetworkInterfaceResult: armnetwork.InterfacesClientGetVirtualMachineScaleSetNetworkInterfaceResult{
Interface: a.getInterface,
},
}, a.getErr
}
func (a *stubNetworkInterfacesAPI) Get(ctx context.Context, resourceGroupName string, networkInterfaceName string,
options *armnetwork.InterfacesClientGetOptions,
) (armnetwork.InterfacesClientGetResponse, error) {
return armnetwork.InterfacesClientGetResponse{
InterfacesClientGetResult: armnetwork.InterfacesClientGetResult{
Interface: a.getInterface,
},
}, a.getErr
}
type stubVirtualMachineScaleSetVMsClientListPager struct {
pagesCounter int
pages [][]*armcompute.VirtualMachineScaleSetVM
}
func (p *stubVirtualMachineScaleSetVMsClientListPager) NextPage(ctx context.Context) bool {
return p.pagesCounter < len(p.pages)
}
func (p *stubVirtualMachineScaleSetVMsClientListPager) PageResponse() armcompute.VirtualMachineScaleSetVMsClientListResponse {
if p.pagesCounter >= len(p.pages) {
return armcompute.VirtualMachineScaleSetVMsClientListResponse{}
}
p.pagesCounter = p.pagesCounter + 1
return armcompute.VirtualMachineScaleSetVMsClientListResponse{
VirtualMachineScaleSetVMsClientListResult: armcompute.VirtualMachineScaleSetVMsClientListResult{
VirtualMachineScaleSetVMListResult: armcompute.VirtualMachineScaleSetVMListResult{
Value: p.pages[p.pagesCounter-1],
},
},
}
}
type stubVirtualMachineScaleSetVMsAPI struct {
getVM armcompute.VirtualMachineScaleSetVM
getErr error
listPages [][]*armcompute.VirtualMachineScaleSetVM
}
func (a *stubVirtualMachineScaleSetVMsAPI) Get(ctx context.Context, resourceGroupName string, vmScaleSetName string, instanceID string, options *armcompute.VirtualMachineScaleSetVMsClientGetOptions) (armcompute.VirtualMachineScaleSetVMsClientGetResponse, error) {
return armcompute.VirtualMachineScaleSetVMsClientGetResponse{
VirtualMachineScaleSetVMsClientGetResult: armcompute.VirtualMachineScaleSetVMsClientGetResult{
VirtualMachineScaleSetVM: a.getVM,
},
}, a.getErr
}
func (a *stubVirtualMachineScaleSetVMsAPI) List(resourceGroupName string, virtualMachineScaleSetName string, options *armcompute.VirtualMachineScaleSetVMsClientListOptions) virtualMachineScaleSetVMsClientListPager {
return &stubVirtualMachineScaleSetVMsClientListPager{
pages: a.listPages,
}
}
type stubVirtualMachineScaleSetsClientListPager struct {
pagesCounter int
pages [][]*armcompute.VirtualMachineScaleSet
}
func (p *stubVirtualMachineScaleSetsClientListPager) NextPage(ctx context.Context) bool {
return p.pagesCounter < len(p.pages)
}
func (p *stubVirtualMachineScaleSetsClientListPager) PageResponse() armcompute.VirtualMachineScaleSetsClientListResponse {
if p.pagesCounter >= len(p.pages) {
return armcompute.VirtualMachineScaleSetsClientListResponse{}
}
p.pagesCounter = p.pagesCounter + 1
return armcompute.VirtualMachineScaleSetsClientListResponse{
VirtualMachineScaleSetsClientListResult: armcompute.VirtualMachineScaleSetsClientListResult{
VirtualMachineScaleSetListResult: armcompute.VirtualMachineScaleSetListResult{
Value: p.pages[p.pagesCounter-1],
},
},
}
}
type stubScaleSetsAPI struct {
listPages [][]*armcompute.VirtualMachineScaleSet
}
func (a *stubScaleSetsAPI) List(resourceGroupName string, options *armcompute.VirtualMachineScaleSetsClientListOptions) virtualMachineScaleSetsClientListPager {
return &stubVirtualMachineScaleSetsClientListPager{
pages: a.listPages,
}
}
type stubTagsAPI struct {
createOrUpdateAtScopeErr error
updateAtScopeErr error
}
func (a *stubTagsAPI) CreateOrUpdateAtScope(ctx context.Context, scope string, parameters armresources.TagsResource, options *armresources.TagsClientCreateOrUpdateAtScopeOptions) (armresources.TagsClientCreateOrUpdateAtScopeResponse, error) {
return armresources.TagsClientCreateOrUpdateAtScopeResponse{}, a.createOrUpdateAtScopeErr
}
func (a *stubTagsAPI) UpdateAtScope(ctx context.Context, scope string, parameters armresources.TagsPatchResource, options *armresources.TagsClientUpdateAtScopeOptions) (armresources.TagsClientUpdateAtScopeResponse, error) {
return armresources.TagsClientUpdateAtScopeResponse{}, a.updateAtScopeErr
}
type stubSecurityGroupsClientListPager struct {
pagesCounter int
pages [][]*armnetwork.SecurityGroup
}
func (p *stubSecurityGroupsClientListPager) NextPage(ctx context.Context) bool {
return p.pagesCounter < len(p.pages)
}
func (p *stubSecurityGroupsClientListPager) PageResponse() armnetwork.SecurityGroupsClientListResponse {
if p.pagesCounter >= len(p.pages) {
return armnetwork.SecurityGroupsClientListResponse{}
}
p.pagesCounter = p.pagesCounter + 1
return armnetwork.SecurityGroupsClientListResponse{
SecurityGroupsClientListResult: armnetwork.SecurityGroupsClientListResult{
SecurityGroupListResult: armnetwork.SecurityGroupListResult{
Value: p.pages[p.pagesCounter-1],
},
},
}
}
type stubSecurityGroupsAPI struct {
listPages [][]*armnetwork.SecurityGroup
}
func (a *stubSecurityGroupsAPI) List(resourceGroupName string, options *armnetwork.SecurityGroupsClientListOptions) securityGroupsClientListPager {
return &stubSecurityGroupsClientListPager{
pages: a.listPages,
}
}
type stubVirtualNetworksClientListPager struct {
pagesCounter int
pages [][]*armnetwork.VirtualNetwork
}
func (p *stubVirtualNetworksClientListPager) NextPage(ctx context.Context) bool {
return p.pagesCounter < len(p.pages)
}
func (p *stubVirtualNetworksClientListPager) PageResponse() armnetwork.VirtualNetworksClientListResponse {
if p.pagesCounter >= len(p.pages) {
return armnetwork.VirtualNetworksClientListResponse{}
}
p.pagesCounter = p.pagesCounter + 1
return armnetwork.VirtualNetworksClientListResponse{
VirtualNetworksClientListResult: armnetwork.VirtualNetworksClientListResult{
VirtualNetworkListResult: armnetwork.VirtualNetworkListResult{
Value: p.pages[p.pagesCounter-1],
},
},
}
}
type stubVirtualNetworksAPI struct {
listPages [][]*armnetwork.VirtualNetwork
}
func (a *stubVirtualNetworksAPI) List(resourceGroupName string, options *armnetwork.VirtualNetworksClientListOptions) virtualNetworksClientListPager {
return &stubVirtualNetworksClientListPager{
pages: a.listPages,
}
}
type stubLoadBalancersClientListPager struct {
pagesCounter int
pages [][]*armnetwork.LoadBalancer
}
func (p *stubLoadBalancersClientListPager) NextPage(ctx context.Context) bool {
return p.pagesCounter < len(p.pages)
}
func (p *stubLoadBalancersClientListPager) PageResponse() armnetwork.LoadBalancersClientListResponse {
if p.pagesCounter >= len(p.pages) {
return armnetwork.LoadBalancersClientListResponse{}
}
p.pagesCounter = p.pagesCounter + 1
return armnetwork.LoadBalancersClientListResponse{
LoadBalancersClientListResult: armnetwork.LoadBalancersClientListResult{
LoadBalancerListResult: armnetwork.LoadBalancerListResult{
Value: p.pages[p.pagesCounter-1],
},
},
}
}
type stubLoadBalancersAPI struct {
listPages [][]*armnetwork.LoadBalancer
}
func (a *stubLoadBalancersAPI) List(resourceGroupName string, options *armnetwork.LoadBalancersClientListOptions) loadBalancersClientListPager {
return &stubLoadBalancersClientListPager{
pages: a.listPages,
}
}
type stubPublicIPAddressesAPI struct {
getResponse armnetwork.PublicIPAddressesClientGetResponse
getVirtualMachineScaleSetPublicIPAddressResponse armnetwork.PublicIPAddressesClientGetVirtualMachineScaleSetPublicIPAddressResponse
getErr error
}
func (a *stubPublicIPAddressesAPI) Get(ctx context.Context, resourceGroupName string, publicIPAddressName string,
options *armnetwork.PublicIPAddressesClientGetOptions,
) (armnetwork.PublicIPAddressesClientGetResponse, error) {
return a.getResponse, a.getErr
}
func (a *stubPublicIPAddressesAPI) GetVirtualMachineScaleSetPublicIPAddress(ctx context.Context, resourceGroupName string, virtualMachineScaleSetName string,
virtualmachineIndex string, networkInterfaceName string, IPConfigurationName string, publicIPAddressName string,
options *armnetwork.PublicIPAddressesClientGetVirtualMachineScaleSetPublicIPAddressOptions,
) (armnetwork.PublicIPAddressesClientGetVirtualMachineScaleSetPublicIPAddressResponse, error) {
return a.getVirtualMachineScaleSetPublicIPAddressResponse, a.getErr
}

View file

@ -0,0 +1,123 @@
package azure
import (
"github.com/edgelesssys/constellation/bootstrapper/internal/kubernetes/k8sapi/resources"
"github.com/edgelesssys/constellation/internal/azureshared"
k8s "k8s.io/api/core/v1"
meta "k8s.io/apimachinery/pkg/apis/meta/v1"
)
// Autoscaler holds the Azure cluster-autoscaler configuration.
type Autoscaler struct{}
// Name returns the cloud-provider name as used by k8s cluster-autoscaler.
func (a *Autoscaler) Name() string {
return "azure"
}
// Secrets returns a list of secrets to deploy together with the k8s cluster-autoscaler.
func (a *Autoscaler) Secrets(providerID string, cloudServiceAccountURI string) (resources.Secrets, error) {
subscriptionID, resourceGroup, err := azureshared.BasicsFromProviderID(providerID)
if err != nil {
return resources.Secrets{}, err
}
creds, err := azureshared.ApplicationCredentialsFromURI(cloudServiceAccountURI)
if err != nil {
return resources.Secrets{}, err
}
return resources.Secrets{
&k8s.Secret{
TypeMeta: meta.TypeMeta{
Kind: "Secret",
APIVersion: "v1",
},
ObjectMeta: meta.ObjectMeta{
Name: "cluster-autoscaler-azure",
Namespace: "kube-system",
},
Data: map[string][]byte{
"ClientID": []byte(creds.ClientID),
"ClientSecret": []byte(creds.ClientSecret),
"ResourceGroup": []byte(resourceGroup),
"SubscriptionID": []byte(subscriptionID),
"TenantID": []byte(creds.TenantID),
"VMType": []byte("vmss"),
},
},
}, nil
}
// Volumes returns a list of volumes to deploy together with the k8s cluster-autoscaler.
func (a *Autoscaler) Volumes() []k8s.Volume {
return []k8s.Volume{}
}
// VolumeMounts returns a list of volume mounts to deploy together with the k8s cluster-autoscaler.
func (a *Autoscaler) VolumeMounts() []k8s.VolumeMount {
return []k8s.VolumeMount{}
}
// Env returns a list of k8s environment key-value pairs to deploy together with the k8s cluster-autoscaler.
func (a *Autoscaler) Env() []k8s.EnvVar {
return []k8s.EnvVar{
{
Name: "ARM_SUBSCRIPTION_ID",
ValueFrom: &k8s.EnvVarSource{
SecretKeyRef: &k8s.SecretKeySelector{
Key: "SubscriptionID",
LocalObjectReference: k8s.LocalObjectReference{Name: "cluster-autoscaler-azure"},
},
},
},
{
Name: "ARM_RESOURCE_GROUP",
ValueFrom: &k8s.EnvVarSource{
SecretKeyRef: &k8s.SecretKeySelector{
Key: "ResourceGroup",
LocalObjectReference: k8s.LocalObjectReference{Name: "cluster-autoscaler-azure"},
},
},
},
{
Name: "ARM_TENANT_ID",
ValueFrom: &k8s.EnvVarSource{
SecretKeyRef: &k8s.SecretKeySelector{
Key: "TenantID",
LocalObjectReference: k8s.LocalObjectReference{Name: "cluster-autoscaler-azure"},
},
},
},
{
Name: "ARM_CLIENT_ID",
ValueFrom: &k8s.EnvVarSource{
SecretKeyRef: &k8s.SecretKeySelector{
Key: "ClientID",
LocalObjectReference: k8s.LocalObjectReference{Name: "cluster-autoscaler-azure"},
},
},
},
{
Name: "ARM_CLIENT_SECRET",
ValueFrom: &k8s.EnvVarSource{
SecretKeyRef: &k8s.SecretKeySelector{
Key: "ClientSecret",
LocalObjectReference: k8s.LocalObjectReference{Name: "cluster-autoscaler-azure"},
},
},
},
{
Name: "ARM_VM_TYPE",
ValueFrom: &k8s.EnvVarSource{
SecretKeyRef: &k8s.SecretKeySelector{
Key: "VMType",
LocalObjectReference: k8s.LocalObjectReference{Name: "cluster-autoscaler-azure"},
},
},
},
}
}
// Supported is used to determine if we support autoscaling for the cloud provider.
func (a *Autoscaler) Supported() bool {
return true
}

View file

@ -0,0 +1,81 @@
package azure
import (
"testing"
"github.com/edgelesssys/constellation/bootstrapper/internal/kubernetes/k8sapi/resources"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
k8s "k8s.io/api/core/v1"
meta "k8s.io/apimachinery/pkg/apis/meta/v1"
)
func TestAutoscalerSecrets(t *testing.T) {
testCases := map[string]struct {
providerID string
cloudServiceAccountURI string
wantSecrets resources.Secrets
wantErr bool
}{
"Secrets works": {
providerID: "azure:///subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/virtualMachines/instance-name",
cloudServiceAccountURI: "serviceaccount://azure?tenant_id=tenant-id&client_id=client-id&client_secret=client-secret",
wantSecrets: resources.Secrets{
&k8s.Secret{
TypeMeta: meta.TypeMeta{
Kind: "Secret",
APIVersion: "v1",
},
ObjectMeta: meta.ObjectMeta{
Name: "cluster-autoscaler-azure",
Namespace: "kube-system",
},
Data: map[string][]byte{
"ClientID": []byte("client-id"),
"ClientSecret": []byte("client-secret"),
"ResourceGroup": []byte("resource-group"),
"SubscriptionID": []byte("subscription-id"),
"TenantID": []byte("tenant-id"),
"VMType": []byte("vmss"),
},
},
},
},
"invalid providerID fails": {
providerID: "invalid",
wantErr: true,
},
"invalid cloudServiceAccountURI fails": {
providerID: "azure:///subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/virtualMachines/instance-name",
cloudServiceAccountURI: "invalid",
wantErr: true,
},
}
for name, tc := range testCases {
t.Run(name, func(t *testing.T) {
assert := assert.New(t)
require := require.New(t)
autoscaler := Autoscaler{}
secrets, err := autoscaler.Secrets(tc.providerID, tc.cloudServiceAccountURI)
if tc.wantErr {
assert.Error(err)
return
}
require.NoError(err)
assert.Equal(tc.wantSecrets, secrets)
})
}
}
func TestTrivialAutoscalerFunctions(t *testing.T) {
assert := assert.New(t)
autoscaler := Autoscaler{}
assert.NotEmpty(autoscaler.Name())
assert.Empty(autoscaler.Volumes())
assert.Empty(autoscaler.VolumeMounts())
assert.NotEmpty(autoscaler.Env())
assert.True(autoscaler.Supported())
}

View file

@ -0,0 +1,183 @@
package azure
import (
"context"
"encoding/json"
"github.com/edgelesssys/constellation/bootstrapper/cloudprovider"
"github.com/edgelesssys/constellation/bootstrapper/internal/kubernetes/k8sapi/resources"
"github.com/edgelesssys/constellation/internal/azureshared"
"github.com/edgelesssys/constellation/internal/cloud/metadata"
k8s "k8s.io/api/core/v1"
meta "k8s.io/apimachinery/pkg/apis/meta/v1"
)
type ccmMetadata interface {
GetNetworkSecurityGroupName(ctx context.Context) (string, error)
GetLoadBalancerName(ctx context.Context) (string, error)
}
// CloudControllerManager holds the Azure cloud-controller-manager configuration.
type CloudControllerManager struct {
metadata ccmMetadata
}
func NewCloudControllerManager(metadata ccmMetadata) *CloudControllerManager {
return &CloudControllerManager{
metadata: metadata,
}
}
// Image returns the container image used to provide cloud-controller-manager for the cloud-provider.
func (c *CloudControllerManager) Image() string {
return cloudprovider.CloudControllerManagerImageAzure
}
// Path returns the path used by cloud-controller-manager executable within the container image.
func (c *CloudControllerManager) Path() string {
return "cloud-controller-manager"
}
// Name returns the cloud-provider name as used by k8s cloud-controller-manager (k8s.gcr.io/cloud-controller-manager).
func (c *CloudControllerManager) Name() string {
return "azure"
}
// ExtraArgs returns a list of arguments to append to the cloud-controller-manager command.
func (c *CloudControllerManager) ExtraArgs() []string {
return []string{
"--controllers=*,-cloud-node",
"--cloud-config=/etc/azure/azure.json",
"--allocate-node-cidrs=false",
"--configure-cloud-routes=true",
}
}
// ConfigMaps returns a list of ConfigMaps to deploy together with the k8s cloud-controller-manager
// Reference: https://kubernetes.io/docs/concepts/configuration/configmap/ .
func (c *CloudControllerManager) ConfigMaps(instance metadata.InstanceMetadata) (resources.ConfigMaps, error) {
return resources.ConfigMaps{}, nil
}
// Secrets returns a list of secrets to deploy together with the k8s cloud-controller-manager.
// Reference: https://kubernetes.io/docs/concepts/configuration/secret/ .
func (c *CloudControllerManager) Secrets(ctx context.Context, providerID string, cloudServiceAccountURI string) (resources.Secrets, error) {
// Azure CCM expects cloud provider config to contain cluster configuration and service principal client secrets
// reference: https://kubernetes-sigs.github.io/cloud-provider-azure/install/configs/
subscriptionID, resourceGroup, err := azureshared.BasicsFromProviderID(providerID)
if err != nil {
return resources.Secrets{}, err
}
creds, err := azureshared.ApplicationCredentialsFromURI(cloudServiceAccountURI)
if err != nil {
return resources.Secrets{}, err
}
vmType := "standard"
if _, _, _, _, err := azureshared.ScaleSetInformationFromProviderID(providerID); err == nil {
vmType = "vmss"
}
securityGroupName, err := c.metadata.GetNetworkSecurityGroupName(ctx)
if err != nil {
return resources.Secrets{}, err
}
loadBalancerName, err := c.metadata.GetLoadBalancerName(ctx)
if err != nil {
return resources.Secrets{}, err
}
config := cloudConfig{
Cloud: "AzurePublicCloud",
TenantID: creds.TenantID,
SubscriptionID: subscriptionID,
ResourceGroup: resourceGroup,
LoadBalancerSku: "standard",
SecurityGroupName: securityGroupName,
LoadBalancerName: loadBalancerName,
UseInstanceMetadata: true,
VmType: vmType,
Location: creds.Location,
AADClientID: creds.ClientID,
AADClientSecret: creds.ClientSecret,
}
rawConfig, err := json.Marshal(config)
if err != nil {
return resources.Secrets{}, err
}
return resources.Secrets{
&k8s.Secret{
TypeMeta: meta.TypeMeta{
Kind: "Secret",
APIVersion: "v1",
},
ObjectMeta: meta.ObjectMeta{
Name: "azureconfig",
Namespace: "kube-system",
},
Data: map[string][]byte{
"azure.json": rawConfig,
},
},
}, nil
}
// Volumes returns a list of volumes to deploy together with the k8s cloud-controller-manager.
// Reference: https://kubernetes.io/docs/concepts/storage/volumes/ .
func (c *CloudControllerManager) Volumes() []k8s.Volume {
return []k8s.Volume{
{
Name: "azureconfig",
VolumeSource: k8s.VolumeSource{
Secret: &k8s.SecretVolumeSource{
SecretName: "azureconfig",
},
},
},
}
}
// VolumeMounts a list of of volume mounts to deploy together with the k8s cloud-controller-manager.
func (c *CloudControllerManager) VolumeMounts() []k8s.VolumeMount {
return []k8s.VolumeMount{
{
Name: "azureconfig",
ReadOnly: true,
MountPath: "/etc/azure",
},
}
}
// Env returns a list of k8s environment key-value pairs to deploy together with the k8s cloud-controller-manager.
func (c *CloudControllerManager) Env() []k8s.EnvVar {
return []k8s.EnvVar{}
}
// Supported is used to determine if cloud controller manager is implemented for this cloud provider.
func (c *CloudControllerManager) Supported() bool {
return true
}
type cloudConfig struct {
Cloud string `json:"cloud,omitempty"`
TenantID string `json:"tenantId,omitempty"`
SubscriptionID string `json:"subscriptionId,omitempty"`
ResourceGroup string `json:"resourceGroup,omitempty"`
Location string `json:"location,omitempty"`
SubnetName string `json:"subnetName,omitempty"`
SecurityGroupName string `json:"securityGroupName,omitempty"`
SecurityGroupResourceGroup string `json:"securityGroupResourceGroup,omitempty"`
LoadBalancerName string `json:"loadBalancerName,omitempty"`
LoadBalancerSku string `json:"loadBalancerSku,omitempty"`
VNetName string `json:"vnetName,omitempty"`
VNetResourceGroup string `json:"vnetResourceGroup,omitempty"`
CloudProviderBackoff bool `json:"cloudProviderBackoff,omitempty"`
UseInstanceMetadata bool `json:"useInstanceMetadata,omitempty"`
VmType string `json:"vmType,omitempty"`
AADClientID string `json:"aadClientId,omitempty"`
AADClientSecret string `json:"aadClientSecret,omitempty"`
}

View file

@ -0,0 +1,136 @@
package azure
import (
"context"
"errors"
"testing"
"github.com/edgelesssys/constellation/bootstrapper/internal/kubernetes/k8sapi/resources"
"github.com/edgelesssys/constellation/internal/cloud/metadata"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
k8s "k8s.io/api/core/v1"
meta "k8s.io/apimachinery/pkg/apis/meta/v1"
)
func TestSecrets(t *testing.T) {
someErr := errors.New("some error")
testCases := map[string]struct {
providerID string
metadata ccmMetadata
cloudServiceAccountURI string
wantSecrets resources.Secrets
wantErr bool
}{
"Secrets works": {
providerID: "azure:///subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/virtualMachines/instance-name",
cloudServiceAccountURI: "serviceaccount://azure?tenant_id=tenant-id&client_id=client-id&client_secret=client-secret&location=location",
metadata: &ccmMetadataStub{loadBalancerName: "load-balancer-name", networkSecurityGroupName: "network-security-group-name"},
wantSecrets: resources.Secrets{
&k8s.Secret{
TypeMeta: meta.TypeMeta{
Kind: "Secret",
APIVersion: "v1",
},
ObjectMeta: meta.ObjectMeta{
Name: "azureconfig",
Namespace: "kube-system",
},
Data: map[string][]byte{
"azure.json": []byte(`{"cloud":"AzurePublicCloud","tenantId":"tenant-id","subscriptionId":"subscription-id","resourceGroup":"resource-group","location":"location","securityGroupName":"network-security-group-name","loadBalancerName":"load-balancer-name","loadBalancerSku":"standard","useInstanceMetadata":true,"vmType":"standard","aadClientId":"client-id","aadClientSecret":"client-secret"}`),
},
},
},
},
"Secrets works for scale sets": {
providerID: "azure:///subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/virtualMachineScaleSets/scale-set-name/virtualMachines/instance-id",
cloudServiceAccountURI: "serviceaccount://azure?tenant_id=tenant-id&client_id=client-id&client_secret=client-secret&location=location",
metadata: &ccmMetadataStub{loadBalancerName: "load-balancer-name", networkSecurityGroupName: "network-security-group-name"},
wantSecrets: resources.Secrets{
&k8s.Secret{
TypeMeta: meta.TypeMeta{
Kind: "Secret",
APIVersion: "v1",
},
ObjectMeta: meta.ObjectMeta{
Name: "azureconfig",
Namespace: "kube-system",
},
Data: map[string][]byte{
"azure.json": []byte(`{"cloud":"AzurePublicCloud","tenantId":"tenant-id","subscriptionId":"subscription-id","resourceGroup":"resource-group","location":"location","securityGroupName":"network-security-group-name","loadBalancerName":"load-balancer-name","loadBalancerSku":"standard","useInstanceMetadata":true,"vmType":"vmss","aadClientId":"client-id","aadClientSecret":"client-secret"}`),
},
},
},
},
"cannot get load balancer Name": {
providerID: "azure:///subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/virtualMachineScaleSets/scale-set-name/virtualMachines/instance-id",
cloudServiceAccountURI: "serviceaccount://azure?tenant_id=tenant-id&client_id=client-id&client_secret=client-secret&location=location",
metadata: &ccmMetadataStub{getLoadBalancerNameErr: someErr},
wantErr: true,
},
"cannot get network security group name": {
providerID: "azure:///subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/virtualMachineScaleSets/scale-set-name/virtualMachines/instance-id",
cloudServiceAccountURI: "serviceaccount://azure?tenant_id=tenant-id&client_id=client-id&client_secret=client-secret&location=location",
metadata: &ccmMetadataStub{getNetworkSecurityGroupNameErr: someErr},
wantErr: true,
},
"invalid providerID fails": {
providerID: "invalid",
metadata: &ccmMetadataStub{},
wantErr: true,
},
"invalid cloudServiceAccountURI fails": {
providerID: "azure:///subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/virtualMachines/instance-name",
metadata: &ccmMetadataStub{},
cloudServiceAccountURI: "invalid",
wantErr: true,
},
}
for name, tc := range testCases {
t.Run(name, func(t *testing.T) {
assert := assert.New(t)
require := require.New(t)
cloud := NewCloudControllerManager(tc.metadata)
secrets, err := cloud.Secrets(context.Background(), tc.providerID, tc.cloudServiceAccountURI)
if tc.wantErr {
assert.Error(err)
return
}
require.NoError(err)
assert.Equal(tc.wantSecrets, secrets)
})
}
}
func TestTrivialCCMFunctions(t *testing.T) {
assert := assert.New(t)
cloud := CloudControllerManager{}
assert.NotEmpty(cloud.Image())
assert.NotEmpty(cloud.Path())
assert.NotEmpty(cloud.Name())
assert.NotEmpty(cloud.ExtraArgs())
assert.Empty(cloud.ConfigMaps(metadata.InstanceMetadata{}))
assert.NotEmpty(cloud.Volumes())
assert.NotEmpty(cloud.VolumeMounts())
assert.Empty(cloud.Env())
assert.True(cloud.Supported())
}
type ccmMetadataStub struct {
networkSecurityGroupName string
loadBalancerName string
getNetworkSecurityGroupNameErr error
getLoadBalancerNameErr error
}
func (c *ccmMetadataStub) GetNetworkSecurityGroupName(ctx context.Context) (string, error) {
return c.networkSecurityGroupName, c.getNetworkSecurityGroupNameErr
}
func (c *ccmMetadataStub) GetLoadBalancerName(ctx context.Context) (string, error) {
return c.loadBalancerName, c.getLoadBalancerNameErr
}

View file

@ -0,0 +1,29 @@
package azure
import "github.com/edgelesssys/constellation/bootstrapper/cloudprovider"
// CloudNodeManager holds the Azure cloud-node-manager configuration.
// reference: https://raw.githubusercontent.com/kubernetes-sigs/cloud-provider-azure/master/examples/out-of-tree/cloud-node-manager.yaml .
type CloudNodeManager struct{}
// Image returns the container image used to provide cloud-node-manager for the cloud-provider.
func (c *CloudNodeManager) Image() string {
return cloudprovider.CloudNodeManagerImageAzure
}
// Path returns the path used by cloud-node-manager executable within the container image.
func (c *CloudNodeManager) Path() string {
return "cloud-node-manager"
}
// ExtraArgs returns a list of arguments to append to the cloud-node-manager command.
func (c *CloudNodeManager) ExtraArgs() []string {
return []string{
"--wait-routes=true",
}
}
// Supported is used to determine if cloud node manager is implemented for this cloud provider.
func (c *CloudNodeManager) Supported() bool {
return true
}

View file

@ -0,0 +1,17 @@
package azure
import (
"testing"
"github.com/stretchr/testify/assert"
)
func TestTrivialCNMFunctions(t *testing.T) {
assert := assert.New(t)
cloud := CloudNodeManager{}
assert.NotEmpty(cloud.Image())
assert.NotEmpty(cloud.Path())
assert.NotEmpty(cloud.ExtraArgs())
assert.True(cloud.Supported())
}

View file

@ -0,0 +1,54 @@
package azure
import (
"context"
"encoding/json"
"io"
"net/http"
)
// subset of azure imds API: https://docs.microsoft.com/en-us/azure/virtual-machines/windows/instance-metadata-service?tabs=linux
// this is not yet available through the azure sdk (see https://github.com/Azure/azure-rest-api-specs/issues/4408)
const (
imdsURL = "http://169.254.169.254/metadata/instance"
imdsAPIVersion = "2021-02-01"
)
type imdsClient struct {
client *http.Client
}
// Retrieve retrieves instance metadata from the azure imds API.
func (c *imdsClient) Retrieve(ctx context.Context) (metadataResponse, error) {
req, err := http.NewRequestWithContext(ctx, "GET", imdsURL, http.NoBody)
if err != nil {
return metadataResponse{}, err
}
req.Header.Add("Metadata", "True")
query := req.URL.Query()
query.Add("format", "json")
query.Add("api-version", imdsAPIVersion)
req.URL.RawQuery = query.Encode()
resp, err := c.client.Do(req)
if err != nil {
return metadataResponse{}, err
}
defer resp.Body.Close()
body, err := io.ReadAll(resp.Body)
if err != nil {
return metadataResponse{}, err
}
var res metadataResponse
if err := json.Unmarshal(body, &res); err != nil {
return metadataResponse{}, err
}
return res, nil
}
// metadataResponse contains metadataResponse with only the required values.
type metadataResponse struct {
Compute struct {
ResourceID string `json:"resourceId,omitempty"`
} `json:"compute,omitempty"`
}

View file

@ -0,0 +1,120 @@
package azure
import (
"context"
"encoding/json"
"fmt"
"net"
"net/http"
"net/http/httptest"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"google.golang.org/grpc/test/bufconn"
)
func TestRetrieve(t *testing.T) {
response := metadataResponse{
Compute: struct {
ResourceID string `json:"resourceId,omitempty"`
}{
ResourceID: "resource-id",
},
}
testCases := map[string]struct {
server httpBufconnServer
wantErr bool
wantResponse metadataResponse
}{
"metadata response parsed": {
server: newHTTPBufconnServerWithMetadataResponse(response),
wantResponse: response,
},
"invalid imds response detected": {
server: newHTTPBufconnServer(func(writer http.ResponseWriter, request *http.Request) {
fmt.Fprintln(writer, "invalid-result")
}),
wantErr: true,
},
}
for name, tc := range testCases {
t.Run(name, func(t *testing.T) {
assert := assert.New(t)
require := require.New(t)
defer tc.server.Close()
hClient := http.Client{
Transport: &http.Transport{
DialContext: tc.server.DialContext,
Dial: tc.server.Dial,
DialTLSContext: tc.server.DialContext,
DialTLS: tc.server.Dial,
},
}
iClient := imdsClient{
client: &hClient,
}
resp, err := iClient.Retrieve(context.Background())
if tc.wantErr {
assert.Error(err)
return
}
require.NoError(err)
assert.Equal(tc.wantResponse, resp)
})
}
}
type httpBufconnServer struct {
*httptest.Server
*bufconn.Listener
}
func (s *httpBufconnServer) DialContext(ctx context.Context, network, addr string) (net.Conn, error) {
return s.Listener.DialContext(ctx)
}
func (s *httpBufconnServer) Dial(network, addr string) (net.Conn, error) {
return s.Listener.Dial()
}
func (s *httpBufconnServer) Close() {
s.Server.Close()
s.Listener.Close()
}
func newHTTPBufconnServer(handlerFunc http.HandlerFunc) httpBufconnServer {
server := httptest.NewUnstartedServer(handlerFunc)
listener := bufconn.Listen(1024)
server.Listener = listener
server.Start()
return httpBufconnServer{
Server: server,
Listener: listener,
}
}
func newHTTPBufconnServerWithMetadataResponse(res metadataResponse) httpBufconnServer {
return newHTTPBufconnServer(func(writer http.ResponseWriter, request *http.Request) {
if request.Host != "169.254.169.254" || request.Header.Get("Metadata") != "True" || request.URL.Query().Get("format") != "json" || request.URL.Query().Get("api-version") != imdsAPIVersion {
writer.WriteHeader(http.StatusInternalServerError)
_, err := writer.Write([]byte("error"))
if err != nil {
panic(err)
}
return
}
rawResp, err := json.Marshal(res)
if err != nil {
panic(err)
}
_, err = writer.Write(rawResp)
if err != nil {
panic(err)
}
})
}

View file

@ -0,0 +1,58 @@
package azure
import (
"context"
"errors"
"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/applicationinsights/armapplicationinsights"
"github.com/edgelesssys/constellation/internal/azureshared"
"github.com/microsoft/ApplicationInsights-Go/appinsights"
)
type Logger struct {
client appinsights.TelemetryClient
}
// NewLogger creates a new client to store information in Azure Application Insights
// https://github.com/Microsoft/ApplicationInsights-go
func NewLogger(ctx context.Context, metadata *Metadata) (*Logger, error) {
providerID, err := metadata.providerID(ctx)
if err != nil {
return nil, err
}
_, resourceGroup, err := azureshared.BasicsFromProviderID(providerID)
if err != nil {
return nil, err
}
uid, err := azureshared.UIDFromProviderID(providerID)
if err != nil {
return nil, err
}
resourceName := "constellation-insights-" + uid
resp, err := metadata.applicationInsightsAPI.Get(ctx, resourceGroup, resourceName, &armapplicationinsights.ComponentsClientGetOptions{})
if err != nil {
return nil, err
}
if resp.Properties == nil || resp.Properties.InstrumentationKey == nil {
return nil, errors.New("unable to get instrumentation key")
}
return &Logger{
client: appinsights.NewTelemetryClient(*resp.Properties.InstrumentationKey),
}, nil
}
// Disclose stores log information in Azure Application Insights!
// Do **NOT** log sensitive information!
func (l *Logger) Disclose(msg string) {
l.client.Track(appinsights.NewTraceTelemetry(msg, appinsights.Information))
}
// Close blocks until all information are written to cloud API.
func (l *Logger) Close() error {
<-l.client.Channel().Close()
return nil
}

View file

@ -0,0 +1,285 @@
package azure
import (
"context"
"fmt"
"net/http"
"regexp"
"github.com/Azure/azure-sdk-for-go/sdk/azidentity"
"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/applicationinsights/armapplicationinsights"
"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute"
"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork"
"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources"
"github.com/edgelesssys/constellation/internal/azureshared"
"github.com/edgelesssys/constellation/internal/cloud/metadata"
)
var (
publicIPAddressRegexp = regexp.MustCompile(`/subscriptions/[^/]+/resourceGroups/[^/]+/providers/Microsoft.Network/publicIPAddresses/(?P<IPname>[^/]+)`)
keyPathRegexp = regexp.MustCompile(`^\/home\/([^\/]+)\/\.ssh\/authorized_keys$`)
)
// Metadata implements azure metadata APIs.
type Metadata struct {
imdsAPI
virtualNetworksAPI
securityGroupsAPI
networkInterfacesAPI
publicIPAddressesAPI
scaleSetsAPI
loadBalancerAPI
virtualMachineScaleSetVMsAPI
tagsAPI
applicationInsightsAPI
}
// NewMetadata creates a new Metadata.
func NewMetadata(ctx context.Context) (*Metadata, error) {
cred, err := azidentity.NewDefaultAzureCredential(nil)
if err != nil {
return nil, err
}
// The default http client may use a system-wide proxy and it is recommended to disable the proxy explicitly:
// https://docs.microsoft.com/en-us/azure/virtual-machines/windows/instance-metadata-service?tabs=linux#proxies
// See also: https://github.com/microsoft/azureimds/blob/master/imdssample.go#L10
imdsAPI := imdsClient{
client: &http.Client{Transport: &http.Transport{Proxy: nil}},
}
instanceMetadata, err := imdsAPI.Retrieve(ctx)
if err != nil {
return nil, err
}
subscriptionID, _, err := azureshared.BasicsFromProviderID("azure://" + instanceMetadata.Compute.ResourceID)
if err != nil {
return nil, err
}
virtualNetworksAPI := armnetwork.NewVirtualNetworksClient(subscriptionID, cred, nil)
networkInterfacesAPI := armnetwork.NewInterfacesClient(subscriptionID, cred, nil)
publicIPAddressesAPI := armnetwork.NewPublicIPAddressesClient(subscriptionID, cred, nil)
securityGroupsAPI := armnetwork.NewSecurityGroupsClient(subscriptionID, cred, nil)
scaleSetsAPI := armcompute.NewVirtualMachineScaleSetsClient(subscriptionID, cred, nil)
loadBalancerAPI := armnetwork.NewLoadBalancersClient(subscriptionID, cred, nil)
virtualMachineScaleSetVMsAPI := armcompute.NewVirtualMachineScaleSetVMsClient(subscriptionID, cred, nil)
tagsAPI := armresources.NewTagsClient(subscriptionID, cred, nil)
applicationInsightsAPI := armapplicationinsights.NewComponentsClient(subscriptionID, cred, nil)
return &Metadata{
imdsAPI: &imdsAPI,
virtualNetworksAPI: &virtualNetworksClient{virtualNetworksAPI},
networkInterfacesAPI: &networkInterfacesClient{networkInterfacesAPI},
securityGroupsAPI: &securityGroupsClient{securityGroupsAPI},
publicIPAddressesAPI: &publicIPAddressesClient{publicIPAddressesAPI},
loadBalancerAPI: &loadBalancersClient{loadBalancerAPI},
scaleSetsAPI: &scaleSetsClient{scaleSetsAPI},
virtualMachineScaleSetVMsAPI: &virtualMachineScaleSetVMsClient{virtualMachineScaleSetVMsAPI},
tagsAPI: &tagsClient{tagsAPI},
applicationInsightsAPI: &applicationInsightsClient{applicationInsightsAPI},
}, nil
}
// List retrieves all instances belonging to the current constellation.
func (m *Metadata) List(ctx context.Context) ([]metadata.InstanceMetadata, error) {
providerID, err := m.providerID(ctx)
if err != nil {
return nil, err
}
_, resourceGroup, err := azureshared.BasicsFromProviderID(providerID)
if err != nil {
return nil, err
}
scaleSetInstances, err := m.listScaleSetVMs(ctx, resourceGroup)
if err != nil {
return nil, err
}
return scaleSetInstances, nil
}
// Self retrieves the current instance.
func (m *Metadata) Self(ctx context.Context) (metadata.InstanceMetadata, error) {
providerID, err := m.providerID(ctx)
if err != nil {
return metadata.InstanceMetadata{}, err
}
return m.GetInstance(ctx, providerID)
}
// GetInstance retrieves an instance using its providerID.
func (m *Metadata) GetInstance(ctx context.Context, providerID string) (metadata.InstanceMetadata, error) {
instance, scaleSetErr := m.getScaleSetVM(ctx, providerID)
if scaleSetErr == nil {
return instance, nil
}
return metadata.InstanceMetadata{}, fmt.Errorf("retrieving instance given providerID %v: %w", providerID, scaleSetErr)
}
// GetNetworkSecurityGroupName returns the security group name of the resource group.
func (m *Metadata) GetNetworkSecurityGroupName(ctx context.Context) (string, error) {
providerID, err := m.providerID(ctx)
if err != nil {
return "", err
}
_, resourceGroup, err := azureshared.BasicsFromProviderID(providerID)
if err != nil {
return "", err
}
nsg, err := m.getNetworkSecurityGroup(ctx, resourceGroup)
if err != nil {
return "", err
}
if nsg == nil || nsg.Name == nil {
return "", fmt.Errorf("could not dereference network security group name")
}
return *nsg.Name, nil
}
// GetSubnetworkCIDR retrieves the subnetwork CIDR from cloud provider metadata.
func (m *Metadata) GetSubnetworkCIDR(ctx context.Context) (string, error) {
providerID, err := m.providerID(ctx)
if err != nil {
return "", err
}
_, resourceGroup, err := azureshared.BasicsFromProviderID(providerID)
if err != nil {
return "", err
}
virtualNetwork, err := m.getVirtualNetwork(ctx, resourceGroup)
if err != nil {
return "", err
}
if virtualNetwork == nil || virtualNetwork.Properties == nil || len(virtualNetwork.Properties.Subnets) == 0 ||
virtualNetwork.Properties.Subnets[0].Properties == nil || virtualNetwork.Properties.Subnets[0].Properties.AddressPrefix == nil {
return "", fmt.Errorf("could not retrieve subnetwork CIDR from virtual network %v", virtualNetwork)
}
return *virtualNetwork.Properties.Subnets[0].Properties.AddressPrefix, nil
}
// getLoadBalancer retrieves the load balancer from cloud provider metadata.
func (m *Metadata) getLoadBalancer(ctx context.Context) (*armnetwork.LoadBalancer, error) {
providerID, err := m.providerID(ctx)
if err != nil {
return nil, err
}
_, resourceGroup, err := azureshared.BasicsFromProviderID(providerID)
if err != nil {
return nil, err
}
pager := m.loadBalancerAPI.List(resourceGroup, nil)
for pager.NextPage(ctx) {
for _, lb := range pager.PageResponse().Value {
if lb != nil && lb.Properties != nil {
return lb, nil
}
}
}
return nil, fmt.Errorf("could not get any load balancer")
}
// SupportsLoadBalancer returns true if the cloud provider supports load balancers.
func (m *Metadata) SupportsLoadBalancer() bool {
return true
}
// GetLoadBalancerName returns the load balancer name of the resource group.
func (m *Metadata) GetLoadBalancerName(ctx context.Context) (string, error) {
lb, err := m.getLoadBalancer(ctx)
if err != nil {
return "", err
}
if lb == nil || lb.Name == nil {
return "", fmt.Errorf("could not dereference load balancer name")
}
return *lb.Name, nil
}
// GetLoadBalancerIP retrieves the first load balancer IP from cloud provider metadata.
func (m *Metadata) GetLoadBalancerIP(ctx context.Context) (string, error) {
lb, err := m.getLoadBalancer(ctx)
if err != nil {
return "", err
}
if lb == nil || lb.Properties == nil {
return "", fmt.Errorf("could not dereference load balancer IP configuration")
}
var pubIPID string
for _, fipConf := range lb.Properties.FrontendIPConfigurations {
if fipConf == nil || fipConf.Properties == nil || fipConf.Properties.PublicIPAddress == nil || fipConf.Properties.PublicIPAddress.ID == nil {
continue
}
pubIPID = *fipConf.Properties.PublicIPAddress.ID
break
}
if pubIPID == "" {
return "", fmt.Errorf("could not find public IP address reference in load balancer")
}
matches := publicIPAddressRegexp.FindStringSubmatch(pubIPID)
if len(matches) != 2 {
return "", fmt.Errorf("could not find public IP address name in load balancer: %v", pubIPID)
}
pubIPName := matches[1]
providerID, err := m.providerID(ctx)
if err != nil {
return "", err
}
_, resourceGroup, err := azureshared.BasicsFromProviderID(providerID)
if err != nil {
return "", err
}
resp, err := m.publicIPAddressesAPI.Get(ctx, resourceGroup, pubIPName, nil)
if err != nil {
return "", fmt.Errorf("could not retrieve public IP address: %w", err)
}
if resp.Properties == nil || resp.Properties.IPAddress == nil {
return "", fmt.Errorf("could not resolve public IP address reference for load balancer")
}
return *resp.Properties.IPAddress, nil
}
// Supported is used to determine if metadata API is implemented for this cloud provider.
func (m *Metadata) Supported() bool {
return true
}
// providerID retrieves the current instances providerID.
func (m *Metadata) providerID(ctx context.Context) (string, error) {
instanceMetadata, err := m.imdsAPI.Retrieve(ctx)
if err != nil {
return "", err
}
return "azure://" + instanceMetadata.Compute.ResourceID, nil
}
// extractInstanceTags converts azure tags into metadata key-value pairs.
func extractInstanceTags(tags map[string]*string) map[string]string {
metadataMap := map[string]string{}
for key, value := range tags {
if value == nil {
continue
}
metadataMap[key] = *value
}
return metadataMap
}
// extractSSHKeys extracts SSH public keys from azure instance OS Profile.
func extractSSHKeys(sshConfig armcompute.SSHConfiguration) map[string][]string {
sshKeys := map[string][]string{}
for _, key := range sshConfig.PublicKeys {
if key == nil || key.Path == nil || key.KeyData == nil {
continue
}
matches := keyPathRegexp.FindStringSubmatch(*key.Path)
if len(matches) != 2 {
continue
}
sshKeys[matches[1]] = append(sshKeys[matches[1]], *key.KeyData)
}
return sshKeys
}

View file

@ -0,0 +1,757 @@
package azure
import (
"context"
"errors"
"testing"
"github.com/Azure/azure-sdk-for-go/sdk/azcore/to"
"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute"
"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork"
"github.com/edgelesssys/constellation/internal/cloud/metadata"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestList(t *testing.T) {
wantInstances := []metadata.InstanceMetadata{
{
Name: "scale-set-name-instance-id",
ProviderID: "azure:///subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/virtualMachineScaleSets/scale-set-name/virtualMachines/instance-id",
PrivateIPs: []string{"192.0.2.0"},
SSHKeys: map[string][]string{"user": {"key-data"}},
},
}
testCases := map[string]struct {
imdsAPI imdsAPI
networkInterfacesAPI networkInterfacesAPI
scaleSetsAPI scaleSetsAPI
virtualMachineScaleSetVMsAPI virtualMachineScaleSetVMsAPI
tagsAPI tagsAPI
wantErr bool
wantInstances []metadata.InstanceMetadata
}{
"List works": {
imdsAPI: newIMDSStub(),
networkInterfacesAPI: newNetworkInterfacesStub(),
scaleSetsAPI: newScaleSetsStub(),
virtualMachineScaleSetVMsAPI: newVirtualMachineScaleSetsVMsStub(),
tagsAPI: newTagsStub(),
wantInstances: wantInstances,
},
"providerID cannot be retrieved": {
imdsAPI: &stubIMDSAPI{retrieveErr: errors.New("imds err")},
wantErr: true,
},
"providerID cannot be parsed": {
imdsAPI: newInvalidIMDSStub(),
wantErr: true,
},
"listScaleSetVMs fails": {
imdsAPI: newIMDSStub(),
networkInterfacesAPI: newNetworkInterfacesStub(),
scaleSetsAPI: newScaleSetsStub(),
virtualMachineScaleSetVMsAPI: newFailingListsVirtualMachineScaleSetsVMsStub(),
tagsAPI: newTagsStub(),
wantErr: true,
},
}
for name, tc := range testCases {
t.Run(name, func(t *testing.T) {
assert := assert.New(t)
require := require.New(t)
azureMetadata := Metadata{
imdsAPI: tc.imdsAPI,
networkInterfacesAPI: tc.networkInterfacesAPI,
scaleSetsAPI: tc.scaleSetsAPI,
virtualMachineScaleSetVMsAPI: tc.virtualMachineScaleSetVMsAPI,
tagsAPI: tc.tagsAPI,
}
instances, err := azureMetadata.List(context.Background())
if tc.wantErr {
assert.Error(err)
return
}
require.NoError(err)
assert.ElementsMatch(tc.wantInstances, instances)
})
}
}
func TestSelf(t *testing.T) {
wantVMInstance := metadata.InstanceMetadata{
Name: "instance-name",
ProviderID: "azure:///subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/virtualMachines/instance-name",
PrivateIPs: []string{"192.0.2.0"},
SSHKeys: map[string][]string{"user": {"key-data"}},
}
wantScaleSetInstance := metadata.InstanceMetadata{
Name: "scale-set-name-instance-id",
ProviderID: "azure:///subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/virtualMachineScaleSets/scale-set-name/virtualMachines/instance-id",
PrivateIPs: []string{"192.0.2.0"},
SSHKeys: map[string][]string{"user": {"key-data"}},
}
testCases := map[string]struct {
imdsAPI imdsAPI
networkInterfacesAPI networkInterfacesAPI
virtualMachineScaleSetVMsAPI virtualMachineScaleSetVMsAPI
wantErr bool
wantInstance metadata.InstanceMetadata
}{
"self for individual instance works": {
imdsAPI: newIMDSStub(),
networkInterfacesAPI: newNetworkInterfacesStub(),
virtualMachineScaleSetVMsAPI: newVirtualMachineScaleSetsVMsStub(),
wantInstance: wantVMInstance,
},
"self for scale set instance works": {
imdsAPI: newScaleSetIMDSStub(),
networkInterfacesAPI: newNetworkInterfacesStub(),
virtualMachineScaleSetVMsAPI: newVirtualMachineScaleSetsVMsStub(),
wantInstance: wantScaleSetInstance,
},
"providerID cannot be retrieved": {
imdsAPI: &stubIMDSAPI{retrieveErr: errors.New("imds err")},
wantErr: true,
},
"GetInstance fails": {
imdsAPI: newIMDSStub(),
wantErr: true,
},
}
for name, tc := range testCases {
t.Run(name, func(t *testing.T) {
assert := assert.New(t)
require := require.New(t)
metadata := Metadata{
imdsAPI: tc.imdsAPI,
networkInterfacesAPI: tc.networkInterfacesAPI,
virtualMachineScaleSetVMsAPI: tc.virtualMachineScaleSetVMsAPI,
}
instance, err := metadata.Self(context.Background())
if tc.wantErr {
assert.Error(err)
return
}
require.NoError(err)
assert.Equal(tc.wantInstance, instance)
})
}
}
func TestGetNetworkSecurityGroupName(t *testing.T) {
name := "network-security-group-name"
testCases := map[string]struct {
securityGroupsAPI securityGroupsAPI
imdsAPI imdsAPI
wantName string
wantErr bool
}{
"GetNetworkSecurityGroupName works": {
imdsAPI: newIMDSStub(),
securityGroupsAPI: &stubSecurityGroupsAPI{
listPages: [][]*armnetwork.SecurityGroup{
{
{
Name: to.StringPtr(name),
},
},
},
},
wantName: name,
},
"no security group": {
imdsAPI: newIMDSStub(),
securityGroupsAPI: &stubSecurityGroupsAPI{},
wantErr: true,
},
"missing name in security group struct": {
imdsAPI: newIMDSStub(),
securityGroupsAPI: &stubSecurityGroupsAPI{listPages: [][]*armnetwork.SecurityGroup{{{}}}},
wantErr: true,
},
}
for name, tc := range testCases {
t.Run(name, func(t *testing.T) {
assert := assert.New(t)
require := require.New(t)
metadata := Metadata{
imdsAPI: tc.imdsAPI,
securityGroupsAPI: tc.securityGroupsAPI,
}
name, err := metadata.GetNetworkSecurityGroupName(context.Background())
if tc.wantErr {
assert.Error(err)
return
}
require.NoError(err)
assert.Equal(tc.wantName, name)
})
}
}
func TestGetSubnetworkCIDR(t *testing.T) {
subnetworkCIDR := "192.0.2.0/24"
name := "name"
testCases := map[string]struct {
virtualNetworksAPI virtualNetworksAPI
imdsAPI imdsAPI
wantNetworkCIDR string
wantErr bool
}{
"GetSubnetworkCIDR works": {
imdsAPI: newIMDSStub(),
virtualNetworksAPI: &stubVirtualNetworksAPI{listPages: [][]*armnetwork.VirtualNetwork{
{
{
Name: to.StringPtr(name),
Properties: &armnetwork.VirtualNetworkPropertiesFormat{
Subnets: []*armnetwork.Subnet{
{Properties: &armnetwork.SubnetPropertiesFormat{AddressPrefix: to.StringPtr(subnetworkCIDR)}},
},
},
},
},
}},
wantNetworkCIDR: subnetworkCIDR,
},
"no virtual networks found": {
imdsAPI: newIMDSStub(),
virtualNetworksAPI: &stubVirtualNetworksAPI{listPages: [][]*armnetwork.VirtualNetwork{
{},
}},
wantErr: true,
wantNetworkCIDR: subnetworkCIDR,
},
"malformed network struct": {
imdsAPI: newIMDSStub(),
virtualNetworksAPI: &stubVirtualNetworksAPI{listPages: [][]*armnetwork.VirtualNetwork{
{
{},
},
}},
wantErr: true,
wantNetworkCIDR: subnetworkCIDR,
},
}
for name, tc := range testCases {
t.Run(name, func(t *testing.T) {
assert := assert.New(t)
require := require.New(t)
metadata := Metadata{
imdsAPI: tc.imdsAPI,
virtualNetworksAPI: tc.virtualNetworksAPI,
}
subnetworkCIDR, err := metadata.GetSubnetworkCIDR(context.Background())
if tc.wantErr {
assert.Error(err)
return
}
require.NoError(err)
assert.Equal(tc.wantNetworkCIDR, subnetworkCIDR)
})
}
}
func TestGetLoadBalancerName(t *testing.T) {
loadBalancerName := "load-balancer-name"
testCases := map[string]struct {
loadBalancerAPI loadBalancerAPI
imdsAPI imdsAPI
wantName string
wantErr bool
}{
"GetLoadBalancerName works": {
imdsAPI: newIMDSStub(),
loadBalancerAPI: &stubLoadBalancersAPI{
listPages: [][]*armnetwork.LoadBalancer{
{
{
Name: to.StringPtr(loadBalancerName),
Properties: &armnetwork.LoadBalancerPropertiesFormat{},
},
},
},
},
wantName: loadBalancerName,
},
"invalid load balancer struct": {
imdsAPI: newIMDSStub(),
loadBalancerAPI: &stubLoadBalancersAPI{listPages: [][]*armnetwork.LoadBalancer{{{}}}},
wantErr: true,
},
"invalid missing name": {
imdsAPI: newIMDSStub(),
loadBalancerAPI: &stubLoadBalancersAPI{listPages: [][]*armnetwork.LoadBalancer{{{
Properties: &armnetwork.LoadBalancerPropertiesFormat{},
}}}},
wantErr: true,
},
}
for name, tc := range testCases {
t.Run(name, func(t *testing.T) {
assert := assert.New(t)
require := require.New(t)
metadata := Metadata{
imdsAPI: tc.imdsAPI,
loadBalancerAPI: tc.loadBalancerAPI,
}
loadbalancerName, err := metadata.GetLoadBalancerName(context.Background())
if tc.wantErr {
assert.Error(err)
return
}
require.NoError(err)
assert.Equal(tc.wantName, loadbalancerName)
})
}
}
func TestGetLoadBalancerIP(t *testing.T) {
loadBalancerName := "load-balancer-name"
publicIP := "192.0.2.1"
correctPublicIPID := "/subscriptions/subscription/resourceGroups/resourceGroup/providers/Microsoft.Network/publicIPAddresses/pubIPName"
someErr := errors.New("some error")
testCases := map[string]struct {
loadBalancerAPI loadBalancerAPI
publicIPAddressesAPI publicIPAddressesAPI
imdsAPI imdsAPI
wantIP string
wantErr bool
}{
"GetLoadBalancerIP works": {
imdsAPI: newIMDSStub(),
loadBalancerAPI: &stubLoadBalancersAPI{
listPages: [][]*armnetwork.LoadBalancer{
{
{
Name: to.StringPtr(loadBalancerName),
Properties: &armnetwork.LoadBalancerPropertiesFormat{
FrontendIPConfigurations: []*armnetwork.FrontendIPConfiguration{
{
Properties: &armnetwork.FrontendIPConfigurationPropertiesFormat{
PublicIPAddress: &armnetwork.PublicIPAddress{
ID: &correctPublicIPID,
},
},
},
},
},
},
},
},
},
publicIPAddressesAPI: &stubPublicIPAddressesAPI{getResponse: armnetwork.PublicIPAddressesClientGetResponse{
PublicIPAddressesClientGetResult: armnetwork.PublicIPAddressesClientGetResult{
PublicIPAddress: armnetwork.PublicIPAddress{
Properties: &armnetwork.PublicIPAddressPropertiesFormat{
IPAddress: &publicIP,
},
},
},
}},
wantIP: publicIP,
},
"no load balancer": {
imdsAPI: newIMDSStub(),
loadBalancerAPI: &stubLoadBalancersAPI{
listPages: [][]*armnetwork.LoadBalancer{
{},
},
},
wantErr: true,
},
"load balancer missing public IP reference": {
imdsAPI: newIMDSStub(),
loadBalancerAPI: &stubLoadBalancersAPI{
listPages: [][]*armnetwork.LoadBalancer{
{
{
Name: to.StringPtr(loadBalancerName),
Properties: &armnetwork.LoadBalancerPropertiesFormat{
FrontendIPConfigurations: []*armnetwork.FrontendIPConfiguration{},
},
},
},
},
},
wantErr: true,
},
"public IP reference has wrong format": {
imdsAPI: newIMDSStub(),
loadBalancerAPI: &stubLoadBalancersAPI{
listPages: [][]*armnetwork.LoadBalancer{
{
{
Name: to.StringPtr(loadBalancerName),
Properties: &armnetwork.LoadBalancerPropertiesFormat{
FrontendIPConfigurations: []*armnetwork.FrontendIPConfiguration{
{
Properties: &armnetwork.FrontendIPConfigurationPropertiesFormat{
PublicIPAddress: &armnetwork.PublicIPAddress{
ID: to.StringPtr("wrong-format"),
},
},
},
},
},
},
},
},
},
wantErr: true,
},
"no public IP address found": {
imdsAPI: newIMDSStub(),
loadBalancerAPI: &stubLoadBalancersAPI{
listPages: [][]*armnetwork.LoadBalancer{
{
{
Name: to.StringPtr(loadBalancerName),
Properties: &armnetwork.LoadBalancerPropertiesFormat{
FrontendIPConfigurations: []*armnetwork.FrontendIPConfiguration{
{
Properties: &armnetwork.FrontendIPConfigurationPropertiesFormat{
PublicIPAddress: &armnetwork.PublicIPAddress{
ID: &correctPublicIPID,
},
},
},
},
},
},
},
},
},
publicIPAddressesAPI: &stubPublicIPAddressesAPI{getErr: someErr},
wantErr: true,
},
"found public IP has no address field": {
imdsAPI: newIMDSStub(),
loadBalancerAPI: &stubLoadBalancersAPI{
listPages: [][]*armnetwork.LoadBalancer{
{
{
Name: to.StringPtr(loadBalancerName),
Properties: &armnetwork.LoadBalancerPropertiesFormat{
FrontendIPConfigurations: []*armnetwork.FrontendIPConfiguration{
{
Properties: &armnetwork.FrontendIPConfigurationPropertiesFormat{
PublicIPAddress: &armnetwork.PublicIPAddress{
ID: &correctPublicIPID,
},
},
},
},
},
},
},
},
},
publicIPAddressesAPI: &stubPublicIPAddressesAPI{getResponse: armnetwork.PublicIPAddressesClientGetResponse{
PublicIPAddressesClientGetResult: armnetwork.PublicIPAddressesClientGetResult{
PublicIPAddress: armnetwork.PublicIPAddress{
Properties: &armnetwork.PublicIPAddressPropertiesFormat{},
},
},
}},
wantErr: true,
},
}
for name, tc := range testCases {
t.Run(name, func(t *testing.T) {
assert := assert.New(t)
require := require.New(t)
metadata := Metadata{
imdsAPI: tc.imdsAPI,
loadBalancerAPI: tc.loadBalancerAPI,
publicIPAddressesAPI: tc.publicIPAddressesAPI,
}
loadbalancerName, err := metadata.GetLoadBalancerIP(context.Background())
if tc.wantErr {
assert.Error(err)
return
}
require.NoError(err)
assert.Equal(tc.wantIP, loadbalancerName)
})
}
}
func TestMetadataSupported(t *testing.T) {
assert := assert.New(t)
metadata := Metadata{}
assert.True(metadata.Supported())
}
func TestProviderID(t *testing.T) {
testCases := map[string]struct {
imdsAPI imdsAPI
wantErr bool
wantProviderID string
}{
"providerID for individual instance works": {
imdsAPI: newIMDSStub(),
wantProviderID: "azure:///subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/virtualMachines/instance-name",
},
"providerID for scale set instance works": {
imdsAPI: newScaleSetIMDSStub(),
wantProviderID: "azure:///subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/virtualMachineScaleSets/scale-set-name/virtualMachines/instance-id",
},
"imds retrieval fails": {
imdsAPI: &stubIMDSAPI{retrieveErr: errors.New("imds err")},
wantErr: true,
},
}
for name, tc := range testCases {
t.Run(name, func(t *testing.T) {
assert := assert.New(t)
require := require.New(t)
metadata := Metadata{
imdsAPI: tc.imdsAPI,
}
providerID, err := metadata.providerID(context.Background())
if tc.wantErr {
assert.Error(err)
return
}
require.NoError(err)
assert.Equal(tc.wantProviderID, providerID)
})
}
}
func TestExtractInstanceTags(t *testing.T) {
testCases := map[string]struct {
in map[string]*string
wantTags map[string]string
}{
"tags are extracted": {
in: map[string]*string{"key": to.StringPtr("value")},
wantTags: map[string]string{"key": "value"},
},
"nil values are skipped": {
in: map[string]*string{"key": nil},
wantTags: map[string]string{},
},
}
for name, tc := range testCases {
t.Run(name, func(t *testing.T) {
assert := assert.New(t)
tags := extractInstanceTags(tc.in)
assert.Equal(tc.wantTags, tags)
})
}
}
func TestExtractSSHKeys(t *testing.T) {
testCases := map[string]struct {
in armcompute.SSHConfiguration
wantKeys map[string][]string
}{
"ssh key is extracted": {
in: armcompute.SSHConfiguration{
PublicKeys: []*armcompute.SSHPublicKey{
{
KeyData: to.StringPtr("key-data"),
Path: to.StringPtr("/home/user/.ssh/authorized_keys"),
},
},
},
wantKeys: map[string][]string{"user": {"key-data"}},
},
"invalid path is skipped": {
in: armcompute.SSHConfiguration{
PublicKeys: []*armcompute.SSHPublicKey{
{
KeyData: to.StringPtr("key-data"),
Path: to.StringPtr("invalid-path"),
},
},
},
wantKeys: map[string][]string{},
},
"key data is nil": {
in: armcompute.SSHConfiguration{
PublicKeys: []*armcompute.SSHPublicKey{
{
Path: to.StringPtr("/home/user/.ssh/authorized_keys"),
},
},
},
wantKeys: map[string][]string{},
},
"path is nil": {
in: armcompute.SSHConfiguration{
PublicKeys: []*armcompute.SSHPublicKey{
{
KeyData: to.StringPtr("key-data"),
},
},
},
wantKeys: map[string][]string{},
},
"public keys are nil": {
in: armcompute.SSHConfiguration{},
wantKeys: map[string][]string{},
},
}
for name, tc := range testCases {
t.Run(name, func(t *testing.T) {
assert := assert.New(t)
keys := extractSSHKeys(tc.in)
assert.Equal(tc.wantKeys, keys)
})
}
}
func newIMDSStub() *stubIMDSAPI {
return &stubIMDSAPI{
res: metadataResponse{Compute: struct {
ResourceID string `json:"resourceId,omitempty"`
}{"/subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/virtualMachines/instance-name"}},
}
}
func newScaleSetIMDSStub() *stubIMDSAPI {
return &stubIMDSAPI{
res: metadataResponse{Compute: struct {
ResourceID string `json:"resourceId,omitempty"`
}{"/subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/virtualMachineScaleSets/scale-set-name/virtualMachines/instance-id"}},
}
}
func newInvalidIMDSStub() *stubIMDSAPI {
return &stubIMDSAPI{
res: metadataResponse{Compute: struct {
ResourceID string `json:"resourceId,omitempty"`
}{"invalid-resource-id"}},
}
}
func newNetworkInterfacesStub() *stubNetworkInterfacesAPI {
return &stubNetworkInterfacesAPI{
getInterface: armnetwork.Interface{
Name: to.StringPtr("interface-name"),
Properties: &armnetwork.InterfacePropertiesFormat{
IPConfigurations: []*armnetwork.InterfaceIPConfiguration{
{
Properties: &armnetwork.InterfaceIPConfigurationPropertiesFormat{
PrivateIPAddress: to.StringPtr("192.0.2.0"),
},
},
},
},
},
}
}
func newScaleSetsStub() *stubScaleSetsAPI {
return &stubScaleSetsAPI{
listPages: [][]*armcompute.VirtualMachineScaleSet{
{
&armcompute.VirtualMachineScaleSet{
Name: to.StringPtr("scale-set-name"),
},
},
},
}
}
func newVirtualMachineScaleSetsVMsStub() *stubVirtualMachineScaleSetVMsAPI {
return &stubVirtualMachineScaleSetVMsAPI{
getVM: armcompute.VirtualMachineScaleSetVM{
Name: to.StringPtr("scale-set-name_instance-id"),
InstanceID: to.StringPtr("instance-id"),
ID: to.StringPtr("/subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/virtualMachineScaleSets/scale-set-name/virtualMachines/instance-id"),
Properties: &armcompute.VirtualMachineScaleSetVMProperties{
NetworkProfile: &armcompute.NetworkProfile{
NetworkInterfaces: []*armcompute.NetworkInterfaceReference{
{
ID: to.StringPtr("/subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/virtualMachineScaleSets/scale-set-name/virtualMachines/instance-id/networkInterfaces/interface-name"),
},
},
},
OSProfile: &armcompute.OSProfile{
ComputerName: to.StringPtr("scale-set-name-instance-id"),
LinuxConfiguration: &armcompute.LinuxConfiguration{
SSH: &armcompute.SSHConfiguration{
PublicKeys: []*armcompute.SSHPublicKey{
{
KeyData: to.StringPtr("key-data"),
Path: to.StringPtr("/home/user/.ssh/authorized_keys"),
},
},
},
},
},
},
},
listPages: [][]*armcompute.VirtualMachineScaleSetVM{
{
&armcompute.VirtualMachineScaleSetVM{
Name: to.StringPtr("scale-set-name_instance-id"),
InstanceID: to.StringPtr("instance-id"),
ID: to.StringPtr("/subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/virtualMachineScaleSets/scale-set-name/virtualMachines/instance-id"),
Properties: &armcompute.VirtualMachineScaleSetVMProperties{
NetworkProfile: &armcompute.NetworkProfile{
NetworkInterfaces: []*armcompute.NetworkInterfaceReference{
{
ID: to.StringPtr("/subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/virtualMachineScaleSets/scale-set-name/virtualMachines/instance-id/networkInterfaces/interface-name"),
},
},
},
OSProfile: &armcompute.OSProfile{
ComputerName: to.StringPtr("scale-set-name-instance-id"),
LinuxConfiguration: &armcompute.LinuxConfiguration{
SSH: &armcompute.SSHConfiguration{
PublicKeys: []*armcompute.SSHPublicKey{
{
KeyData: to.StringPtr("key-data"),
Path: to.StringPtr("/home/user/.ssh/authorized_keys"),
},
},
},
},
},
},
},
},
},
}
}
func newFailingListsVirtualMachineScaleSetsVMsStub() *stubVirtualMachineScaleSetVMsAPI {
return &stubVirtualMachineScaleSetVMsAPI{
listPages: [][]*armcompute.VirtualMachineScaleSetVM{
{
{
InstanceID: to.StringPtr("invalid-instance-id"),
},
},
},
}
}
func newTagsStub() *stubTagsAPI {
return &stubTagsAPI{}
}

View file

@ -0,0 +1,111 @@
package azure
import (
"context"
"errors"
"fmt"
"strings"
"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute"
"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork"
)
// getVMInterfaces retrieves all network interfaces referenced by a virtual machine.
func (m *Metadata) getVMInterfaces(ctx context.Context, vm armcompute.VirtualMachine, resourceGroup string) ([]armnetwork.Interface, error) {
if vm.Properties == nil || vm.Properties.NetworkProfile == nil {
return []armnetwork.Interface{}, nil
}
interfaceNames := extractInterfaceNamesFromInterfaceReferences(vm.Properties.NetworkProfile.NetworkInterfaces)
networkInterfaces := []armnetwork.Interface{}
for _, interfaceName := range interfaceNames {
networkInterfacesResp, err := m.networkInterfacesAPI.Get(ctx, resourceGroup, interfaceName, nil)
if err != nil {
return nil, fmt.Errorf("retrieving network interface %v: %w", interfaceName, err)
}
networkInterfaces = append(networkInterfaces, networkInterfacesResp.Interface)
}
return networkInterfaces, nil
}
// getScaleSetVMInterfaces retrieves all network interfaces referenced by a scale set virtual machine.
func (m *Metadata) getScaleSetVMInterfaces(ctx context.Context, vm armcompute.VirtualMachineScaleSetVM, resourceGroup, scaleSet, instanceID string) ([]armnetwork.Interface, error) {
if vm.Properties == nil || vm.Properties.NetworkProfile == nil {
return []armnetwork.Interface{}, nil
}
interfaceNames := extractInterfaceNamesFromInterfaceReferences(vm.Properties.NetworkProfile.NetworkInterfaces)
networkInterfaces := []armnetwork.Interface{}
for _, interfaceName := range interfaceNames {
networkInterfacesResp, err := m.networkInterfacesAPI.GetVirtualMachineScaleSetNetworkInterface(ctx, resourceGroup, scaleSet, instanceID, interfaceName, nil)
if err != nil {
return nil, fmt.Errorf("retrieving network interface %v: %w", interfaceName, err)
}
networkInterfaces = append(networkInterfaces, networkInterfacesResp.Interface)
}
return networkInterfaces, nil
}
// getScaleSetVMPublicIPAddresses retrieves all public IP addresses from a network interface which is referenced by a scale set virtual machine.
func (m *Metadata) getScaleSetVMPublicIPAddresses(ctx context.Context, resourceGroup, scaleSet, instanceID string,
networkInterfaces []armnetwork.Interface,
) ([]string, error) {
var publicIPAddresses []string
for _, networkInterface := range networkInterfaces {
if networkInterface.Properties == nil || networkInterface.Name == nil {
continue
}
for _, config := range networkInterface.Properties.IPConfigurations {
if config == nil || config.Properties == nil || config.Properties.PublicIPAddress == nil || config.Name == nil {
continue
}
publicIPAddressName := *config.Properties.PublicIPAddress.ID
publicIPAddressNameParts := strings.Split(publicIPAddressName, "/")
publicIPAddressName = publicIPAddressNameParts[len(publicIPAddressNameParts)-1]
publicIPAddress, err := m.publicIPAddressesAPI.GetVirtualMachineScaleSetPublicIPAddress(ctx, resourceGroup, scaleSet, instanceID, *networkInterface.Name, *config.Name, publicIPAddressName, nil)
if err != nil {
return nil, fmt.Errorf("failed to retrieve public ip address %v: %w", publicIPAddressName, err)
}
if publicIPAddress.Properties == nil || publicIPAddress.Properties.IPAddress == nil {
return nil, errors.New("retrieved public ip address has invalid ip address")
}
publicIPAddresses = append(publicIPAddresses, *publicIPAddress.Properties.IPAddress)
}
}
return publicIPAddresses, nil
}
// extractPrivateIPs extracts private IPs from a list of network interface IP configurations.
func extractPrivateIPs(networkInterfaces []armnetwork.Interface) []string {
addresses := []string{}
for _, networkInterface := range networkInterfaces {
if networkInterface.Properties == nil || len(networkInterface.Properties.IPConfigurations) == 0 {
continue
}
for _, config := range networkInterface.Properties.IPConfigurations {
if config == nil || config.Properties == nil || config.Properties.PrivateIPAddress == nil {
continue
}
addresses = append(addresses, *config.Properties.PrivateIPAddress)
}
}
return addresses
}
// extractInterfaceNamesFromInterfaceReferences extracts the name of a network interface from a reference id.
// Format:
// - "/subscriptions/<subscription>/resourceGroups/<resource-group>/providers/Microsoft.Network/networkInterfaces/<interface-name>"
// - "/subscriptions/<subscription>/resourceGroups/<resource-group>/providers/Microsoft.Compute/virtualMachineScaleSets/<scale-set-name>/virtualMachines/<instanceID>/networkInterfaces/<interface-name>".
func extractInterfaceNamesFromInterfaceReferences(references []*armcompute.NetworkInterfaceReference) []string {
interfaceNames := []string{}
for _, interfaceReference := range references {
if interfaceReference == nil || interfaceReference.ID == nil {
continue
}
interfaceIDParts := strings.Split(*interfaceReference.ID, "/")
if len(interfaceIDParts) < 1 {
continue
}
interfaceName := interfaceIDParts[len(interfaceIDParts)-1]
interfaceNames = append(interfaceNames, interfaceName)
}
return interfaceNames
}

View file

@ -0,0 +1,399 @@
package azure
import (
"context"
"errors"
"testing"
"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute"
"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork"
"github.com/Azure/go-autorest/autorest/to"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestGetVMInterfaces(t *testing.T) {
wantNetworkInterfaces := []armnetwork.Interface{
{
Name: to.StringPtr("interface-name"),
Properties: &armnetwork.InterfacePropertiesFormat{
IPConfigurations: []*armnetwork.InterfaceIPConfiguration{
{
Properties: &armnetwork.InterfaceIPConfigurationPropertiesFormat{
PrivateIPAddress: to.StringPtr("192.0.2.0"),
},
},
},
},
},
}
vm := armcompute.VirtualMachine{
Properties: &armcompute.VirtualMachineProperties{
NetworkProfile: &armcompute.NetworkProfile{
NetworkInterfaces: []*armcompute.NetworkInterfaceReference{
{
ID: to.StringPtr("/subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Network/networkInterfaces/interface-name"),
},
},
},
},
}
testCases := map[string]struct {
vm armcompute.VirtualMachine
networkInterfacesAPI networkInterfacesAPI
wantErr bool
wantNetworkInterfaces []armnetwork.Interface
}{
"retrieval works": {
vm: vm,
networkInterfacesAPI: &stubNetworkInterfacesAPI{
getInterface: armnetwork.Interface{
Name: to.StringPtr("interface-name"),
Properties: &armnetwork.InterfacePropertiesFormat{
IPConfigurations: []*armnetwork.InterfaceIPConfiguration{
{
Properties: &armnetwork.InterfaceIPConfigurationPropertiesFormat{
PrivateIPAddress: to.StringPtr("192.0.2.0"),
},
},
},
},
},
},
wantNetworkInterfaces: wantNetworkInterfaces,
},
"vm can have 0 interfaces": {
vm: armcompute.VirtualMachine{},
networkInterfacesAPI: &stubNetworkInterfacesAPI{
getInterface: armnetwork.Interface{
Name: to.StringPtr("interface-name"),
Properties: &armnetwork.InterfacePropertiesFormat{
IPConfigurations: []*armnetwork.InterfaceIPConfiguration{
{
Properties: &armnetwork.InterfaceIPConfigurationPropertiesFormat{
PrivateIPAddress: to.StringPtr("192.0.2.0"),
},
},
},
},
},
},
wantNetworkInterfaces: []armnetwork.Interface{},
},
"interface retrieval fails": {
vm: vm,
networkInterfacesAPI: &stubNetworkInterfacesAPI{
getErr: errors.New("get err"),
},
wantErr: true,
},
}
for name, tc := range testCases {
t.Run(name, func(t *testing.T) {
assert := assert.New(t)
require := require.New(t)
metadata := Metadata{
networkInterfacesAPI: tc.networkInterfacesAPI,
}
vmNetworkInteraces, err := metadata.getVMInterfaces(context.Background(), tc.vm, "resource-group")
if tc.wantErr {
assert.Error(err)
return
}
require.NoError(err)
assert.Equal(tc.wantNetworkInterfaces, vmNetworkInteraces)
})
}
}
func TestGetScaleSetVMInterfaces(t *testing.T) {
wantNetworkInterfaces := []armnetwork.Interface{
{
Name: to.StringPtr("interface-name"),
Properties: &armnetwork.InterfacePropertiesFormat{
IPConfigurations: []*armnetwork.InterfaceIPConfiguration{
{
Properties: &armnetwork.InterfaceIPConfigurationPropertiesFormat{
PrivateIPAddress: to.StringPtr("192.0.2.0"),
},
},
},
},
},
}
vm := armcompute.VirtualMachineScaleSetVM{
Properties: &armcompute.VirtualMachineScaleSetVMProperties{
NetworkProfile: &armcompute.NetworkProfile{
NetworkInterfaces: []*armcompute.NetworkInterfaceReference{
{
ID: to.StringPtr("/subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/virtualMachineScaleSets/scale-set-name/virtualMachines/instance-id/networkInterfaces/interface-name"),
},
},
},
},
}
testCases := map[string]struct {
vm armcompute.VirtualMachineScaleSetVM
networkInterfacesAPI networkInterfacesAPI
wantErr bool
wantNetworkInterfaces []armnetwork.Interface
}{
"retrieval works": {
vm: vm,
networkInterfacesAPI: &stubNetworkInterfacesAPI{
getInterface: armnetwork.Interface{
Name: to.StringPtr("interface-name"),
Properties: &armnetwork.InterfacePropertiesFormat{
IPConfigurations: []*armnetwork.InterfaceIPConfiguration{
{
Properties: &armnetwork.InterfaceIPConfigurationPropertiesFormat{
PrivateIPAddress: to.StringPtr("192.0.2.0"),
},
},
},
},
},
},
wantNetworkInterfaces: wantNetworkInterfaces,
},
"vm can have 0 interfaces": {
vm: armcompute.VirtualMachineScaleSetVM{},
networkInterfacesAPI: &stubNetworkInterfacesAPI{
getInterface: armnetwork.Interface{
Name: to.StringPtr("interface-name"),
Properties: &armnetwork.InterfacePropertiesFormat{
IPConfigurations: []*armnetwork.InterfaceIPConfiguration{
{
Properties: &armnetwork.InterfaceIPConfigurationPropertiesFormat{
PrivateIPAddress: to.StringPtr("192.0.2.0"),
},
},
},
},
},
},
wantNetworkInterfaces: []armnetwork.Interface{},
},
"interface retrieval fails": {
vm: vm,
networkInterfacesAPI: &stubNetworkInterfacesAPI{
getErr: errors.New("get err"),
},
wantErr: true,
},
}
for name, tc := range testCases {
t.Run(name, func(t *testing.T) {
assert := assert.New(t)
require := require.New(t)
metadata := Metadata{
networkInterfacesAPI: tc.networkInterfacesAPI,
}
configs, err := metadata.getScaleSetVMInterfaces(context.Background(), tc.vm, "resource-group", "scale-set-name", "instance-id")
if tc.wantErr {
assert.Error(err)
return
}
require.NoError(err)
assert.Equal(tc.wantNetworkInterfaces, configs)
})
}
}
func TestGetScaleSetVMPublicIPAddresses(t *testing.T) {
someErr := errors.New("some err")
newNetworkInterfaces := func() []armnetwork.Interface {
return []armnetwork.Interface{{
Name: to.StringPtr("interface-name"),
Properties: &armnetwork.InterfacePropertiesFormat{
IPConfigurations: []*armnetwork.InterfaceIPConfiguration{
{
Name: to.StringPtr("ip-config-name"),
Properties: &armnetwork.InterfaceIPConfigurationPropertiesFormat{
PublicIPAddress: &armnetwork.PublicIPAddress{
ID: to.StringPtr("/subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Network/publicIPAddresses/public-ip-name"),
},
},
},
},
},
}, {
Name: to.StringPtr("interface-name2"),
Properties: &armnetwork.InterfacePropertiesFormat{
IPConfigurations: []*armnetwork.InterfaceIPConfiguration{
{
Name: to.StringPtr("ip-config-name2"),
Properties: &armnetwork.InterfaceIPConfigurationPropertiesFormat{
PublicIPAddress: &armnetwork.PublicIPAddress{
ID: to.StringPtr("/subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Network/publicIPAddresses/public-ip-name2"),
},
},
},
},
},
}}
}
testCases := map[string]struct {
networkInterfacesMutator func(*[]armnetwork.Interface)
networkInterfaces []armnetwork.Interface
publicIPAddressesAPI publicIPAddressesAPI
wantIPs []string
wantErr bool
}{
"retrieval works": {
publicIPAddressesAPI: &stubPublicIPAddressesAPI{getVirtualMachineScaleSetPublicIPAddressResponse: armnetwork.PublicIPAddressesClientGetVirtualMachineScaleSetPublicIPAddressResponse{
PublicIPAddressesClientGetVirtualMachineScaleSetPublicIPAddressResult: armnetwork.PublicIPAddressesClientGetVirtualMachineScaleSetPublicIPAddressResult{
PublicIPAddress: armnetwork.PublicIPAddress{
Properties: &armnetwork.PublicIPAddressPropertiesFormat{
IPAddress: to.StringPtr("192.0.2.1"),
},
},
},
}},
networkInterfaces: newNetworkInterfaces(),
wantIPs: []string{"192.0.2.1", "192.0.2.1"},
},
"retrieval works for no valid interfaces": {
publicIPAddressesAPI: &stubPublicIPAddressesAPI{getVirtualMachineScaleSetPublicIPAddressResponse: armnetwork.PublicIPAddressesClientGetVirtualMachineScaleSetPublicIPAddressResponse{
PublicIPAddressesClientGetVirtualMachineScaleSetPublicIPAddressResult: armnetwork.PublicIPAddressesClientGetVirtualMachineScaleSetPublicIPAddressResult{
PublicIPAddress: armnetwork.PublicIPAddress{
Properties: &armnetwork.PublicIPAddressPropertiesFormat{
IPAddress: to.StringPtr("192.0.2.1"),
},
},
},
}},
networkInterfaces: newNetworkInterfaces(),
networkInterfacesMutator: func(nets *[]armnetwork.Interface) {
(*nets)[0].Properties.IPConfigurations = []*armnetwork.InterfaceIPConfiguration{nil}
(*nets)[1] = armnetwork.Interface{Name: nil}
},
},
"fail to get public IP": {
publicIPAddressesAPI: &stubPublicIPAddressesAPI{getErr: someErr},
networkInterfaces: newNetworkInterfaces(),
wantErr: true,
},
"fail to parse IPv4 address of public IP": {
publicIPAddressesAPI: &stubPublicIPAddressesAPI{getVirtualMachineScaleSetPublicIPAddressResponse: armnetwork.PublicIPAddressesClientGetVirtualMachineScaleSetPublicIPAddressResponse{
PublicIPAddressesClientGetVirtualMachineScaleSetPublicIPAddressResult: armnetwork.PublicIPAddressesClientGetVirtualMachineScaleSetPublicIPAddressResult{
PublicIPAddress: armnetwork.PublicIPAddress{},
},
}},
networkInterfaces: newNetworkInterfaces(),
wantErr: true,
},
}
for name, tc := range testCases {
t.Run(name, func(t *testing.T) {
assert := assert.New(t)
require := require.New(t)
if tc.networkInterfacesMutator != nil {
tc.networkInterfacesMutator(&tc.networkInterfaces)
}
metadata := Metadata{
publicIPAddressesAPI: tc.publicIPAddressesAPI,
}
ips, err := metadata.getScaleSetVMPublicIPAddresses(context.Background(), "resource-group", "scale-set-name", "instance-id", tc.networkInterfaces)
if tc.wantErr {
assert.Error(err)
return
}
require.NoError(err)
assert.Equal(tc.wantIPs, ips)
})
}
}
func TestExtractPrivateIPs(t *testing.T) {
testCases := map[string]struct {
networkInterfaces []armnetwork.Interface
wantIPs []string
}{
"extraction works": {
networkInterfaces: []armnetwork.Interface{
{
Properties: &armnetwork.InterfacePropertiesFormat{
IPConfigurations: []*armnetwork.InterfaceIPConfiguration{
{
Properties: &armnetwork.InterfaceIPConfigurationPropertiesFormat{
PrivateIPAddress: to.StringPtr("192.0.2.0"),
},
},
},
},
},
},
wantIPs: []string{"192.0.2.0"},
},
"can be empty": {
networkInterfaces: []armnetwork.Interface{},
},
"invalid interface is skipped": {
networkInterfaces: []armnetwork.Interface{{}},
},
}
for name, tc := range testCases {
t.Run(name, func(t *testing.T) {
assert := assert.New(t)
ips := extractPrivateIPs(tc.networkInterfaces)
assert.ElementsMatch(tc.wantIPs, ips)
})
}
}
func TestExtractInterfaceNamesFromInterfaceReferences(t *testing.T) {
testCases := map[string]struct {
references []*armcompute.NetworkInterfaceReference
wantNames []string
}{
"extraction with individual interface reference works": {
references: []*armcompute.NetworkInterfaceReference{
{
ID: to.StringPtr("/subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Network/networkInterfaces/interface-name"),
},
},
wantNames: []string{"interface-name"},
},
"extraction with scale set interface reference works": {
references: []*armcompute.NetworkInterfaceReference{
{
ID: to.StringPtr("/subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/virtualMachineScaleSets/scale-set-name/virtualMachines/instance-id/networkInterfaces/interface-name"),
},
},
wantNames: []string{"interface-name"},
},
"can be empty": {
references: []*armcompute.NetworkInterfaceReference{},
},
"interface reference containing nil fields is skipped": {
references: []*armcompute.NetworkInterfaceReference{
{},
},
},
}
for name, tc := range testCases {
t.Run(name, func(t *testing.T) {
assert := assert.New(t)
names := extractInterfaceNamesFromInterfaceReferences(tc.references)
assert.ElementsMatch(tc.wantNames, names)
})
}
}

View file

@ -0,0 +1,106 @@
package azure
import (
"context"
"errors"
"regexp"
"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute"
"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork"
"github.com/edgelesssys/constellation/bootstrapper/role"
"github.com/edgelesssys/constellation/internal/azureshared"
"github.com/edgelesssys/constellation/internal/cloud/metadata"
)
var (
controlPlaneScaleSetRegexp = regexp.MustCompile(`constellation-scale-set-controlplanes-[0-9a-zA-Z]+$`)
workerScaleSetRegexp = regexp.MustCompile(`constellation-scale-set-workers-[0-9a-zA-Z]+$`)
)
// getScaleSetVM tries to get an azure vm belonging to a scale set.
func (m *Metadata) getScaleSetVM(ctx context.Context, providerID string) (metadata.InstanceMetadata, error) {
_, resourceGroup, scaleSet, instanceID, err := azureshared.ScaleSetInformationFromProviderID(providerID)
if err != nil {
return metadata.InstanceMetadata{}, err
}
vmResp, err := m.virtualMachineScaleSetVMsAPI.Get(ctx, resourceGroup, scaleSet, instanceID, nil)
if err != nil {
return metadata.InstanceMetadata{}, err
}
networkInterfaces, err := m.getScaleSetVMInterfaces(ctx, vmResp.VirtualMachineScaleSetVM, resourceGroup, scaleSet, instanceID)
if err != nil {
return metadata.InstanceMetadata{}, err
}
publicIPAddresses, err := m.getScaleSetVMPublicIPAddresses(ctx, resourceGroup, scaleSet, instanceID, networkInterfaces)
if err != nil {
return metadata.InstanceMetadata{}, err
}
return convertScaleSetVMToCoreInstance(scaleSet, vmResp.VirtualMachineScaleSetVM, networkInterfaces, publicIPAddresses)
}
// listScaleSetVMs lists all scale set VMs in the current resource group.
func (m *Metadata) listScaleSetVMs(ctx context.Context, resourceGroup string) ([]metadata.InstanceMetadata, error) {
instances := []metadata.InstanceMetadata{}
scaleSetPager := m.scaleSetsAPI.List(resourceGroup, nil)
for scaleSetPager.NextPage(ctx) {
for _, scaleSet := range scaleSetPager.PageResponse().Value {
if scaleSet == nil || scaleSet.Name == nil {
continue
}
vmPager := m.virtualMachineScaleSetVMsAPI.List(resourceGroup, *scaleSet.Name, nil)
for vmPager.NextPage(ctx) {
for _, vm := range vmPager.PageResponse().Value {
if vm == nil || vm.InstanceID == nil {
continue
}
interfaces, err := m.getScaleSetVMInterfaces(ctx, *vm, resourceGroup, *scaleSet.Name, *vm.InstanceID)
if err != nil {
return nil, err
}
instance, err := convertScaleSetVMToCoreInstance(*scaleSet.Name, *vm, interfaces, nil)
if err != nil {
return nil, err
}
instances = append(instances, instance)
}
}
}
}
return instances, nil
}
// convertScaleSetVMToCoreInstance converts an azure scale set virtual machine with interface configurations into a core.Instance.
func convertScaleSetVMToCoreInstance(scaleSet string, vm armcompute.VirtualMachineScaleSetVM, networkInterfaces []armnetwork.Interface, publicIPAddresses []string) (metadata.InstanceMetadata, error) {
if vm.ID == nil {
return metadata.InstanceMetadata{}, errors.New("retrieving instance from armcompute API client returned no instance ID")
}
if vm.Properties == nil || vm.Properties.OSProfile == nil || vm.Properties.OSProfile.ComputerName == nil {
return metadata.InstanceMetadata{}, errors.New("retrieving instance from armcompute API client returned no computer name")
}
var sshKeys map[string][]string
if vm.Properties.OSProfile.LinuxConfiguration == nil || vm.Properties.OSProfile.LinuxConfiguration.SSH == nil {
sshKeys = map[string][]string{}
} else {
sshKeys = extractSSHKeys(*vm.Properties.OSProfile.LinuxConfiguration.SSH)
}
return metadata.InstanceMetadata{
Name: *vm.Properties.OSProfile.ComputerName,
ProviderID: "azure://" + *vm.ID,
Role: extractScaleSetVMRole(scaleSet),
PrivateIPs: extractPrivateIPs(networkInterfaces),
PublicIPs: publicIPAddresses,
SSHKeys: sshKeys,
}, nil
}
// extractScaleSetVMRole extracts the constellation role of a scale set using its name.
func extractScaleSetVMRole(scaleSet string) role.Role {
if controlPlaneScaleSetRegexp.MatchString(scaleSet) {
return role.ControlPlane
}
if workerScaleSetRegexp.MatchString(scaleSet) {
return role.Worker
}
return role.Unknown
}

View file

@ -0,0 +1,335 @@
package azure
import (
"context"
"errors"
"testing"
"github.com/Azure/azure-sdk-for-go/sdk/azcore/to"
"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute"
"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork"
"github.com/edgelesssys/constellation/bootstrapper/role"
"github.com/edgelesssys/constellation/internal/cloud/metadata"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestGetScaleSetVM(t *testing.T) {
wantInstance := metadata.InstanceMetadata{
Name: "scale-set-name-instance-id",
ProviderID: "azure:///subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/virtualMachineScaleSets/scale-set-name/virtualMachines/instance-id",
PrivateIPs: []string{"192.0.2.0"},
SSHKeys: map[string][]string{"user": {"key-data"}},
}
testCases := map[string]struct {
providerID string
networkInterfacesAPI networkInterfacesAPI
virtualMachineScaleSetVMsAPI virtualMachineScaleSetVMsAPI
wantErr bool
wantInstance metadata.InstanceMetadata
}{
"getVM for scale set instance works": {
providerID: "azure:///subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/virtualMachineScaleSets/scale-set-name/virtualMachines/instance-id",
networkInterfacesAPI: newNetworkInterfacesStub(),
virtualMachineScaleSetVMsAPI: newVirtualMachineScaleSetsVMsStub(),
wantInstance: wantInstance,
},
"getVM for individual instance must fail": {
providerID: "azure:///subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/virtualMachines/instance-name",
wantErr: true,
},
"Get fails": {
providerID: "azure:///subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/virtualMachineScaleSets/scale-set-name/virtualMachines/instance-id",
virtualMachineScaleSetVMsAPI: newFailingGetScaleSetVirtualMachinesStub(),
wantErr: true,
},
"conversion fails": {
providerID: "azure:///subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/virtualMachineScaleSets/scale-set-name/virtualMachines/instance-id",
virtualMachineScaleSetVMsAPI: newGetInvalidScaleSetVirtualMachinesStub(),
networkInterfacesAPI: newNetworkInterfacesStub(),
wantErr: true,
},
}
for name, tc := range testCases {
t.Run(name, func(t *testing.T) {
assert := assert.New(t)
require := require.New(t)
metadata := Metadata{
networkInterfacesAPI: tc.networkInterfacesAPI,
virtualMachineScaleSetVMsAPI: tc.virtualMachineScaleSetVMsAPI,
}
instance, err := metadata.getScaleSetVM(context.Background(), tc.providerID)
if tc.wantErr {
assert.Error(err)
return
}
require.NoError(err)
assert.Equal(tc.wantInstance, instance)
})
}
}
func TestListScaleSetVMs(t *testing.T) {
wantInstances := []metadata.InstanceMetadata{
{
Name: "scale-set-name-instance-id",
ProviderID: "azure:///subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/virtualMachineScaleSets/scale-set-name/virtualMachines/instance-id",
PrivateIPs: []string{"192.0.2.0"},
SSHKeys: map[string][]string{"user": {"key-data"}},
},
}
testCases := map[string]struct {
imdsAPI imdsAPI
networkInterfacesAPI networkInterfacesAPI
virtualMachineScaleSetVMsAPI virtualMachineScaleSetVMsAPI
scaleSetsAPI scaleSetsAPI
wantErr bool
wantInstances []metadata.InstanceMetadata
}{
"listVMs works": {
imdsAPI: newIMDSStub(),
networkInterfacesAPI: newNetworkInterfacesStub(),
virtualMachineScaleSetVMsAPI: newVirtualMachineScaleSetsVMsStub(),
scaleSetsAPI: newScaleSetsStub(),
wantInstances: wantInstances,
},
"invalid scale sets are skipped": {
imdsAPI: newIMDSStub(),
networkInterfacesAPI: newNetworkInterfacesStub(),
virtualMachineScaleSetVMsAPI: newVirtualMachineScaleSetsVMsStub(),
scaleSetsAPI: newListContainingNilScaleSetStub(),
wantInstances: wantInstances,
},
"listVMs can return 0 VMs": {
imdsAPI: newIMDSStub(),
networkInterfacesAPI: newNetworkInterfacesStub(),
virtualMachineScaleSetVMsAPI: &stubVirtualMachineScaleSetVMsAPI{},
scaleSetsAPI: newScaleSetsStub(),
wantInstances: []metadata.InstanceMetadata{},
},
"can skip nil in VM list": {
imdsAPI: newIMDSStub(),
networkInterfacesAPI: newNetworkInterfacesStub(),
virtualMachineScaleSetVMsAPI: newListContainingNilScaleSetVirtualMachinesStub(),
scaleSetsAPI: newScaleSetsStub(),
wantInstances: wantInstances,
},
"converting instance fails": {
imdsAPI: newIMDSStub(),
networkInterfacesAPI: newNetworkInterfacesStub(),
virtualMachineScaleSetVMsAPI: newListContainingInvalidScaleSetVirtualMachinesStub(),
scaleSetsAPI: newScaleSetsStub(),
wantErr: true,
},
}
for name, tc := range testCases {
t.Run(name, func(t *testing.T) {
assert := assert.New(t)
require := require.New(t)
metadata := Metadata{
imdsAPI: tc.imdsAPI,
networkInterfacesAPI: tc.networkInterfacesAPI,
virtualMachineScaleSetVMsAPI: tc.virtualMachineScaleSetVMsAPI,
scaleSetsAPI: tc.scaleSetsAPI,
}
instances, err := metadata.listScaleSetVMs(context.Background(), "resource-group")
if tc.wantErr {
assert.Error(err)
return
}
require.NoError(err)
assert.ElementsMatch(tc.wantInstances, instances)
})
}
}
func TestConvertScaleSetVMToCoreInstance(t *testing.T) {
testCases := map[string]struct {
inVM armcompute.VirtualMachineScaleSetVM
inInterface []armnetwork.Interface
inPublicIPs []string
wantErr bool
wantInstance metadata.InstanceMetadata
}{
"conversion works": {
inVM: armcompute.VirtualMachineScaleSetVM{
Name: to.StringPtr("scale-set-name_instance-id"),
ID: to.StringPtr("/subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/virtualMachineScaleSets/scale-set-name/virtualMachines/instance-id"),
Tags: map[string]*string{"tag-key": to.StringPtr("tag-value")},
Properties: &armcompute.VirtualMachineScaleSetVMProperties{
OSProfile: &armcompute.OSProfile{
ComputerName: to.StringPtr("scale-set-name-instance-id"),
},
},
},
inInterface: []armnetwork.Interface{
{
Name: to.StringPtr("scale-set-name_instance-id"),
ID: to.StringPtr("/subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Network/networkInterfaces/interface-name"),
Properties: &armnetwork.InterfacePropertiesFormat{
IPConfigurations: []*armnetwork.InterfaceIPConfiguration{
{
Properties: &armnetwork.InterfaceIPConfigurationPropertiesFormat{
PrivateIPAddress: to.StringPtr("192.0.2.0"),
},
},
},
},
},
},
inPublicIPs: []string{"192.0.2.100", "192.0.2.101"},
wantInstance: metadata.InstanceMetadata{
Name: "scale-set-name-instance-id",
ProviderID: "azure:///subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/virtualMachineScaleSets/scale-set-name/virtualMachines/instance-id",
PrivateIPs: []string{"192.0.2.0"},
PublicIPs: []string{"192.0.2.100", "192.0.2.101"},
SSHKeys: map[string][]string{},
},
},
"invalid instance": {
inVM: armcompute.VirtualMachineScaleSetVM{},
wantErr: true,
},
}
for name, tc := range testCases {
t.Run(name, func(t *testing.T) {
assert := assert.New(t)
require := require.New(t)
instance, err := convertScaleSetVMToCoreInstance("scale-set", tc.inVM, tc.inInterface, tc.inPublicIPs)
if tc.wantErr {
assert.Error(err)
return
}
require.NoError(err)
assert.Equal(tc.wantInstance, instance)
})
}
}
func TestExtractScaleSetVMRole(t *testing.T) {
testCases := map[string]struct {
scaleSet string
wantRole role.Role
}{
"bootstrapper role": {
scaleSet: "constellation-scale-set-bootstrappers-abcd123",
wantRole: role.ControlPlane,
},
"node role": {
scaleSet: "constellation-scale-set-nodes-abcd123",
wantRole: role.Worker,
},
"unknown role": {
scaleSet: "unknown",
wantRole: role.Unknown,
},
}
for name, tc := range testCases {
t.Run(name, func(t *testing.T) {
assert := assert.New(t)
role := extractScaleSetVMRole(tc.scaleSet)
assert.Equal(tc.wantRole, role)
})
}
}
func newFailingGetScaleSetVirtualMachinesStub() *stubVirtualMachineScaleSetVMsAPI {
return &stubVirtualMachineScaleSetVMsAPI{
getErr: errors.New("get err"),
}
}
func newGetInvalidScaleSetVirtualMachinesStub() *stubVirtualMachineScaleSetVMsAPI {
return &stubVirtualMachineScaleSetVMsAPI{
getVM: armcompute.VirtualMachineScaleSetVM{},
}
}
func newListContainingNilScaleSetVirtualMachinesStub() *stubVirtualMachineScaleSetVMsAPI {
return &stubVirtualMachineScaleSetVMsAPI{
listPages: [][]*armcompute.VirtualMachineScaleSetVM{
{
nil,
{
Name: to.StringPtr("scale-set-name_instance-id"),
ID: to.StringPtr("/subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/virtualMachineScaleSets/scale-set-name/virtualMachines/instance-id"),
InstanceID: to.StringPtr("instance-id"),
Tags: map[string]*string{
"tag-key": to.StringPtr("tag-value"),
},
Properties: &armcompute.VirtualMachineScaleSetVMProperties{
NetworkProfile: &armcompute.NetworkProfile{
NetworkInterfaces: []*armcompute.NetworkInterfaceReference{
{
ID: to.StringPtr("/subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/virtualMachineScaleSets/scale-set-name/virtualMachines/instance-id/networkInterfaces/interface-name"),
},
},
},
OSProfile: &armcompute.OSProfile{
ComputerName: to.StringPtr("scale-set-name-instance-id"),
LinuxConfiguration: &armcompute.LinuxConfiguration{
SSH: &armcompute.SSHConfiguration{
PublicKeys: []*armcompute.SSHPublicKey{
{
KeyData: to.StringPtr("key-data"),
Path: to.StringPtr("/home/user/.ssh/authorized_keys"),
},
},
},
},
},
},
},
},
},
}
}
func newListContainingInvalidScaleSetVirtualMachinesStub() *stubVirtualMachineScaleSetVMsAPI {
return &stubVirtualMachineScaleSetVMsAPI{
listPages: [][]*armcompute.VirtualMachineScaleSetVM{
{
{
Name: nil,
ID: nil,
InstanceID: to.StringPtr("instance-id"),
Properties: &armcompute.VirtualMachineScaleSetVMProperties{
OSProfile: &armcompute.OSProfile{
ComputerName: nil,
},
NetworkProfile: &armcompute.NetworkProfile{
NetworkInterfaces: []*armcompute.NetworkInterfaceReference{
{
ID: to.StringPtr("/subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/virtualMachineScaleSets/scale-set-name/virtualMachines/instance-id/networkInterfaces/interface-name"),
},
},
},
},
},
},
},
}
}
func newListContainingNilScaleSetStub() *stubScaleSetsAPI {
return &stubScaleSetsAPI{
listPages: [][]*armcompute.VirtualMachineScaleSet{
{
nil,
{Name: to.StringPtr("scale-set-name")},
},
},
}
}

View file

@ -0,0 +1,19 @@
package azure
import (
"context"
"fmt"
"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork"
)
// getNetworkSecurityGroup retrieves the list of security groups for the given resource group.
func (m *Metadata) getNetworkSecurityGroup(ctx context.Context, resourceGroup string) (*armnetwork.SecurityGroup, error) {
pager := m.securityGroupsAPI.List(resourceGroup, nil)
for pager.NextPage(ctx) {
for _, securityGroup := range pager.PageResponse().Value {
return securityGroup, nil
}
}
return nil, fmt.Errorf("no security group found for resource group %q", resourceGroup)
}

View file

@ -0,0 +1,21 @@
package azure
import (
"context"
"fmt"
"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork"
)
// getVirtualNetwork return the first virtual network found in the resource group.
func (m *Metadata) getVirtualNetwork(ctx context.Context, resourceGroup string) (*armnetwork.VirtualNetwork, error) {
pager := m.virtualNetworksAPI.List(resourceGroup, nil)
for pager.NextPage(ctx) {
for _, network := range pager.PageResponse().Value {
if network != nil {
return network, nil
}
}
}
return nil, fmt.Errorf("no virtual network found in resource group %s", resourceGroup)
}

View file

@ -0,0 +1,109 @@
package azure
import (
"context"
"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/applicationinsights/armapplicationinsights"
"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute"
"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork"
"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources"
)
type virtualNetworksClient struct {
*armnetwork.VirtualNetworksClient
}
func (c *virtualNetworksClient) List(resourceGroupName string, options *armnetwork.VirtualNetworksClientListOptions) virtualNetworksClientListPager {
return c.VirtualNetworksClient.List(resourceGroupName, options)
}
type securityGroupsClient struct {
*armnetwork.SecurityGroupsClient
}
func (c *securityGroupsClient) List(resourceGroupName string, options *armnetwork.SecurityGroupsClientListOptions) securityGroupsClientListPager {
return c.SecurityGroupsClient.List(resourceGroupName, options)
}
type networkInterfacesClient struct {
*armnetwork.InterfacesClient
}
func (c *networkInterfacesClient) GetVirtualMachineScaleSetNetworkInterface(ctx context.Context, resourceGroupName string,
virtualMachineScaleSetName string, virtualmachineIndex string, networkInterfaceName string,
options *armnetwork.InterfacesClientGetVirtualMachineScaleSetNetworkInterfaceOptions,
) (armnetwork.InterfacesClientGetVirtualMachineScaleSetNetworkInterfaceResponse, error) {
return c.InterfacesClient.GetVirtualMachineScaleSetNetworkInterface(ctx, resourceGroupName, virtualMachineScaleSetName, virtualmachineIndex, networkInterfaceName, options)
}
func (c *networkInterfacesClient) Get(ctx context.Context, resourceGroupName string, networkInterfaceName string,
options *armnetwork.InterfacesClientGetOptions,
) (armnetwork.InterfacesClientGetResponse, error) {
return c.InterfacesClient.Get(ctx, resourceGroupName, networkInterfaceName, options)
}
type publicIPAddressesClient struct {
*armnetwork.PublicIPAddressesClient
}
func (c *publicIPAddressesClient) GetVirtualMachineScaleSetPublicIPAddress(ctx context.Context, resourceGroupName string,
virtualMachineScaleSetName string, virtualmachineIndex string, networkInterfaceName string,
ipConfigurationName string, publicIPAddressName string,
options *armnetwork.PublicIPAddressesClientGetVirtualMachineScaleSetPublicIPAddressOptions,
) (armnetwork.PublicIPAddressesClientGetVirtualMachineScaleSetPublicIPAddressResponse, error) {
return c.PublicIPAddressesClient.GetVirtualMachineScaleSetPublicIPAddress(ctx, resourceGroupName, virtualMachineScaleSetName, virtualmachineIndex, networkInterfaceName, ipConfigurationName, publicIPAddressName, options)
}
func (c *publicIPAddressesClient) Get(ctx context.Context, resourceGroupName string, publicIPAddressName string,
options *armnetwork.PublicIPAddressesClientGetOptions,
) (armnetwork.PublicIPAddressesClientGetResponse, error) {
return c.PublicIPAddressesClient.Get(ctx, resourceGroupName, publicIPAddressName, options)
}
type loadBalancersClient struct {
*armnetwork.LoadBalancersClient
}
func (c *loadBalancersClient) List(resourceGroupName string, options *armnetwork.LoadBalancersClientListOptions) loadBalancersClientListPager {
return c.LoadBalancersClient.List(resourceGroupName, options)
}
type virtualMachineScaleSetVMsClient struct {
*armcompute.VirtualMachineScaleSetVMsClient
}
func (c *virtualMachineScaleSetVMsClient) Get(ctx context.Context, resourceGroupName, vmScaleSetName, instanceID string, options *armcompute.VirtualMachineScaleSetVMsClientGetOptions) (armcompute.VirtualMachineScaleSetVMsClientGetResponse, error) {
return c.VirtualMachineScaleSetVMsClient.Get(ctx, resourceGroupName, vmScaleSetName, instanceID, options)
}
func (c *virtualMachineScaleSetVMsClient) List(resourceGroupName, virtualMachineScaleSetName string, options *armcompute.VirtualMachineScaleSetVMsClientListOptions) virtualMachineScaleSetVMsClientListPager {
return c.VirtualMachineScaleSetVMsClient.List(resourceGroupName, virtualMachineScaleSetName, options)
}
type tagsClient struct {
*armresources.TagsClient
}
func (c *tagsClient) CreateOrUpdateAtScope(ctx context.Context, scope string, parameters armresources.TagsResource, options *armresources.TagsClientCreateOrUpdateAtScopeOptions) (armresources.TagsClientCreateOrUpdateAtScopeResponse, error) {
return c.TagsClient.CreateOrUpdateAtScope(ctx, scope, parameters, options)
}
func (c *tagsClient) UpdateAtScope(ctx context.Context, scope string, parameters armresources.TagsPatchResource, options *armresources.TagsClientUpdateAtScopeOptions) (armresources.TagsClientUpdateAtScopeResponse, error) {
return c.TagsClient.UpdateAtScope(ctx, scope, parameters, options)
}
type scaleSetsClient struct {
*armcompute.VirtualMachineScaleSetsClient
}
func (c *scaleSetsClient) List(resourceGroupName string, options *armcompute.VirtualMachineScaleSetsClientListOptions) virtualMachineScaleSetsClientListPager {
return c.VirtualMachineScaleSetsClient.List(resourceGroupName, options)
}
type applicationInsightsClient struct {
*armapplicationinsights.ComponentsClient
}
func (c *applicationInsightsClient) Get(ctx context.Context, resourceGroupName string, resourceName string, options *armapplicationinsights.ComponentsClientGetOptions) (armapplicationinsights.ComponentsClientGetResponse, error) {
return c.ComponentsClient.Get(ctx, resourceGroupName, resourceName, options)
}