Git-Mirrors/constellation
1
0
Fork 0
mirror of https://github.com/edgelesssys/constellation.git synced 2024-12-24 23:19:39 -05:00
Code Issues Packages Projects Releases Wiki Activity

Refactor provider metadata

Browse Source
This commit is contained in:
katexochen 2022-06-28 16:08:05 +02:00 committed by Paul Meyer
parent 32f1f5fd3e
commit 09e86e6c5d
36 changed files with 198 additions and 1340 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

40
coordinator/cloudprovider/aws/autoscaler.go
View File

@ -1,40 +0,0 @@
package aws
import (
"github.com/edgelesssys/constellation/coordinator/cloudprovider/cloudtypes"
"github.com/edgelesssys/constellation/coordinator/kubernetes/k8sapi/resources"
k8s "k8s.io/api/core/v1"
)
// Autoscaler holds the AWS cluster-autoscaler configuration.
type Autoscaler struct{}
// Name returns the cloud-provider name as used by k8s cluster-autoscaler.
func (a Autoscaler) Name() string {
return "aws"
}
// Secrets returns a list of secrets to deploy together with the k8s cluster-autoscaler.
func (a Autoscaler) Secrets(instance cloudtypes.Instance, cloudServiceAccountURI string) (resources.Secrets, error) {
return resources.Secrets{}, nil
}
// Volumes returns a list of volumes to deploy together with the k8s cluster-autoscaler.
func (a Autoscaler) Volumes() []k8s.Volume {
return []k8s.Volume{}
}
// VolumeMounts returns a list of volume mounts to deploy together with the k8s cluster-autoscaler.
func (a Autoscaler) VolumeMounts() []k8s.VolumeMount {
return []k8s.VolumeMount{}
}
// Env returns a list of k8s environment key-value pairs to deploy together with the k8s cluster-autoscaler.
func (a Autoscaler) Env() []k8s.EnvVar {
return []k8s.EnvVar{}
}
// Supported is used to determine if we support autoscaling for the cloud provider.
func (a Autoscaler) Supported() bool {
return false
}

66
coordinator/cloudprovider/aws/ccm.go
View File

@ -1,66 +0,0 @@
package aws
import (
"context"
"github.com/edgelesssys/constellation/coordinator/cloudprovider"
"github.com/edgelesssys/constellation/coordinator/cloudprovider/cloudtypes"
"github.com/edgelesssys/constellation/coordinator/kubernetes/k8sapi/resources"
k8s "k8s.io/api/core/v1"
)
// CloudControllerManager holds the AWS cloud-controller-manager configuration.
type CloudControllerManager struct{}
// Image returns the container image used to provide cloud-controller-manager for the cloud-provider.
func (c CloudControllerManager) Image() string {
return cloudprovider.CloudControllerManagerImageAWS
}
// Path returns the path used by cloud-controller-manager executable within the container image.
func (c CloudControllerManager) Path() string {
return "/aws-cloud-controller-manager"
}
// Name returns the cloud-provider name as used by k8s cloud-controller-manager (k8s.gcr.io/cloud-controller-manager).
func (c CloudControllerManager) Name() string {
return "aws"
}
// ExtraArgs returns a list of arguments to append to the cloud-controller-manager command.
func (c CloudControllerManager) ExtraArgs() []string {
return []string{}
}
// ConfigMaps returns a list of ConfigMaps to deploy together with the k8s cloud-controller-manager
// Reference: https://kubernetes.io/docs/concepts/configuration/configmap/ .
func (c CloudControllerManager) ConfigMaps(instance cloudtypes.Instance) (resources.ConfigMaps, error) {
return resources.ConfigMaps{}, nil
}
// Secrets returns a list of secrets to deploy together with the k8s cloud-controller-manager.
// Reference: https://kubernetes.io/docs/concepts/configuration/secret/ .
func (c CloudControllerManager) Secrets(ctx context.Context, instance cloudtypes.Instance, cloudServiceAccountURI string) (resources.Secrets, error) {
return resources.Secrets{}, nil
}
// Volumes returns a list of volumes to deploy together with the k8s cloud-controller-manager.
// Reference: https://kubernetes.io/docs/concepts/storage/volumes/ .
func (c CloudControllerManager) Volumes() []k8s.Volume {
return []k8s.Volume{}
}
// VolumeMounts a list of of volume mounts to deploy together with the k8s cloud-controller-manager.
func (c CloudControllerManager) VolumeMounts() []k8s.VolumeMount {
return []k8s.VolumeMount{}
}
// Env returns a list of k8s environment key-value pairs to deploy together with the k8s cloud-controller-manager.
func (c CloudControllerManager) Env() []k8s.EnvVar {
return []k8s.EnvVar{}
}
// Supported is used to determine if cloud controller manager is implemented for this cloud provider.
func (c CloudControllerManager) Supported() bool {
return false
}

27
coordinator/cloudprovider/aws/cloudnodemanager.go
View File

@ -1,27 +0,0 @@
package aws
// CloudNodeManager holds the AWS cloud-node-manager configuration.
type CloudNodeManager struct{}
// Image returns the container image used to provide cloud-node-manager for the cloud-provider.
// Not used on AWS.
func (c *CloudNodeManager) Image() string {
return ""
}
// Path returns the path used by cloud-node-manager executable within the container image.
// Not used on AWS.
func (c *CloudNodeManager) Path() string {
return ""
}
// ExtraArgs returns a list of arguments to append to the cloud-node-manager command.
// Not used on AWS.
func (c *CloudNodeManager) ExtraArgs() []string {
return []string{}
}
// Supported is used to determine if cloud node manager is implemented for this cloud provider.
func (c *CloudNodeManager) Supported() bool {
return false
}

79
coordinator/cloudprovider/aws/metadata.go
View File

@ -1,79 +0,0 @@
package aws
import (
"context"
"fmt"
"github.com/aws/aws-sdk-go-v2/config"
"github.com/aws/aws-sdk-go-v2/feature/ec2/imds"
"github.com/edgelesssys/constellation/coordinator/cloudprovider/cloudtypes"
"github.com/edgelesssys/constellation/coordinator/role"
)
// Metadata implements core.ProviderMetadata interface.
type Metadata struct{}
// List retrieves all instances belonging to the current constellation.
func (m Metadata) List(ctx context.Context) ([]cloudtypes.Instance, error) {
// TODO: implement using https://pkg.go.dev/github.com/aws/aws-sdk-go-v2/service/ec2#Client.DescribeInstances
// And using AWS ec2 instance tags
panic("function *Metadata.List not implemented")
}
// Self retrieves the current instance.
func (m Metadata) Self(ctx context.Context) (cloudtypes.Instance, error) {
identityDocument, err := retrieveIdentityDocument(ctx)
if err != nil {
return cloudtypes.Instance{}, err
}
// TODO: implement metadata using AWS ec2 instance tags
return cloudtypes.Instance{
Name: identityDocument.InstanceID,
ProviderID: providerID(identityDocument),
PrivateIPs: []string{
identityDocument.PrivateIP,
},
}, nil
}
// GetInstance retrieves an instance using its providerID.
func (m Metadata) GetInstance(ctx context.Context, providerID string) (cloudtypes.Instance, error) {
// TODO: implement using https://pkg.go.dev/github.com/aws/aws-sdk-go-v2/service/ec2#DescribeInstancesAPIClient.DescribeInstances
// And using AWS ec2 instance tags
// Filter request to only return info on this instance
panic("function *Metadata.GetInstance not implemented")
}
// SignalRole signals the constellation role via cloud provider metadata (if supported by the CSP and deployment type, otherwise does nothing).
func (m Metadata) SignalRole(ctx context.Context, role role.Role) error {
panic("function *Metadata.SignalRole not implemented")
}
// SetVPNIP stores the internally used VPN IP in cloud provider metadata (if supported and required for autoscaling by the CSP, otherwise does nothing).
func (m Metadata) SetVPNIP(ctx context.Context, vpnIP string) error {
panic("function *Metadata.SetVPNIP not implemented")
}
// Supported is used to determine if metadata API is implemented for this cloud provider.
func (m Metadata) Supported() bool {
return false
}
// retrieveIdentityDocument retrieves an AWS instance identity document.
func retrieveIdentityDocument(ctx context.Context) (*imds.GetInstanceIdentityDocumentOutput, error) {
cfg, err := config.LoadDefaultConfig(ctx)
if err != nil {
return nil, fmt.Errorf("loading default AWS configuration: %w", err)
}
client := imds.NewFromConfig(cfg)
identityDocument, err := client.GetInstanceIdentityDocument(ctx, &imds.GetInstanceIdentityDocumentInput{})
if err != nil {
return nil, fmt.Errorf("retrieving AWS instance identity document: %w", err)
}
return identityDocument, nil
}
func providerID(identityDocument *imds.GetInstanceIdentityDocumentOutput) string {
// On AWS, the ProviderID has the form "aws:///<AVAILABILITY_ZONE>/<EC2_INSTANCE_ID>"
return fmt.Sprintf("aws://%v/%v", identityDocument.AvailabilityZone, identityDocument.InstanceID)
}

10
coordinator/cloudprovider/azure/api.go
View File

@ -50,16 +50,6 @@ type publicIPAddressesAPI interface {
options *armnetwork.PublicIPAddressesClientGetOptions) (armnetwork.PublicIPAddressesClientGetResponse, error)
}
type virtualMachinesAPI interface {
Get(ctx context.Context, resourceGroupName string, vmName string, options *armcompute.VirtualMachinesClientGetOptions) (armcompute.VirtualMachinesClientGetResponse, error)
List(resourceGroupName string, options *armcompute.VirtualMachinesClientListOptions) virtualMachinesClientListPager
}
type virtualMachinesClientListPager interface {
NextPage(ctx context.Context) bool
PageResponse() armcompute.VirtualMachinesClientListResponse
}
type virtualMachineScaleSetVMsAPI interface {
Get(ctx context.Context, resourceGroupName string, vmScaleSetName string, instanceID string, options *armcompute.VirtualMachineScaleSetVMsClientGetOptions) (armcompute.VirtualMachineScaleSetVMsClientGetResponse, error)
List(resourceGroupName string, virtualMachineScaleSetName string, options *armcompute.VirtualMachineScaleSetVMsClientListOptions) virtualMachineScaleSetVMsClientListPager

43
coordinator/cloudprovider/azure/api_test.go
View File

@ -52,49 +52,6 @@ func (a *stubNetworkInterfacesAPI) Get(ctx context.Context, resourceGroupName st
}, a.getErr
}
type stubVirtualMachinesClientListPager struct {
pagesCounter int
pages [][]*armcompute.VirtualMachine
}
func (p *stubVirtualMachinesClientListPager) NextPage(ctx context.Context) bool {
return p.pagesCounter < len(p.pages)
}
func (p *stubVirtualMachinesClientListPager) PageResponse() armcompute.VirtualMachinesClientListResponse {
if p.pagesCounter >= len(p.pages) {
return armcompute.VirtualMachinesClientListResponse{}
}
p.pagesCounter = p.pagesCounter + 1
return armcompute.VirtualMachinesClientListResponse{
VirtualMachinesClientListResult: armcompute.VirtualMachinesClientListResult{
VirtualMachineListResult: armcompute.VirtualMachineListResult{
Value: p.pages[p.pagesCounter-1],
},
},
}
}
type stubVirtualMachinesAPI struct {
getVM armcompute.VirtualMachine
getErr error
listPages [][]*armcompute.VirtualMachine
}
func (a *stubVirtualMachinesAPI) Get(ctx context.Context, resourceGroupName string, vmName string, options *armcompute.VirtualMachinesClientGetOptions) (armcompute.VirtualMachinesClientGetResponse, error) {
return armcompute.VirtualMachinesClientGetResponse{
VirtualMachinesClientGetResult: armcompute.VirtualMachinesClientGetResult{
VirtualMachine: a.getVM,
},
}, a.getErr
}
func (a *stubVirtualMachinesAPI) List(resourceGroupName string, options *armcompute.VirtualMachinesClientListOptions) virtualMachinesClientListPager {
return &stubVirtualMachinesClientListPager{
pages: a.listPages,
}
}
type stubVirtualMachineScaleSetVMsClientListPager struct {
pagesCounter int
pages [][]*armcompute.VirtualMachineScaleSetVM

5
coordinator/cloudprovider/azure/autoscaler.go
View File

@ -1,7 +1,6 @@
package azure
import (
"github.com/edgelesssys/constellation/coordinator/cloudprovider/cloudtypes"
"github.com/edgelesssys/constellation/coordinator/kubernetes/k8sapi/resources"
"github.com/edgelesssys/constellation/internal/azureshared"
k8s "k8s.io/api/core/v1"
@ -17,8 +16,8 @@ func (a *Autoscaler) Name() string {
}
// Secrets returns a list of secrets to deploy together with the k8s cluster-autoscaler.
func (a *Autoscaler) Secrets(instance cloudtypes.Instance, cloudServiceAccountURI string) (resources.Secrets, error) {
subscriptionID, resourceGroup, err := azureshared.BasicsFromProviderID(instance.ProviderID)
func (a *Autoscaler) Secrets(providerID string, cloudServiceAccountURI string) (resources.Secrets, error) {
subscriptionID, resourceGroup, err := azureshared.BasicsFromProviderID(providerID)
if err != nil {
return resources.Secrets{}, err
}

11
coordinator/cloudprovider/azure/autoscaler_test.go
View File

@ -3,7 +3,6 @@ package azure
import (
"testing"
"github.com/edgelesssys/constellation/coordinator/cloudprovider/cloudtypes"
"github.com/edgelesssys/constellation/coordinator/kubernetes/k8sapi/resources"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
@ -13,13 +12,13 @@ import (
func TestAutoscalerSecrets(t *testing.T) {
testCases := map[string]struct {
instance cloudtypes.Instance
providerID string
cloudServiceAccountURI string
wantSecrets resources.Secrets
wantErr bool
}{
"Secrets works": {
instance: cloudtypes.Instance{ProviderID: "azure:///subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/virtualMachines/instance-name"},
providerID: "azure:///subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/virtualMachines/instance-name",
cloudServiceAccountURI: "serviceaccount://azure?tenant_id=tenant-id&client_id=client-id&client_secret=client-secret",
wantSecrets: resources.Secrets{
&k8s.Secret{
@ -43,11 +42,11 @@ func TestAutoscalerSecrets(t *testing.T) {
},
},
"invalid providerID fails": {
instance: cloudtypes.Instance{ProviderID: "invalid"},
providerID: "invalid",
wantErr: true,
},
"invalid cloudServiceAccountURI fails": {
instance: cloudtypes.Instance{ProviderID: "azure:///subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/virtualMachines/instance-name"},
providerID: "azure:///subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/virtualMachines/instance-name",
cloudServiceAccountURI: "invalid",
wantErr: true,
},
@ -59,7 +58,7 @@ func TestAutoscalerSecrets(t *testing.T) {
require := require.New(t)
autoscaler := Autoscaler{}
secrets, err := autoscaler.Secrets(tc.instance, tc.cloudServiceAccountURI)
secrets, err := autoscaler.Secrets(tc.providerID, tc.cloudServiceAccountURI)
if tc.wantErr {
assert.Error(err)
return

10
coordinator/cloudprovider/azure/ccm.go
View File

@ -5,9 +5,9 @@ import (
"encoding/json"
"github.com/edgelesssys/constellation/coordinator/cloudprovider"
"github.com/edgelesssys/constellation/coordinator/cloudprovider/cloudtypes"
"github.com/edgelesssys/constellation/coordinator/kubernetes/k8sapi/resources"
"github.com/edgelesssys/constellation/internal/azureshared"
"github.com/edgelesssys/constellation/internal/cloud/metadata"
k8s "k8s.io/api/core/v1"
meta "k8s.io/apimachinery/pkg/apis/meta/v1"
)
@ -55,17 +55,17 @@ func (c *CloudControllerManager) ExtraArgs() []string {
// ConfigMaps returns a list of ConfigMaps to deploy together with the k8s cloud-controller-manager
// Reference: https://kubernetes.io/docs/concepts/configuration/configmap/ .
func (c *CloudControllerManager) ConfigMaps(instance cloudtypes.Instance) (resources.ConfigMaps, error) {
func (c *CloudControllerManager) ConfigMaps(instance metadata.InstanceMetadata) (resources.ConfigMaps, error) {
return resources.ConfigMaps{}, nil
}
// Secrets returns a list of secrets to deploy together with the k8s cloud-controller-manager.
// Reference: https://kubernetes.io/docs/concepts/configuration/secret/ .
func (c *CloudControllerManager) Secrets(ctx context.Context, instance cloudtypes.Instance, cloudServiceAccountURI string) (resources.Secrets, error) {
func (c *CloudControllerManager) Secrets(ctx context.Context, providerID string, cloudServiceAccountURI string) (resources.Secrets, error) {
// Azure CCM expects cloud provider config to contain cluster configuration and service principal client secrets
// reference: https://kubernetes-sigs.github.io/cloud-provider-azure/install/configs/
subscriptionID, resourceGroup, err := azureshared.BasicsFromProviderID(instance.ProviderID)
subscriptionID, resourceGroup, err := azureshared.BasicsFromProviderID(providerID)
if err != nil {
return resources.Secrets{}, err
}
@ -75,7 +75,7 @@ func (c *CloudControllerManager) Secrets(ctx context.Context, instance cloudtype
}
vmType := "standard"
if _, _, _, _, err := azureshared.ScaleSetInformationFromProviderID(instance.ProviderID); err == nil {
if _, _, _, _, err := azureshared.ScaleSetInformationFromProviderID(providerID); err == nil {
vmType = "vmss"
}

20
coordinator/cloudprovider/azure/ccm_test.go
View File

@ -5,8 +5,8 @@ import (
"errors"
"testing"
"github.com/edgelesssys/constellation/coordinator/cloudprovider/cloudtypes"
"github.com/edgelesssys/constellation/coordinator/kubernetes/k8sapi/resources"
"github.com/edgelesssys/constellation/internal/cloud/metadata"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
k8s "k8s.io/api/core/v1"
@ -16,14 +16,14 @@ import (
func TestSecrets(t *testing.T) {
someErr := errors.New("some error")
testCases := map[string]struct {
instance cloudtypes.Instance
providerID string
metadata ccmMetadata
cloudServiceAccountURI string
wantSecrets resources.Secrets
wantErr bool
}{
"Secrets works": {
instance: cloudtypes.Instance{ProviderID: "azure:///subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/virtualMachines/instance-name"},
providerID: "azure:///subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/virtualMachines/instance-name",
cloudServiceAccountURI: "serviceaccount://azure?tenant_id=tenant-id&client_id=client-id&client_secret=client-secret&location=location",
metadata: &ccmMetadataStub{loadBalancerName: "load-balancer-name", networkSecurityGroupName: "network-security-group-name"},
wantSecrets: resources.Secrets{
@ -43,7 +43,7 @@ func TestSecrets(t *testing.T) {
},
},
"Secrets works for scale sets": {
instance: cloudtypes.Instance{ProviderID: "azure:///subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/virtualMachineScaleSets/scale-set-name/virtualMachines/instance-id"},
providerID: "azure:///subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/virtualMachineScaleSets/scale-set-name/virtualMachines/instance-id",
cloudServiceAccountURI: "serviceaccount://azure?tenant_id=tenant-id&client_id=client-id&client_secret=client-secret&location=location",
metadata: &ccmMetadataStub{loadBalancerName: "load-balancer-name", networkSecurityGroupName: "network-security-group-name"},
wantSecrets: resources.Secrets{
@ -63,24 +63,24 @@ func TestSecrets(t *testing.T) {
},
},
"cannot get load balancer Name": {
instance: cloudtypes.Instance{ProviderID: "azure:///subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/virtualMachineScaleSets/scale-set-name/virtualMachines/instance-id"},
providerID: "azure:///subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/virtualMachineScaleSets/scale-set-name/virtualMachines/instance-id",
cloudServiceAccountURI: "serviceaccount://azure?tenant_id=tenant-id&client_id=client-id&client_secret=client-secret&location=location",
metadata: &ccmMetadataStub{getLoadBalancerNameErr: someErr},
wantErr: true,
},
"cannot get network security group name": {
instance: cloudtypes.Instance{ProviderID: "azure:///subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/virtualMachineScaleSets/scale-set-name/virtualMachines/instance-id"},
providerID: "azure:///subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/virtualMachineScaleSets/scale-set-name/virtualMachines/instance-id",
cloudServiceAccountURI: "serviceaccount://azure?tenant_id=tenant-id&client_id=client-id&client_secret=client-secret&location=location",
metadata: &ccmMetadataStub{getNetworkSecurityGroupNameErr: someErr},
wantErr: true,
},
"invalid providerID fails": {
instance: cloudtypes.Instance{ProviderID: "invalid"},
providerID: "invalid",
metadata: &ccmMetadataStub{},
wantErr: true,
},
"invalid cloudServiceAccountURI fails": {
instance: cloudtypes.Instance{ProviderID: "azure:///subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/virtualMachines/instance-name"},
providerID: "azure:///subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/virtualMachines/instance-name",
metadata: &ccmMetadataStub{},
cloudServiceAccountURI: "invalid",
wantErr: true,
@ -93,7 +93,7 @@ func TestSecrets(t *testing.T) {
require := require.New(t)
cloud := NewCloudControllerManager(tc.metadata)
secrets, err := cloud.Secrets(context.Background(), tc.instance, tc.cloudServiceAccountURI)
secrets, err := cloud.Secrets(context.Background(), tc.providerID, tc.cloudServiceAccountURI)
if tc.wantErr {
assert.Error(err)
return
@ -112,7 +112,7 @@ func TestTrivialCCMFunctions(t *testing.T) {
assert.NotEmpty(cloud.Path())
assert.NotEmpty(cloud.Name())
assert.NotEmpty(cloud.ExtraArgs())
assert.Empty(cloud.ConfigMaps(cloudtypes.Instance{}))
assert.Empty(cloud.ConfigMaps(metadata.InstanceMetadata{}))
assert.NotEmpty(cloud.Volumes())
assert.NotEmpty(cloud.VolumeMounts())
assert.Empty(cloud.Env())

44
coordinator/cloudprovider/azure/metadata.go
View File

@ -11,10 +11,8 @@ import (
"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute"
"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork"
"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources"
"github.com/edgelesssys/constellation/coordinator/cloudprovider/cloudtypes"
"github.com/edgelesssys/constellation/coordinator/core"
"github.com/edgelesssys/constellation/coordinator/role"
"github.com/edgelesssys/constellation/internal/azureshared"
"github.com/edgelesssys/constellation/internal/cloud/metadata"
)
var (
@ -31,7 +29,6 @@ type Metadata struct {
publicIPAddressesAPI
scaleSetsAPI
loadBalancerAPI
virtualMachinesAPI
virtualMachineScaleSetVMsAPI
tagsAPI
applicationInsightsAPI
@ -63,7 +60,6 @@ func NewMetadata(ctx context.Context) (*Metadata, error) {
securityGroupsAPI := armnetwork.NewSecurityGroupsClient(subscriptionID, cred, nil)
scaleSetsAPI := armcompute.NewVirtualMachineScaleSetsClient(subscriptionID, cred, nil)
loadBalancerAPI := armnetwork.NewLoadBalancersClient(subscriptionID, cred, nil)
virtualMachinesAPI := armcompute.NewVirtualMachinesClient(subscriptionID, cred, nil)
virtualMachineScaleSetVMsAPI := armcompute.NewVirtualMachineScaleSetVMsClient(subscriptionID, cred, nil)
tagsAPI := armresources.NewTagsClient(subscriptionID, cred, nil)
applicationInsightsAPI := armapplicationinsights.NewComponentsClient(subscriptionID, cred, nil)
@ -76,7 +72,6 @@ func NewMetadata(ctx context.Context) (*Metadata, error) {
publicIPAddressesAPI: &publicIPAddressesClient{publicIPAddressesAPI},
loadBalancerAPI: &loadBalancersClient{loadBalancerAPI},
scaleSetsAPI: &scaleSetsClient{scaleSetsAPI},
virtualMachinesAPI: &virtualMachinesClient{virtualMachinesAPI},
virtualMachineScaleSetVMsAPI: &virtualMachineScaleSetVMsClient{virtualMachineScaleSetVMsAPI},
tagsAPI: &tagsClient{tagsAPI},
applicationInsightsAPI: &applicationInsightsClient{applicationInsightsAPI},
@ -84,7 +79,7 @@ func NewMetadata(ctx context.Context) (*Metadata, error) {
}
// List retrieves all instances belonging to the current constellation.
func (m *Metadata) List(ctx context.Context) ([]cloudtypes.Instance, error) {
func (m *Metadata) List(ctx context.Context) ([]metadata.InstanceMetadata, error) {
providerID, err := m.providerID(ctx)
if err != nil {
return nil, err
@ -93,54 +88,29 @@ func (m *Metadata) List(ctx context.Context) ([]cloudtypes.Instance, error) {
if err != nil {
return nil, err
}
singleInstances, err := m.listVMs(ctx, resourceGroup)
if err != nil {
return nil, err
}
scaleSetInstances, err := m.listScaleSetVMs(ctx, resourceGroup)
if err != nil {
return nil, err
}
instances := make([]cloudtypes.Instance, 0, len(singleInstances)+len(scaleSetInstances))
instances = append(instances, singleInstances...)
instances = append(instances, scaleSetInstances...)
return instances, nil
return scaleSetInstances, nil
}
// Self retrieves the current instance.
func (m *Metadata) Self(ctx context.Context) (cloudtypes.Instance, error) {
func (m *Metadata) Self(ctx context.Context) (metadata.InstanceMetadata, error) {
providerID, err := m.providerID(ctx)
if err != nil {
return cloudtypes.Instance{}, err
return metadata.InstanceMetadata{}, err
}
return m.GetInstance(ctx, providerID)
}
// GetInstance retrieves an instance using its providerID.
func (m *Metadata) GetInstance(ctx context.Context, providerID string) (cloudtypes.Instance, error) {
instance, singleErr := m.getVM(ctx, providerID)
if singleErr == nil {
return instance, nil
}
func (m *Metadata) GetInstance(ctx context.Context, providerID string) (metadata.InstanceMetadata, error) {
instance, scaleSetErr := m.getScaleSetVM(ctx, providerID)
if scaleSetErr == nil {
return instance, nil
}
return cloudtypes.Instance{}, fmt.Errorf("retrieving instance given providerID %v as either single VM or scale set VM: %v; %v", providerID, singleErr, scaleSetErr)
}
// SignalRole signals the constellation role via cloud provider metadata.
// On single VMs, the role is stored in tags, on scale set VMs, the role is inferred from the scale set and not signalied explicitly.
func (m *Metadata) SignalRole(ctx context.Context, role role.Role) error {
providerID, err := m.providerID(ctx)
if err != nil {
return err
}
if _, _, _, _, err := azureshared.ScaleSetInformationFromProviderID(providerID); err == nil {
// scale set instances cannot store tags and role can be inferred from scale set name.
return nil
}
return m.setTag(ctx, core.RoleMetadataKey, role.String())
return metadata.InstanceMetadata{}, fmt.Errorf("retrieving instance given providerID %v: %w", providerID, scaleSetErr)
}
// GetNetworkSecurityGroupName returns the security group name of the resource group.

169
coordinator/cloudprovider/azure/metadata_test.go
View File

@ -8,20 +8,13 @@ import (
"github.com/Azure/azure-sdk-for-go/sdk/azcore/to"
"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute"
"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork"
"github.com/edgelesssys/constellation/coordinator/cloudprovider/cloudtypes"
"github.com/edgelesssys/constellation/coordinator/role"
"github.com/edgelesssys/constellation/internal/cloud/metadata"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestList(t *testing.T) {
wantInstances := []cloudtypes.Instance{
{
Name: "instance-name",
ProviderID: "azure:///subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/virtualMachines/instance-name",
PrivateIPs: []string{"192.0.2.0"},
SSHKeys: map[string][]string{"user": {"key-data"}},
},
wantInstances := []metadata.InstanceMetadata{
{
Name: "scale-set-name-instance-id",
ProviderID: "azure:///subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/virtualMachineScaleSets/scale-set-name/virtualMachines/instance-id",
@ -33,17 +26,15 @@ func TestList(t *testing.T) {
imdsAPI imdsAPI
networkInterfacesAPI networkInterfacesAPI
scaleSetsAPI scaleSetsAPI
virtualMachinesAPI virtualMachinesAPI
virtualMachineScaleSetVMsAPI virtualMachineScaleSetVMsAPI
tagsAPI tagsAPI
wantErr bool
wantInstances []cloudtypes.Instance
wantInstances []metadata.InstanceMetadata
}{
"List works": {
imdsAPI: newIMDSStub(),
networkInterfacesAPI: newNetworkInterfacesStub(),
scaleSetsAPI: newScaleSetsStub(),
virtualMachinesAPI: newVirtualMachinesStub(),
virtualMachineScaleSetVMsAPI: newVirtualMachineScaleSetsVMsStub(),
tagsAPI: newTagsStub(),
wantInstances: wantInstances,
@ -56,16 +47,10 @@ func TestList(t *testing.T) {
imdsAPI: newInvalidIMDSStub(),
wantErr: true,
},
"listVMs fails": {
imdsAPI: newIMDSStub(),
virtualMachinesAPI: newFailingListsVirtualMachinesStub(),
wantErr: true,
},
"listScaleSetVMs fails": {
imdsAPI: newIMDSStub(),
networkInterfacesAPI: newNetworkInterfacesStub(),
scaleSetsAPI: newScaleSetsStub(),
virtualMachinesAPI: newVirtualMachinesStub(),
virtualMachineScaleSetVMsAPI: newFailingListsVirtualMachineScaleSetsVMsStub(),
tagsAPI: newTagsStub(),
wantErr: true,
@ -77,15 +62,14 @@ func TestList(t *testing.T) {
assert := assert.New(t)
require := require.New(t)
metadata := Metadata{
azureMetadata := Metadata{
imdsAPI: tc.imdsAPI,
networkInterfacesAPI: tc.networkInterfacesAPI,
scaleSetsAPI: tc.scaleSetsAPI,
virtualMachinesAPI: tc.virtualMachinesAPI,
virtualMachineScaleSetVMsAPI: tc.virtualMachineScaleSetVMsAPI,
tagsAPI: tc.tagsAPI,
}
instances, err := metadata.List(context.Background())
instances, err := azureMetadata.List(context.Background())
if tc.wantErr {
assert.Error(err)
@ -98,13 +82,13 @@ func TestList(t *testing.T) {
}
func TestSelf(t *testing.T) {
wantVMInstance := cloudtypes.Instance{
wantVMInstance := metadata.InstanceMetadata{
Name: "instance-name",
ProviderID: "azure:///subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/virtualMachines/instance-name",
PrivateIPs: []string{"192.0.2.0"},
SSHKeys: map[string][]string{"user": {"key-data"}},
}
wantScaleSetInstance := cloudtypes.Instance{
wantScaleSetInstance := metadata.InstanceMetadata{
Name: "scale-set-name-instance-id",
ProviderID: "azure:///subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/virtualMachineScaleSets/scale-set-name/virtualMachines/instance-id",
PrivateIPs: []string{"192.0.2.0"},
@ -113,22 +97,19 @@ func TestSelf(t *testing.T) {
testCases := map[string]struct {
imdsAPI imdsAPI
networkInterfacesAPI networkInterfacesAPI
virtualMachinesAPI virtualMachinesAPI
virtualMachineScaleSetVMsAPI virtualMachineScaleSetVMsAPI
wantErr bool
wantInstance cloudtypes.Instance
wantInstance metadata.InstanceMetadata
}{
"self for individual instance works": {
imdsAPI: newIMDSStub(),
networkInterfacesAPI: newNetworkInterfacesStub(),
virtualMachinesAPI: newVirtualMachinesStub(),
virtualMachineScaleSetVMsAPI: newVirtualMachineScaleSetsVMsStub(),
wantInstance: wantVMInstance,
},
"self for scale set instance works": {
imdsAPI: newScaleSetIMDSStub(),
networkInterfacesAPI: newNetworkInterfacesStub(),
virtualMachinesAPI: newVirtualMachinesStub(),
virtualMachineScaleSetVMsAPI: newVirtualMachineScaleSetsVMsStub(),
wantInstance: wantScaleSetInstance,
},
@ -138,7 +119,6 @@ func TestSelf(t *testing.T) {
},
"GetInstance fails": {
imdsAPI: newIMDSStub(),
virtualMachinesAPI: newFailingGetVirtualMachinesStub(),
wantErr: true,
},
}
@ -151,7 +131,6 @@ func TestSelf(t *testing.T) {
metadata := Metadata{
imdsAPI: tc.imdsAPI,
networkInterfacesAPI: tc.networkInterfacesAPI,
virtualMachinesAPI: tc.virtualMachinesAPI,
virtualMachineScaleSetVMsAPI: tc.virtualMachineScaleSetVMsAPI,
}
instance, err := metadata.Self(context.Background())
@ -166,50 +145,6 @@ func TestSelf(t *testing.T) {
}
}
func TestSignalRole(t *testing.T) {
testCases := map[string]struct {
imdsAPI imdsAPI
tagsAPI tagsAPI
wantErr bool
}{
"SignalRole works": {
imdsAPI: newIMDSStub(),
tagsAPI: newTagsStub(),
},
"SignalRole is not attempted on scale set vm": {
imdsAPI: newScaleSetIMDSStub(),
},
"providerID cannot be retrieved": {
imdsAPI: &stubIMDSAPI{retrieveErr: errors.New("imds err")},
wantErr: true,
},
"setting tag fails": {
imdsAPI: newIMDSStub(),
tagsAPI: newFailingTagsStub(),
wantErr: true,
},
}
for name, tc := range testCases {
t.Run(name, func(t *testing.T) {
assert := assert.New(t)
require := require.New(t)
metadata := Metadata{
imdsAPI: tc.imdsAPI,
tagsAPI: tc.tagsAPI,
}
err := metadata.SignalRole(context.Background(), role.Coordinator)
if tc.wantErr {
assert.Error(err)
return
}
require.NoError(err)
})
}
}
func TestGetNetworkSecurityGroupName(t *testing.T) {
name := "network-security-group-name"
testCases := map[string]struct {
@ -719,12 +654,6 @@ func newInvalidIMDSStub() *stubIMDSAPI {
}
}
func newFailingIMDSStub() *stubIMDSAPI {
return &stubIMDSAPI{
retrieveErr: errors.New("imds retrieve error"),
}
}
func newNetworkInterfacesStub() *stubNetworkInterfacesAPI {
return &stubNetworkInterfacesAPI{
getInterface: armnetwork.Interface{
@ -754,81 +683,6 @@ func newScaleSetsStub() *stubScaleSetsAPI {
}
}
func newVirtualMachinesStub() *stubVirtualMachinesAPI {
return &stubVirtualMachinesAPI{
getVM: armcompute.VirtualMachine{
Name: to.StringPtr("instance-name"),
ID: to.StringPtr("/subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/virtualMachines/instance-name"),
Properties: &armcompute.VirtualMachineProperties{
NetworkProfile: &armcompute.NetworkProfile{
NetworkInterfaces: []*armcompute.NetworkInterfaceReference{
{
ID: to.StringPtr("/subscriptions/subscription/resourceGroups/resource-group/providers/Microsoft.Network/networkInterfaces/interface-name"),
},
},
},
OSProfile: &armcompute.OSProfile{
LinuxConfiguration: &armcompute.LinuxConfiguration{
SSH: &armcompute.SSHConfiguration{
PublicKeys: []*armcompute.SSHPublicKey{
{
KeyData: to.StringPtr("key-data"),
Path: to.StringPtr("/home/user/.ssh/authorized_keys"),
},
},
},
},
},
},
},
listPages: [][]*armcompute.VirtualMachine{
{
{
Name: to.StringPtr("instance-name"),
ID: to.StringPtr("/subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/virtualMachines/instance-name"),
Properties: &armcompute.VirtualMachineProperties{
NetworkProfile: &armcompute.NetworkProfile{
NetworkInterfaces: []*armcompute.NetworkInterfaceReference{
{
ID: to.StringPtr("/subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Network/networkInterfaces/interface-name"),
},
},
},
OSProfile: &armcompute.OSProfile{
LinuxConfiguration: &armcompute.LinuxConfiguration{
SSH: &armcompute.SSHConfiguration{
PublicKeys: []*armcompute.SSHPublicKey{
{
KeyData: to.StringPtr("key-data"),
Path: to.StringPtr("/home/user/.ssh/authorized_keys"),
},
},
},
},
},
},
},
},
},
}
}
func newFailingListsVirtualMachinesStub() *stubVirtualMachinesAPI {
return &stubVirtualMachinesAPI{
listPages: [][]*armcompute.VirtualMachine{
{
{},
},
},
}
}
func newFailingGetVirtualMachinesStub() *stubVirtualMachinesAPI {
return &stubVirtualMachinesAPI{
getErr: errors.New("get err"),
}
}
func newVirtualMachineScaleSetsVMsStub() *stubVirtualMachineScaleSetVMsAPI {
return &stubVirtualMachineScaleSetVMsAPI{
getVM: armcompute.VirtualMachineScaleSetVM{
@ -907,10 +761,3 @@ func newFailingListsVirtualMachineScaleSetsVMsStub() *stubVirtualMachineScaleSet
func newTagsStub() *stubTagsAPI {
return &stubTagsAPI{}
}
func newFailingTagsStub() *stubTagsAPI {
return &stubTagsAPI{
createOrUpdateAtScopeErr: errors.New("createOrUpdateErr"),
updateAtScopeErr: errors.New("updateErr"),
}
}

24
coordinator/cloudprovider/azure/scaleset.go
View File

@ -7,9 +7,9 @@ import (
"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute"
"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork"
"github.com/edgelesssys/constellation/coordinator/cloudprovider/cloudtypes"
"github.com/edgelesssys/constellation/coordinator/role"
"github.com/edgelesssys/constellation/internal/azureshared"
"github.com/edgelesssys/constellation/internal/cloud/metadata"
)
var (
@ -18,30 +18,30 @@ var (
)
// getScaleSetVM tries to get an azure vm belonging to a scale set.
func (m *Metadata) getScaleSetVM(ctx context.Context, providerID string) (cloudtypes.Instance, error) {
func (m *Metadata) getScaleSetVM(ctx context.Context, providerID string) (metadata.InstanceMetadata, error) {
_, resourceGroup, scaleSet, instanceID, err := azureshared.ScaleSetInformationFromProviderID(providerID)
if err != nil {
return cloudtypes.Instance{}, err
return metadata.InstanceMetadata{}, err
}
vmResp, err := m.virtualMachineScaleSetVMsAPI.Get(ctx, resourceGroup, scaleSet, instanceID, nil)
if err != nil {
return cloudtypes.Instance{}, err
return metadata.InstanceMetadata{}, err
}
networkInterfaces, err := m.getScaleSetVMInterfaces(ctx, vmResp.VirtualMachineScaleSetVM, resourceGroup, scaleSet, instanceID)
if err != nil {
return cloudtypes.Instance{}, err
return metadata.InstanceMetadata{}, err
}
publicIPAddresses, err := m.getScaleSetVMPublicIPAddresses(ctx, resourceGroup, scaleSet, instanceID, networkInterfaces)
if err != nil {
return cloudtypes.Instance{}, err
return metadata.InstanceMetadata{}, err
}
return convertScaleSetVMToCoreInstance(scaleSet, vmResp.VirtualMachineScaleSetVM, networkInterfaces, publicIPAddresses)
}
// listScaleSetVMs lists all scale set VMs in the current resource group.
func (m *Metadata) listScaleSetVMs(ctx context.Context, resourceGroup string) ([]cloudtypes.Instance, error) {
instances := []cloudtypes.Instance{}
func (m *Metadata) listScaleSetVMs(ctx context.Context, resourceGroup string) ([]metadata.InstanceMetadata, error) {
instances := []metadata.InstanceMetadata{}
scaleSetPager := m.scaleSetsAPI.List(resourceGroup, nil)
for scaleSetPager.NextPage(ctx) {
for _, scaleSet := range scaleSetPager.PageResponse().Value {
@ -71,12 +71,12 @@ func (m *Metadata) listScaleSetVMs(ctx context.Context, resourceGroup string) ([
}
// convertScaleSetVMToCoreInstance converts an azure scale set virtual machine with interface configurations into a core.Instance.
func convertScaleSetVMToCoreInstance(scaleSet string, vm armcompute.VirtualMachineScaleSetVM, networkInterfaces []armnetwork.Interface, publicIPAddresses []string) (cloudtypes.Instance, error) {
func convertScaleSetVMToCoreInstance(scaleSet string, vm armcompute.VirtualMachineScaleSetVM, networkInterfaces []armnetwork.Interface, publicIPAddresses []string) (metadata.InstanceMetadata, error) {
if vm.ID == nil {
return cloudtypes.Instance{}, errors.New("retrieving instance from armcompute API client returned no instance ID")
return metadata.InstanceMetadata{}, errors.New("retrieving instance from armcompute API client returned no instance ID")
}
if vm.Properties == nil || vm.Properties.OSProfile == nil || vm.Properties.OSProfile.ComputerName == nil {
return cloudtypes.Instance{}, errors.New("retrieving instance from armcompute API client returned no computer name")
return metadata.InstanceMetadata{}, errors.New("retrieving instance from armcompute API client returned no computer name")
}
var sshKeys map[string][]string
if vm.Properties.OSProfile.LinuxConfiguration == nil || vm.Properties.OSProfile.LinuxConfiguration.SSH == nil {
@ -84,7 +84,7 @@ func convertScaleSetVMToCoreInstance(scaleSet string, vm armcompute.VirtualMachi
} else {
sshKeys = extractSSHKeys(*vm.Properties.OSProfile.LinuxConfiguration.SSH)
}
return cloudtypes.Instance{
return metadata.InstanceMetadata{
Name: *vm.Properties.OSProfile.ComputerName,
ProviderID: "azure://" + *vm.ID,
Role: extractScaleSetVMRole(scaleSet),

29
coordinator/cloudprovider/azure/scaleset_test.go
View File

@ -8,14 +8,14 @@ import (
"github.com/Azure/azure-sdk-for-go/sdk/azcore/to"
"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute"
"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork"
"github.com/edgelesssys/constellation/coordinator/cloudprovider/cloudtypes"
"github.com/edgelesssys/constellation/coordinator/role"
"github.com/edgelesssys/constellation/internal/cloud/metadata"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestGetScaleSetVM(t *testing.T) {
wantInstance := cloudtypes.Instance{
wantInstance := metadata.InstanceMetadata{
Name: "scale-set-name-instance-id",
ProviderID: "azure:///subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/virtualMachineScaleSets/scale-set-name/virtualMachines/instance-id",
PrivateIPs: []string{"192.0.2.0"},
@ -26,7 +26,7 @@ func TestGetScaleSetVM(t *testing.T) {
networkInterfacesAPI networkInterfacesAPI
virtualMachineScaleSetVMsAPI virtualMachineScaleSetVMsAPI
wantErr bool
wantInstance cloudtypes.Instance
wantInstance metadata.InstanceMetadata
}{
"getVM for scale set instance works": {
providerID: "azure:///subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/virtualMachineScaleSets/scale-set-name/virtualMachines/instance-id",
@ -43,12 +43,6 @@ func TestGetScaleSetVM(t *testing.T) {
virtualMachineScaleSetVMsAPI: newFailingGetScaleSetVirtualMachinesStub(),
wantErr: true,
},
"retrieving interfaces fails": {
providerID: "azure:///subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/virtualMachineScaleSets/scale-set-name/virtualMachines/instance-id",
virtualMachineScaleSetVMsAPI: newVirtualMachineScaleSetsVMsStub(),
networkInterfacesAPI: newFailingNetworkInterfacesStub(),
wantErr: true,
},
"conversion fails": {
providerID: "azure:///subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/virtualMachineScaleSets/scale-set-name/virtualMachines/instance-id",
virtualMachineScaleSetVMsAPI: newGetInvalidScaleSetVirtualMachinesStub(),
@ -79,7 +73,7 @@ func TestGetScaleSetVM(t *testing.T) {
}
func TestListScaleSetVMs(t *testing.T) {
wantInstances := []cloudtypes.Instance{
wantInstances := []metadata.InstanceMetadata{
{
Name: "scale-set-name-instance-id",
ProviderID: "azure:///subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/virtualMachineScaleSets/scale-set-name/virtualMachines/instance-id",
@ -93,7 +87,7 @@ func TestListScaleSetVMs(t *testing.T) {
virtualMachineScaleSetVMsAPI virtualMachineScaleSetVMsAPI
scaleSetsAPI scaleSetsAPI
wantErr bool
wantInstances []cloudtypes.Instance
wantInstances []metadata.InstanceMetadata
}{
"listVMs works": {
imdsAPI: newIMDSStub(),
@ -114,7 +108,7 @@ func TestListScaleSetVMs(t *testing.T) {
networkInterfacesAPI: newNetworkInterfacesStub(),
virtualMachineScaleSetVMsAPI: &stubVirtualMachineScaleSetVMsAPI{},
scaleSetsAPI: newScaleSetsStub(),
wantInstances: []cloudtypes.Instance{},
wantInstances: []metadata.InstanceMetadata{},
},
"can skip nil in VM list": {
imdsAPI: newIMDSStub(),
@ -123,13 +117,6 @@ func TestListScaleSetVMs(t *testing.T) {
scaleSetsAPI: newScaleSetsStub(),
wantInstances: wantInstances,
},
"retrieving network interfaces fails": {
imdsAPI: newIMDSStub(),
networkInterfacesAPI: newFailingNetworkInterfacesStub(),
virtualMachineScaleSetVMsAPI: newVirtualMachineScaleSetsVMsStub(),
scaleSetsAPI: newScaleSetsStub(),
wantErr: true,
},
"converting instance fails": {
imdsAPI: newIMDSStub(),
networkInterfacesAPI: newNetworkInterfacesStub(),
@ -168,7 +155,7 @@ func TestConvertScaleSetVMToCoreInstance(t *testing.T) {
inInterface []armnetwork.Interface
inPublicIPs []string
wantErr bool
wantInstance cloudtypes.Instance
wantInstance metadata.InstanceMetadata
}{
"conversion works": {
inVM: armcompute.VirtualMachineScaleSetVM{
@ -197,7 +184,7 @@ func TestConvertScaleSetVMToCoreInstance(t *testing.T) {
},
},
inPublicIPs: []string{"192.0.2.100", "192.0.2.101"},
wantInstance: cloudtypes.Instance{
wantInstance: metadata.InstanceMetadata{
Name: "scale-set-name-instance-id",
ProviderID: "azure:///subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/virtualMachineScaleSets/scale-set-name/virtualMachines/instance-id",
PrivateIPs: []string{"192.0.2.0"},

91
coordinator/cloudprovider/azure/virtualmachine.go
View File

@ -1,91 +0,0 @@
package azure
import (
"context"
"fmt"
"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute"
"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork"
"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources"
"github.com/Azure/go-autorest/autorest/to"
"github.com/edgelesssys/constellation/coordinator/cloudprovider"
"github.com/edgelesssys/constellation/coordinator/cloudprovider/cloudtypes"
"github.com/edgelesssys/constellation/internal/azureshared"
)
func (m *Metadata) getVM(ctx context.Context, providerID string) (cloudtypes.Instance, error) {
_, resourceGroup, instanceName, err := azureshared.VMInformationFromProviderID(providerID)
if err != nil {
return cloudtypes.Instance{}, err
}
vmResp, err := m.virtualMachinesAPI.Get(ctx, resourceGroup, instanceName, nil)
if err != nil {
return cloudtypes.Instance{}, err
}
interfaceIPConfigurations, err := m.getVMInterfaces(ctx, vmResp.VirtualMachine, resourceGroup)
if err != nil {
return cloudtypes.Instance{}, err
}
return convertVMToCoreInstance(vmResp.VirtualMachine, interfaceIPConfigurations)
}
// listVMs lists all individual VMs in the current resource group.
func (m *Metadata) listVMs(ctx context.Context, resourceGroup string) ([]cloudtypes.Instance, error) {
instances := []cloudtypes.Instance{}
pager := m.virtualMachinesAPI.List(resourceGroup, nil)
for pager.NextPage(ctx) {
for _, vm := range pager.PageResponse().Value {
if vm == nil {
continue
}
interfaces, err := m.getVMInterfaces(ctx, *vm, resourceGroup)
if err != nil {
return nil, err
}
instance, err := convertVMToCoreInstance(*vm, interfaces)
if err != nil {
return nil, err
}
instances = append(instances, instance)
}
}
return instances, nil
}
// setTag merges key-value pair into VM tags.
func (m *Metadata) setTag(ctx context.Context, key, value string) error {
instanceMetadata, err := m.imdsAPI.Retrieve(ctx)
if err != nil {
return err
}
_, err = m.tagsAPI.UpdateAtScope(ctx, instanceMetadata.Compute.ResourceID, armresources.TagsPatchResource{
Operation: armresources.TagsPatchOperationMerge.ToPtr(),
Properties: &armresources.Tags{
Tags: map[string]*string{
key: to.StringPtr(value),
},
},
}, nil)
return err
}
// convertVMToCoreInstance converts an azure virtual machine with interface configurations into a cloudtypes.Instance.
func convertVMToCoreInstance(vm armcompute.VirtualMachine, networkInterfaces []armnetwork.Interface) (cloudtypes.Instance, error) {
if vm.Name == nil || vm.ID == nil {
return cloudtypes.Instance{}, fmt.Errorf("retrieving instance from armcompute API client returned invalid instance Name (%v) or ID (%v)", vm.Name, vm.ID)
}
var sshKeys map[string][]string
if vm.Properties == nil || vm.Properties.OSProfile == nil || vm.Properties.OSProfile.LinuxConfiguration == nil || vm.Properties.OSProfile.LinuxConfiguration.SSH == nil {
sshKeys = map[string][]string{}
} else {
sshKeys = extractSSHKeys(*vm.Properties.OSProfile.LinuxConfiguration.SSH)
}
metadata := extractInstanceTags(vm.Tags)
return cloudtypes.Instance{
Name: *vm.Name,
ProviderID: "azure://" + *vm.ID,
Role: cloudprovider.ExtractRole(metadata),
PrivateIPs: extractPrivateIPs(networkInterfaces),
SSHKeys: sshKeys,
}, nil
}

374
coordinator/cloudprovider/azure/virtualmachine_test.go
View File

@ -1,374 +0,0 @@
package azure
import (
"context"
"errors"
"testing"
"github.com/Azure/azure-sdk-for-go/sdk/azcore/to"
"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute"
"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork"
"github.com/edgelesssys/constellation/coordinator/cloudprovider/cloudtypes"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestGetVM(t *testing.T) {
wantInstance := cloudtypes.Instance{
Name: "instance-name",
ProviderID: "azure:///subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/virtualMachines/instance-name",
PrivateIPs: []string{"192.0.2.0"},
SSHKeys: map[string][]string{"user": {"key-data"}},
}
testCases := map[string]struct {
providerID string
networkInterfacesAPI networkInterfacesAPI
virtualMachinesAPI virtualMachinesAPI
wantErr bool
wantInstance cloudtypes.Instance
}{
"getVM for individual instance works": {
providerID: "azure:///subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/virtualMachines/instance-name",
networkInterfacesAPI: newNetworkInterfacesStub(),
virtualMachinesAPI: newVirtualMachinesStub(),
wantInstance: wantInstance,
},
"getVM for scale set instance must fail": {
providerID: "azure:///subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/virtualMachineScaleSets/scale-set-name/virtualMachines/instance-id",
wantErr: true,
},
"Get fails": {
providerID: "azure:///subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/virtualMachines/instance-name",
virtualMachinesAPI: newFailingGetVirtualMachinesStub(),
wantErr: true,
},
"retrieving interfaces fails": {
providerID: "azure:///subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/virtualMachines/instance-name",
virtualMachinesAPI: newVirtualMachinesStub(),
networkInterfacesAPI: newFailingNetworkInterfacesStub(),
wantErr: true,
},
"conversion fails": {
providerID: "azure:///subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/virtualMachines/instance-name",
virtualMachinesAPI: newGetInvalidVirtualMachinesStub(),
networkInterfacesAPI: newNetworkInterfacesStub(),
wantErr: true,
},
}
for name, tc := range testCases {
t.Run(name, func(t *testing.T) {
assert := assert.New(t)
require := require.New(t)
metadata := Metadata{
networkInterfacesAPI: tc.networkInterfacesAPI,
virtualMachinesAPI: tc.virtualMachinesAPI,
}
instance, err := metadata.getVM(context.Background(), tc.providerID)
if tc.wantErr {
assert.Error(err)
return
}
require.NoError(err)
assert.Equal(tc.wantInstance, instance)
})
}
}
func TestListVMs(t *testing.T) {
wantInstances := []cloudtypes.Instance{
{
Name: "instance-name",
ProviderID: "azure:///subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/virtualMachines/instance-name",
PrivateIPs: []string{"192.0.2.0"},
SSHKeys: map[string][]string{"user": {"key-data"}},
},
}
testCases := map[string]struct {
imdsAPI imdsAPI
networkInterfacesAPI networkInterfacesAPI
virtualMachinesAPI virtualMachinesAPI
wantErr bool
wantInstances []cloudtypes.Instance
}{
"listVMs works": {
imdsAPI: newIMDSStub(),
networkInterfacesAPI: newNetworkInterfacesStub(),
virtualMachinesAPI: newVirtualMachinesStub(),
wantInstances: wantInstances,
},
"listVMs can return 0 VMs": {
imdsAPI: newIMDSStub(),
networkInterfacesAPI: newNetworkInterfacesStub(),
virtualMachinesAPI: &stubVirtualMachinesAPI{},
wantInstances: []cloudtypes.Instance{},
},
"can skip nil in VM list": {
imdsAPI: newIMDSStub(),
networkInterfacesAPI: newNetworkInterfacesStub(),
virtualMachinesAPI: newListContainingNilVirtualMachinesStub(),
wantInstances: wantInstances,
},
"retrieving network interfaces fails": {
imdsAPI: newIMDSStub(),
networkInterfacesAPI: newFailingNetworkInterfacesStub(),
virtualMachinesAPI: newVirtualMachinesStub(),
wantErr: true,
},
"converting instance fails": {
imdsAPI: newIMDSStub(),
networkInterfacesAPI: newNetworkInterfacesStub(),
virtualMachinesAPI: newListContainingInvalidVirtualMachinesStub(),
wantErr: true,
},
}
for name, tc := range testCases {
t.Run(name, func(t *testing.T) {
assert := assert.New(t)
require := require.New(t)
metadata := Metadata{
imdsAPI: tc.imdsAPI,
networkInterfacesAPI: tc.networkInterfacesAPI,
virtualMachinesAPI: tc.virtualMachinesAPI,
}
instances, err := metadata.listVMs(context.Background(), "resource-group")
if tc.wantErr {
assert.Error(err)
return
}
require.NoError(err)
assert.ElementsMatch(tc.wantInstances, instances)
})
}
}
func TestSetTag(t *testing.T) {
testCases := map[string]struct {
imdsAPI imdsAPI
tagsAPI tagsAPI
wantErr bool
}{
"setTag works": {
imdsAPI: newIMDSStub(),
tagsAPI: newTagsStub(),
},
"retrieving resource ID fails": {
imdsAPI: newFailingIMDSStub(),
tagsAPI: newTagsStub(),
wantErr: true,
},
"updating tags fails": {
imdsAPI: newIMDSStub(),
tagsAPI: newFailingTagsStub(),
wantErr: true,
},
}
for name, tc := range testCases {
t.Run(name, func(t *testing.T) {
assert := assert.New(t)
require := require.New(t)
metadata := Metadata{
imdsAPI: tc.imdsAPI,
tagsAPI: tc.tagsAPI,
}
err := metadata.setTag(context.Background(), "key", "value")
if tc.wantErr {
assert.Error(err)
return
}
require.NoError(err)
})
}
}
func TestConvertVMToCoreInstance(t *testing.T) {
testCases := map[string]struct {
inVM armcompute.VirtualMachine
inInterface []armnetwork.Interface
wantErr bool
wantInstance cloudtypes.Instance
}{
"conversion works": {
inVM: armcompute.VirtualMachine{
Name: to.StringPtr("instance-name"),
ID: to.StringPtr("/subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/virtualMachines/instance-name"),
Tags: map[string]*string{"tag-key": to.StringPtr("tag-value")},
Properties: &armcompute.VirtualMachineProperties{
OSProfile: &armcompute.OSProfile{
LinuxConfiguration: &armcompute.LinuxConfiguration{
SSH: &armcompute.SSHConfiguration{
PublicKeys: []*armcompute.SSHPublicKey{
{
Path: to.StringPtr("/home/user/.ssh/authorized_keys"),
KeyData: to.StringPtr("key-data"),
},
},
},
},
},
},
},
inInterface: []armnetwork.Interface{
{
Name: to.StringPtr("interface-name"),
ID: to.StringPtr("/subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Network/networkInterfaces/interface-name"),
Properties: &armnetwork.InterfacePropertiesFormat{
IPConfigurations: []*armnetwork.InterfaceIPConfiguration{
{
Properties: &armnetwork.InterfaceIPConfigurationPropertiesFormat{
PrivateIPAddress: to.StringPtr("192.0.2.0"),
},
},
},
},
},
},
wantInstance: cloudtypes.Instance{
Name: "instance-name",
ProviderID: "azure:///subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/virtualMachines/instance-name",
PrivateIPs: []string{"192.0.2.0"},
SSHKeys: map[string][]string{"user": {"key-data"}},
},
},
"conversion without SSH keys works": {
inVM: armcompute.VirtualMachine{
Name: to.StringPtr("instance-name"),
ID: to.StringPtr("/subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/virtualMachines/instance-name"),
Tags: map[string]*string{"tag-key": to.StringPtr("tag-value")},
},
inInterface: []armnetwork.Interface{
{
Name: to.StringPtr("interface-name"),
ID: to.StringPtr("/subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Network/networkInterfaces/interface-name"),
Properties: &armnetwork.InterfacePropertiesFormat{
IPConfigurations: []*armnetwork.InterfaceIPConfiguration{
{
Properties: &armnetwork.InterfaceIPConfigurationPropertiesFormat{
PrivateIPAddress: to.StringPtr("192.0.2.0"),
},
},
},
},
},
},
wantInstance: cloudtypes.Instance{
Name: "instance-name",
ProviderID: "azure:///subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/virtualMachines/instance-name",
PrivateIPs: []string{"192.0.2.0"},
SSHKeys: map[string][]string{},
},
},
"invalid instance": {
inVM: armcompute.VirtualMachine{},
wantErr: true,
},
}
for name, tc := range testCases {
t.Run(name, func(t *testing.T) {
assert := assert.New(t)
require := require.New(t)
instance, err := convertVMToCoreInstance(tc.inVM, tc.inInterface)
if tc.wantErr {
assert.Error(err)
return
}
require.NoError(err)
assert.Equal(tc.wantInstance, instance)
})
}
}
func newFailingNetworkInterfacesStub() *stubNetworkInterfacesAPI {
return &stubNetworkInterfacesAPI{
getErr: errors.New("get err"),
}
}
func newGetInvalidVirtualMachinesStub() *stubVirtualMachinesAPI {
return &stubVirtualMachinesAPI{
getVM: armcompute.VirtualMachine{},
}
}
func newListContainingNilVirtualMachinesStub() *stubVirtualMachinesAPI {
return &stubVirtualMachinesAPI{
listPages: [][]*armcompute.VirtualMachine{
{
nil,
{
Name: to.StringPtr("instance-name"),
ID: to.StringPtr("/subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/virtualMachines/instance-name"),
Tags: map[string]*string{
"tag-key": to.StringPtr("tag-value"),
},
Properties: &armcompute.VirtualMachineProperties{
NetworkProfile: &armcompute.NetworkProfile{
NetworkInterfaces: []*armcompute.NetworkInterfaceReference{
{
ID: to.StringPtr("/subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Network/networkInterfaces/interface-name"),
},
},
},
OSProfile: &armcompute.OSProfile{
LinuxConfiguration: &armcompute.LinuxConfiguration{
SSH: &armcompute.SSHConfiguration{
PublicKeys: []*armcompute.SSHPublicKey{
{
KeyData: to.StringPtr("key-data"),
Path: to.StringPtr("/home/user/.ssh/authorized_keys"),
},
},
},
},
},
},
},
},
},
}
}
func newListContainingInvalidVirtualMachinesStub() *stubVirtualMachinesAPI {
return &stubVirtualMachinesAPI{
listPages: [][]*armcompute.VirtualMachine{
{
{
Name: nil,
ID: nil,
Properties: &armcompute.VirtualMachineProperties{
NetworkProfile: &armcompute.NetworkProfile{
NetworkInterfaces: []*armcompute.NetworkInterfaceReference{
{
ID: to.StringPtr("/subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Network/networkInterfaces/interface-name"),
},
},
},
OSProfile: &armcompute.OSProfile{
LinuxConfiguration: &armcompute.LinuxConfiguration{
SSH: &armcompute.SSHConfiguration{
PublicKeys: []*armcompute.SSHPublicKey{
{
KeyData: to.StringPtr("key-data"),
Path: to.StringPtr("/home/user/.ssh/authorized_keys"),
},
},
},
},
},
},
},
},
},
}
}

12
coordinator/cloudprovider/azure/wrappers.go
View File

@ -68,18 +68,6 @@ func (c *loadBalancersClient) List(resourceGroupName string, options *armnetwork
return c.LoadBalancersClient.List(resourceGroupName, options)
}
type virtualMachinesClient struct {
*armcompute.VirtualMachinesClient
}
func (c *virtualMachinesClient) Get(ctx context.Context, resourceGroupName, vmName string, options *armcompute.VirtualMachinesClientGetOptions) (armcompute.VirtualMachinesClientGetResponse, error) {
return c.VirtualMachinesClient.Get(ctx, resourceGroupName, vmName, options)
}
func (c *virtualMachinesClient) List(resourceGroupName string, options *armcompute.VirtualMachinesClientListOptions) virtualMachinesClientListPager {
return c.VirtualMachinesClient.List(resourceGroupName, options)
}
type virtualMachineScaleSetVMsClient struct {
*armcompute.VirtualMachineScaleSetVMsClient
}

15
coordinator/cloudprovider/cloudtypes/instance.go
View File

@ -1,15 +0,0 @@
package cloudtypes
import "github.com/edgelesssys/constellation/coordinator/role"
// Instance describes metadata of a peer.
type Instance struct {
Name string
ProviderID string
Role role.Role
PrivateIPs []string
PublicIPs []string
AliasIPRanges []string
// SSHKeys maps usernames to ssh public keys.
SSHKeys map[string][]string
}

4
coordinator/cloudprovider/gcp/autoscaler.go
View File

@ -1,8 +1,8 @@
package gcp
import (
"github.com/edgelesssys/constellation/coordinator/cloudprovider/cloudtypes"
"github.com/edgelesssys/constellation/coordinator/kubernetes/k8sapi/resources"
"github.com/edgelesssys/constellation/internal/cloud/metadata"
k8s "k8s.io/api/core/v1"
)
@ -15,7 +15,7 @@ func (a *Autoscaler) Name() string {
}
// Secrets returns a list of secrets to deploy together with the k8s cluster-autoscaler.
func (a *Autoscaler) Secrets(instance cloudtypes.Instance, cloudServiceAccountURI string) (resources.Secrets, error) {
func (a *Autoscaler) Secrets(instance metadata.InstanceMetadata, cloudServiceAccountURI string) (resources.Secrets, error) {
return resources.Secrets{}, nil
}

4
coordinator/cloudprovider/gcp/autoscaler_test.go
View File

@ -3,7 +3,7 @@ package gcp
import (
"testing"
"github.com/edgelesssys/constellation/coordinator/cloudprovider/cloudtypes"
"github.com/edgelesssys/constellation/internal/cloud/metadata"
"github.com/stretchr/testify/assert"
)
@ -12,7 +12,7 @@ func TestTrivialAutoscalerFunctions(t *testing.T) {
autoscaler := Autoscaler{}
assert.NotEmpty(autoscaler.Name())
assert.Empty(autoscaler.Secrets(cloudtypes.Instance{}, ""))
assert.Empty(autoscaler.Secrets(metadata.InstanceMetadata{}, ""))
assert.NotEmpty(autoscaler.Volumes())
assert.NotEmpty(autoscaler.VolumeMounts())
assert.NotEmpty(autoscaler.Env())

6
coordinator/cloudprovider/gcp/ccm.go
View File

@ -7,8 +7,8 @@ import (
"strings"
"github.com/edgelesssys/constellation/coordinator/cloudprovider"
"github.com/edgelesssys/constellation/coordinator/cloudprovider/cloudtypes"
"github.com/edgelesssys/constellation/coordinator/kubernetes/k8sapi/resources"
"github.com/edgelesssys/constellation/internal/cloud/metadata"
"github.com/edgelesssys/constellation/internal/gcpshared"
k8s "k8s.io/api/core/v1"
v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@ -46,7 +46,7 @@ func (c *CloudControllerManager) ExtraArgs() []string {
// ConfigMaps returns a list of ConfigMaps to deploy together with the k8s cloud-controller-manager
// Reference: https://kubernetes.io/docs/concepts/configuration/configmap/ .
func (c *CloudControllerManager) ConfigMaps(instance cloudtypes.Instance) (resources.ConfigMaps, error) {
func (c *CloudControllerManager) ConfigMaps(instance metadata.InstanceMetadata) (resources.ConfigMaps, error) {
// GCP CCM expects cloud config to contain the GCP project-id and other configuration.
// reference: https://github.com/kubernetes/cloud-provider-gcp/blob/master/cluster/gce/gci/configure-helper.sh#L791-L892
var config strings.Builder
@ -80,7 +80,7 @@ func (c *CloudControllerManager) ConfigMaps(instance cloudtypes.Instance) (resou
// Secrets returns a list of secrets to deploy together with the k8s cloud-controller-manager.
// Reference: https://kubernetes.io/docs/concepts/configuration/secret/ .
func (c *CloudControllerManager) Secrets(ctx context.Context, instance cloudtypes.Instance, cloudServiceAccountURI string) (resources.Secrets, error) {
func (c *CloudControllerManager) Secrets(ctx context.Context, _ string, cloudServiceAccountURI string) (resources.Secrets, error) {
serviceAccountKey, err := gcpshared.ServiceAccountKeyFromURI(cloudServiceAccountURI)
if err != nil {
return resources.Secrets{}, err

12
coordinator/cloudprovider/gcp/ccm_test.go
View File

@ -5,8 +5,8 @@ import (
"encoding/json"
"testing"
"github.com/edgelesssys/constellation/coordinator/cloudprovider/cloudtypes"
"github.com/edgelesssys/constellation/coordinator/kubernetes/k8sapi/resources"
"github.com/edgelesssys/constellation/internal/cloud/metadata"
"github.com/edgelesssys/constellation/internal/gcpshared"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
@ -16,12 +16,12 @@ import (
func TestConfigMaps(t *testing.T) {
testCases := map[string]struct {
instance cloudtypes.Instance
instance metadata.InstanceMetadata
wantConfigMaps resources.ConfigMaps
wantErr bool
}{
"ConfigMaps works": {
instance: cloudtypes.Instance{ProviderID: "gce://project-id/zone/instanceName-UID-0", Name: "instanceName-UID-0"},
instance: metadata.InstanceMetadata{ProviderID: "gce://project-id/zone/instanceName-UID-0", Name: "instanceName-UID-0"},
wantConfigMaps: resources.ConfigMaps{
&k8s.ConfigMap{
TypeMeta: v1.TypeMeta{
@ -43,7 +43,7 @@ node-tags = constellation-UID
},
},
"invalid providerID fails": {
instance: cloudtypes.Instance{ProviderID: "invalid"},
instance: metadata.InstanceMetadata{ProviderID: "invalid"},
wantErr: true,
},
}
@ -82,7 +82,7 @@ func TestSecrets(t *testing.T) {
rawKey, err := json.Marshal(serviceAccountKey)
require.NoError(t, err)
testCases := map[string]struct {
instance cloudtypes.Instance
instance metadata.InstanceMetadata
cloudServiceAccountURI string
wantSecrets resources.Secrets
wantErr bool
@ -117,7 +117,7 @@ func TestSecrets(t *testing.T) {
require := require.New(t)
cloud := CloudControllerManager{}
secrets, err := cloud.Secrets(context.Background(), tc.instance, tc.cloudServiceAccountURI)
secrets, err := cloud.Secrets(context.Background(), tc.instance.ProviderID, tc.cloudServiceAccountURI)
if tc.wantErr {
assert.Error(err)
return

33
coordinator/cloudprovider/gcp/client.go
View File

@ -7,16 +7,17 @@ import (
"strings"
compute "cloud.google.com/go/compute/apiv1"
"github.com/edgelesssys/constellation/coordinator/cloudprovider"
"github.com/edgelesssys/constellation/coordinator/cloudprovider/cloudtypes"
"github.com/edgelesssys/constellation/coordinator/core"
"github.com/edgelesssys/constellation/internal/cloud/metadata"
"github.com/edgelesssys/constellation/internal/gcpshared"
"google.golang.org/api/iterator"
computepb "google.golang.org/genproto/googleapis/cloud/compute/v1"
"google.golang.org/protobuf/proto"
)
const gcpSSHMetadataKey = "ssh-keys"
const (
gcpSSHMetadataKey = "ssh-keys"
constellationUIDMetadataKey = "constellation-uid"
)
var zoneFromRegionRegex = regexp.MustCompile("([a-z]*-[a-z]*[0-9])")
@ -51,7 +52,7 @@ func NewClient(ctx context.Context) (*Client, error) {
}
// RetrieveInstances returns list of instances including their ips and metadata.
func (c *Client) RetrieveInstances(ctx context.Context, project, zone string) ([]cloudtypes.Instance, error) {
func (c *Client) RetrieveInstances(ctx context.Context, project, zone string) ([]metadata.InstanceMetadata, error) {
uid, err := c.uid()
if err != nil {
return nil, err
@ -62,7 +63,7 @@ func (c *Client) RetrieveInstances(ctx context.Context, project, zone string) ([
}
instanceIterator := c.instanceAPI.List(ctx, req)
instances := []cloudtypes.Instance{}
instances := []metadata.InstanceMetadata{}
for {
resp, err := instanceIterator.Next()
if err == iterator.Done {
@ -73,7 +74,7 @@ func (c *Client) RetrieveInstances(ctx context.Context, project, zone string) ([
}
metadata := extractInstanceMetadata(resp.Metadata, "", false)
// skip instances not belonging to the current constellation
if instanceUID, ok := metadata[core.ConstellationUIDMetadataKey]; !ok || instanceUID != uid {
if instanceUID, ok := metadata[constellationUIDMetadataKey]; !ok || instanceUID != uid {
continue
}
instance, err := convertToCoreInstance(resp, project, zone)
@ -87,10 +88,10 @@ func (c *Client) RetrieveInstances(ctx context.Context, project, zone string) ([
}
// RetrieveInstance returns a an instance including ips and metadata.
func (c *Client) RetrieveInstance(ctx context.Context, project, zone, instanceName string) (cloudtypes.Instance, error) {
func (c *Client) RetrieveInstance(ctx context.Context, project, zone, instanceName string) (metadata.InstanceMetadata, error) {
instance, err := c.getComputeInstance(ctx, project, zone, instanceName)
if err != nil {
return cloudtypes.Instance{}, err
return metadata.InstanceMetadata{}, err
}
return convertToCoreInstance(instance, project, zone)
@ -286,7 +287,7 @@ func (c *Client) updateInstanceMetadata(ctx context.Context, project, zone, inst
// uid retrieves the current instances uid.
func (c *Client) uid() (string, error) {
// API endpoint: http://metadata.google.internal/computeMetadata/v1/instance/attributes/constellation-uid
uid, err := c.RetrieveInstanceMetadata(core.ConstellationUIDMetadataKey)
uid, err := c.RetrieveInstanceMetadata(constellationUIDMetadataKey)
if err != nil {
return "", fmt.Errorf("retrieving constellation uid: %w", err)
}
@ -367,19 +368,19 @@ func extractSSHKeys(metadata map[string]string) map[string][]string {
}
// convertToCoreInstance converts a *computepb.Instance to a core.Instance.
func convertToCoreInstance(in *computepb.Instance, project string, zone string) (cloudtypes.Instance, error) {
func convertToCoreInstance(in *computepb.Instance, project string, zone string) (metadata.InstanceMetadata, error) {
if in.Name == nil {
return cloudtypes.Instance{}, fmt.Errorf("retrieving instance from compute API client returned invalid instance Name: %v", in.Name)
return metadata.InstanceMetadata{}, fmt.Errorf("retrieving instance from compute API client returned invalid instance Name: %v", in.Name)
}
metadata := extractInstanceMetadata(in.Metadata, "", false)
return cloudtypes.Instance{
mdata := extractInstanceMetadata(in.Metadata, "", false)
return metadata.InstanceMetadata{
Name: *in.Name,
ProviderID: gcpshared.JoinProviderID(project, zone, *in.Name),
Role: cloudprovider.ExtractRole(metadata),
Role: extractRole(mdata),
PrivateIPs: extractPrivateIPs(in.NetworkInterfaces),
PublicIPs: extractPublicIPs(in.NetworkInterfaces),
AliasIPRanges: extractAliasIPRanges(in.NetworkInterfaces),
SSHKeys: extractSSHKeys(metadata),
SSHKeys: extractSSHKeys(mdata),
}, nil
}

43
coordinator/cloudprovider/gcp/client_test.go
View File

@ -6,9 +6,8 @@ import (
"testing"
compute "cloud.google.com/go/compute/apiv1"
"github.com/edgelesssys/constellation/coordinator/cloudprovider/cloudtypes"
"github.com/edgelesssys/constellation/coordinator/core"
"github.com/edgelesssys/constellation/coordinator/role"
"github.com/edgelesssys/constellation/internal/cloud/metadata"
gax "github.com/googleapis/gax-go/v2"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
@ -44,11 +43,11 @@ func TestRetrieveInstances(t *testing.T) {
Value: proto.String("value-2"),
},
{
Key: proto.String(core.ConstellationUIDMetadataKey),
Key: proto.String(constellationUIDMetadataKey),
Value: proto.String(uid),
},
{
Key: proto.String(core.RoleMetadataKey),
Key: proto.String(roleMetadataKey),
Value: proto.String(role.Coordinator.String()),
},
},
@ -70,14 +69,14 @@ func TestRetrieveInstances(t *testing.T) {
metadata stubMetadataClient
instanceIter *stubInstanceIterator
instanceIterMutator func(*stubInstanceIterator)
wantInstances []cloudtypes.Instance
wantInstances []metadata.InstanceMetadata
wantErr bool
}{
"retrieve works": {
client: stubInstancesClient{},
metadata: stubMetadataClient{InstanceValue: uid},
instanceIter: newTestIter(),
wantInstances: []cloudtypes.Instance{
wantInstances: []metadata.InstanceMetadata{
{
Name: "someInstance",
ProviderID: "gce://someProject/someZone/someInstance",
@ -101,7 +100,7 @@ func TestRetrieveInstances(t *testing.T) {
metadata: stubMetadataClient{InstanceValue: uid},
instanceIter: newTestIter(),
instanceIterMutator: func(sii *stubInstanceIterator) { sii.instances[0].NetworkInterfaces = nil },
wantInstances: []cloudtypes.Instance{
wantInstances: []metadata.InstanceMetadata{
{
Name: "someInstance",
ProviderID: "gce://someProject/someZone/someInstance",
@ -118,7 +117,7 @@ func TestRetrieveInstances(t *testing.T) {
metadata: stubMetadataClient{InstanceValue: uid},
instanceIter: newTestIter(),
instanceIterMutator: func(sii *stubInstanceIterator) { sii.instances[0].NetworkInterfaces[0].NetworkIP = nil },
wantInstances: []cloudtypes.Instance{
wantInstances: []metadata.InstanceMetadata{
{
Name: "someInstance",
ProviderID: "gce://someProject/someZone/someInstance",
@ -135,7 +134,7 @@ func TestRetrieveInstances(t *testing.T) {
metadata: stubMetadataClient{InstanceValue: uid},
instanceIter: newTestIter(),
instanceIterMutator: func(sii *stubInstanceIterator) { sii.instances[0].Metadata.Items[2].Key = proto.String("") },
wantInstances: []cloudtypes.Instance{},
wantInstances: []metadata.InstanceMetadata{},
},
"constellation retrieval fails": {
client: stubInstancesClient{},
@ -148,7 +147,7 @@ func TestRetrieveInstances(t *testing.T) {
metadata: stubMetadataClient{InstanceValue: uid},
instanceIter: newTestIter(),
instanceIterMutator: func(sii *stubInstanceIterator) { sii.instances[0].Metadata.Items[3].Key = proto.String("") },
wantInstances: []cloudtypes.Instance{
wantInstances: []metadata.InstanceMetadata{
{
Name: "someInstance",
ProviderID: "gce://someProject/someZone/someInstance",
@ -224,13 +223,13 @@ func TestRetrieveInstance(t *testing.T) {
client stubInstancesClient
clientInstance *computepb.Instance
clientInstanceMutator func(*computepb.Instance)
wantInstance cloudtypes.Instance
wantInstance metadata.InstanceMetadata
wantErr bool
}{
"retrieve works": {
client: stubInstancesClient{},
clientInstance: newTestInstance(),
wantInstance: cloudtypes.Instance{
wantInstance: metadata.InstanceMetadata{
Name: "someInstance",
ProviderID: "gce://someProject/someZone/someInstance",
AliasIPRanges: []string{"192.0.2.0/16"},
@ -246,7 +245,7 @@ func TestRetrieveInstance(t *testing.T) {
i.Metadata.Items[0].Key = proto.String("ssh-keys")
i.Metadata.Items[0].Value = proto.String("bob:ssh-rsa bobskey")
},
wantInstance: cloudtypes.Instance{
wantInstance: metadata.InstanceMetadata{
Name: "someInstance",
ProviderID: "gce://someProject/someZone/someInstance",
AliasIPRanges: []string{"192.0.2.0/16"},
@ -259,10 +258,10 @@ func TestRetrieveInstance(t *testing.T) {
client: stubInstancesClient{},
clientInstance: newTestInstance(),
clientInstanceMutator: func(i *computepb.Instance) {
i.Metadata.Items[0].Key = proto.String(core.RoleMetadataKey)
i.Metadata.Items[0].Key = proto.String(roleMetadataKey)
i.Metadata.Items[0].Value = proto.String(role.Coordinator.String())
},
wantInstance: cloudtypes.Instance{
wantInstance: metadata.InstanceMetadata{
Name: "someInstance",
ProviderID: "gce://someProject/someZone/someInstance",
AliasIPRanges: []string{"192.0.2.0/16"},
@ -283,7 +282,7 @@ func TestRetrieveInstance(t *testing.T) {
client: stubInstancesClient{},
clientInstance: newTestInstance(),
clientInstanceMutator: func(i *computepb.Instance) { i.Metadata.Items[0] = nil },
wantInstance: cloudtypes.Instance{
wantInstance: metadata.InstanceMetadata{
Name: "someInstance",
ProviderID: "gce://someProject/someZone/someInstance",
AliasIPRanges: []string{"192.0.2.0/16"},
@ -296,7 +295,7 @@ func TestRetrieveInstance(t *testing.T) {
client: stubInstancesClient{},
clientInstance: newTestInstance(),
clientInstanceMutator: func(i *computepb.Instance) { i.Metadata.Items[0].Key = nil },
wantInstance: cloudtypes.Instance{
wantInstance: metadata.InstanceMetadata{
Name: "someInstance",
ProviderID: "gce://someProject/someZone/someInstance",
AliasIPRanges: []string{"192.0.2.0/16"},
@ -309,7 +308,7 @@ func TestRetrieveInstance(t *testing.T) {
client: stubInstancesClient{},
clientInstance: newTestInstance(),
clientInstanceMutator: func(i *computepb.Instance) { i.Metadata.Items[0].Value = nil },
wantInstance: cloudtypes.Instance{
wantInstance: metadata.InstanceMetadata{
Name: "someInstance",
ProviderID: "gce://someProject/someZone/someInstance",
AliasIPRanges: []string{"192.0.2.0/16"},
@ -322,7 +321,7 @@ func TestRetrieveInstance(t *testing.T) {
client: stubInstancesClient{},
clientInstance: newTestInstance(),
clientInstanceMutator: func(i *computepb.Instance) { i.NetworkInterfaces[0] = nil },
wantInstance: cloudtypes.Instance{
wantInstance: metadata.InstanceMetadata{
Name: "someInstance",
ProviderID: "gce://someProject/someZone/someInstance",
AliasIPRanges: []string{},
@ -335,7 +334,7 @@ func TestRetrieveInstance(t *testing.T) {
client: stubInstancesClient{},
clientInstance: newTestInstance(),
clientInstanceMutator: func(i *computepb.Instance) { i.NetworkInterfaces[0].NetworkIP = nil },
wantInstance: cloudtypes.Instance{
wantInstance: metadata.InstanceMetadata{
Name: "someInstance",
ProviderID: "gce://someProject/someZone/someInstance",
AliasIPRanges: []string{"192.0.2.0/16"},
@ -348,7 +347,7 @@ func TestRetrieveInstance(t *testing.T) {
client: stubInstancesClient{},
clientInstance: newTestInstance(),
clientInstanceMutator: func(i *computepb.Instance) { i.NetworkInterfaces[0].AliasIpRanges[0].IpCidrRange = nil },
wantInstance: cloudtypes.Instance{
wantInstance: metadata.InstanceMetadata{
Name: "someInstance",
ProviderID: "gce://someProject/someZone/someInstance",
AliasIPRanges: []string{},
@ -361,7 +360,7 @@ func TestRetrieveInstance(t *testing.T) {
client: stubInstancesClient{},
clientInstance: newTestInstance(),
clientInstanceMutator: func(i *computepb.Instance) { i.NetworkInterfaces[0].AccessConfigs[0].NatIP = nil },
wantInstance: cloudtypes.Instance{
wantInstance: metadata.InstanceMetadata{
Name: "someInstance",
ProviderID: "gce://someProject/someZone/someInstance",
AliasIPRanges: []string{"192.0.2.0/16"},

66
coordinator/cloudprovider/gcp/metadata.go
View File

@ -4,18 +4,16 @@ import (
"context"
"fmt"
"github.com/edgelesssys/constellation/coordinator/cloudprovider/cloudtypes"
"github.com/edgelesssys/constellation/coordinator/core"
"github.com/edgelesssys/constellation/coordinator/role"
"github.com/edgelesssys/constellation/internal/cloud/metadata"
"github.com/edgelesssys/constellation/internal/gcpshared"
)
// API handles all GCP API requests.
type API interface {
// RetrieveInstances retrieves a list of all accessible GCP instances with their metadata.
RetrieveInstances(ctx context.Context, project, zone string) ([]cloudtypes.Instance, error)
RetrieveInstances(ctx context.Context, project, zone string) ([]metadata.InstanceMetadata, error)
// RetrieveInstances retrieves a single GCP instances with its metadata.
RetrieveInstance(ctx context.Context, project, zone, instanceName string) (cloudtypes.Instance, error)
RetrieveInstance(ctx context.Context, project, zone, instanceName string) (metadata.InstanceMetadata, error)
// RetrieveInstanceMetadata retrieves the GCP instance metadata of the current instance.
RetrieveInstanceMetadata(attr string) (string, error)
// RetrieveProjectID retrieves the GCP projectID containing the current instance.
@ -47,7 +45,7 @@ func New(api API) *Metadata {
}
// List retrieves all instances belonging to the current constellation.
func (m *Metadata) List(ctx context.Context) ([]cloudtypes.Instance, error) {
func (m *Metadata) List(ctx context.Context) ([]metadata.InstanceMetadata, error) {
project, err := m.api.RetrieveProjectID()
if err != nil {
return nil, err
@ -64,65 +62,31 @@ func (m *Metadata) List(ctx context.Context) ([]cloudtypes.Instance, error) {
}
// Self retrieves the current instance.
func (m *Metadata) Self(ctx context.Context) (cloudtypes.Instance, error) {
func (m *Metadata) Self(ctx context.Context) (metadata.InstanceMetadata, error) {
project, err := m.api.RetrieveProjectID()
if err != nil {
return cloudtypes.Instance{}, err
return metadata.InstanceMetadata{}, err
}
zone, err := m.api.RetrieveZone()
if err != nil {
return cloudtypes.Instance{}, err
return metadata.InstanceMetadata{}, err
}
instanceName, err := m.api.RetrieveInstanceName()
if err != nil {
return cloudtypes.Instance{}, err
return metadata.InstanceMetadata{}, err
}
return m.api.RetrieveInstance(ctx, project, zone, instanceName)
}
// GetInstance retrieves an instance using its providerID.
func (m *Metadata) GetInstance(ctx context.Context, providerID string) (cloudtypes.Instance, error) {
func (m *Metadata) GetInstance(ctx context.Context, providerID string) (metadata.InstanceMetadata, error) {
project, zone, instanceName, err := gcpshared.SplitProviderID(providerID)
if err != nil {
return cloudtypes.Instance{}, fmt.Errorf("invalid providerID: %w", err)
return metadata.InstanceMetadata{}, fmt.Errorf("invalid providerID: %w", err)
}
return m.api.RetrieveInstance(ctx, project, zone, instanceName)
}
// SignalRole signals the constellation role via cloud provider metadata.
func (m *Metadata) SignalRole(ctx context.Context, role role.Role) error {
project, err := m.api.RetrieveProjectID()
if err != nil {
return err
}
zone, err := m.api.RetrieveZone()
if err != nil {
return err
}
instanceName, err := m.api.RetrieveInstanceName()
if err != nil {
return err
}
return m.api.SetInstanceMetadata(ctx, project, zone, instanceName, core.RoleMetadataKey, role.String())
}
// SetVPNIP stores the internally used VPN IP in cloud provider metadata.
func (m *Metadata) SetVPNIP(ctx context.Context, vpnIP string) error {
project, err := m.api.RetrieveProjectID()
if err != nil {
return err
}
zone, err := m.api.RetrieveZone()
if err != nil {
return err
}
instanceName, err := m.api.RetrieveInstanceName()
if err != nil {
return err
}
return m.api.SetInstanceMetadata(ctx, project, zone, instanceName, core.VPNIPMetadataKey, vpnIP)
}
// GetSubnetworkCIDR returns the subnetwork CIDR of the current instance.
func (m *Metadata) GetSubnetworkCIDR(ctx context.Context) (string, error) {
project, err := m.api.RetrieveProjectID()
@ -140,11 +104,6 @@ func (m *Metadata) GetSubnetworkCIDR(ctx context.Context) (string, error) {
return m.api.RetrieveSubnetworkAliasCIDR(ctx, project, zone, instanceName)
}
// SupportsLoadBalancer returns true if the cloud provider supports load balancers.
func (m *Metadata) SupportsLoadBalancer() bool {
return true
}
// GetLoadBalancerIP returns the IP of the load balancer.
func (m *Metadata) GetLoadBalancerIP(ctx context.Context) (string, error) {
project, err := m.api.RetrieveProjectID()
@ -157,8 +116,3 @@ func (m *Metadata) GetLoadBalancerIP(ctx context.Context) (string, error) {
}
return m.api.RetrieveLoadBalancerIP(ctx, project, zone)
}
// Supported is used to determine if metadata API is implemented for this cloud provider.
func (m *Metadata) Supported() bool {
return true
}

161
coordinator/cloudprovider/gcp/metadata_test.go
View File

@ -5,9 +5,7 @@ import (
"errors"
"testing"
"github.com/edgelesssys/constellation/coordinator/cloudprovider/cloudtypes"
"github.com/edgelesssys/constellation/coordinator/core"
"github.com/edgelesssys/constellation/coordinator/role"
"github.com/edgelesssys/constellation/internal/cloud/metadata"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
@ -15,8 +13,8 @@ import (
func TestList(t *testing.T) {
err := errors.New("some err")
uid := "1234"
instancesGenerator := func() *[]cloudtypes.Instance {
return &[]cloudtypes.Instance{
instancesGenerator := func() *[]metadata.InstanceMetadata {
return &[]metadata.InstanceMetadata{
{
Name: "someInstance",
ProviderID: "gce://someProject/someZone/someInstance",
@ -27,10 +25,10 @@ func TestList(t *testing.T) {
testCases := map[string]struct {
client stubGCPClient
instancesGenerator func() *[]cloudtypes.Instance
instancesMutator func(*[]cloudtypes.Instance)
instancesGenerator func() *[]metadata.InstanceMetadata
instancesMutator func(*[]metadata.InstanceMetadata)
wantErr bool
wantInstances []cloudtypes.Instance
wantInstances []metadata.InstanceMetadata
}{
"retrieve works": {
client: stubGCPClient{
@ -41,7 +39,7 @@ func TestList(t *testing.T) {
},
},
instancesGenerator: instancesGenerator,
wantInstances: []cloudtypes.Instance{
wantInstances: []metadata.InstanceMetadata{
{
Name: "someInstance",
ProviderID: "gce://someProject/someZone/someInstance",
@ -106,19 +104,19 @@ func TestSelf(t *testing.T) {
testCases := map[string]struct {
client stubGCPClient
wantErr bool
wantInstance cloudtypes.Instance
wantInstance metadata.InstanceMetadata
}{
"retrieve works": {
client: stubGCPClient{
projectID: "someProjectID",
zone: "someZone",
retrieveInstanceValue: cloudtypes.Instance{
retrieveInstanceValue: metadata.InstanceMetadata{
Name: "someInstance",
ProviderID: "gce://someProject/someZone/someInstance",
PrivateIPs: []string{"192.0.2.0"},
},
},
wantInstance: cloudtypes.Instance{
wantInstance: metadata.InstanceMetadata{
Name: "someInstance",
ProviderID: "gce://someProject/someZone/someInstance",
PrivateIPs: []string{"192.0.2.0"},
@ -180,18 +178,18 @@ func TestGetInstance(t *testing.T) {
providerID string
client stubGCPClient
wantErr bool
wantInstance cloudtypes.Instance
wantInstance metadata.InstanceMetadata
}{
"retrieve works": {
providerID: "gce://someProject/someZone/someInstance",
client: stubGCPClient{
retrieveInstanceValue: cloudtypes.Instance{
retrieveInstanceValue: metadata.InstanceMetadata{
Name: "someInstance",
ProviderID: "gce://someProject/someZone/someInstance",
PrivateIPs: []string{"192.0.2.0"},
},
},
wantInstance: cloudtypes.Instance{
wantInstance: metadata.InstanceMetadata{
Name: "someInstance",
ProviderID: "gce://someProject/someZone/someInstance",
PrivateIPs: []string{"192.0.2.0"},
@ -232,135 +230,10 @@ func TestGetInstance(t *testing.T) {
}
}
func TestSignalRole(t *testing.T) {
err := errors.New("some err")
testCases := map[string]struct {
client stubGCPClient
wantErr bool
wantRole role.Role
}{
"signaling role works": {
client: stubGCPClient{
projectID: "someProjectID",
zone: "someZone",
instanceName: "someName",
},
wantRole: role.Coordinator,
},
"project metadata retrieval error is detected": {
client: stubGCPClient{
retrieveProjectIDErr: err,
},
wantErr: true,
},
"instance zone retrieval error is detected": {
client: stubGCPClient{
retrieveZoneErr: err,
},
wantErr: true,
},
"instance name retrieval error is detected": {
client: stubGCPClient{
retrieveInstanceNameErr: err,
},
wantErr: true,
},
}
for name, tc := range testCases {
t.Run(name, func(t *testing.T) {
assert := assert.New(t)
require := require.New(t)
cloud := New(&tc.client)
err := cloud.SignalRole(context.Background(), tc.wantRole)
if tc.wantErr {
assert.Error(err)
return
}
require.NoError(err)
assert.ElementsMatch([]string{"someProjectID"}, tc.client.instanceMetadataProjects)
assert.ElementsMatch([]string{"someZone"}, tc.client.instanceMetadataZones)
assert.ElementsMatch([]string{"someName"}, tc.client.instanceMetadataInstanceNames)
assert.ElementsMatch([]string{core.RoleMetadataKey}, tc.client.instanceMetadataKeys)
assert.ElementsMatch([]string{tc.wantRole.String()}, tc.client.instanceMetadataValues)
})
}
}
func TestSetVPNIP(t *testing.T) {
err := errors.New("some err")
testCases := map[string]struct {
client stubGCPClient
wantErr bool
wantVPNIP string
}{
"signaling role works": {
client: stubGCPClient{
projectID: "someProjectID",
zone: "someZone",
instanceName: "someName",
},
wantVPNIP: "192.0.2.0",
},
"project metadata retrieval error is detected": {
client: stubGCPClient{
retrieveProjectIDErr: err,
},
wantErr: true,
},
"instance zone retrieval error is detected": {
client: stubGCPClient{
retrieveZoneErr: err,
},
wantErr: true,
},
"instance name retrieval error is detected": {
client: stubGCPClient{
retrieveInstanceNameErr: err,
},
wantErr: true,
},
}
for name, tc := range testCases {
t.Run(name, func(t *testing.T) {
assert := assert.New(t)
require := require.New(t)
cloud := New(&tc.client)
err := cloud.SetVPNIP(context.Background(), tc.wantVPNIP)
if tc.wantErr {
assert.Error(err)
return
}
require.NoError(err)
assert.ElementsMatch([]string{"someProjectID"}, tc.client.instanceMetadataProjects)
assert.ElementsMatch([]string{"someZone"}, tc.client.instanceMetadataZones)
assert.ElementsMatch([]string{"someName"}, tc.client.instanceMetadataInstanceNames)
assert.ElementsMatch([]string{core.VPNIPMetadataKey}, tc.client.instanceMetadataKeys)
assert.ElementsMatch([]string{tc.wantVPNIP}, tc.client.instanceMetadataValues)
})
}
}
func TestTrivialMetadataFunctions(t *testing.T) {
assert := assert.New(t)
metadata := Metadata{}
assert.True(metadata.Supported())
}
type stubGCPClient struct {
retrieveInstanceValue cloudtypes.Instance
retrieveInstanceValue metadata.InstanceMetadata
retrieveInstanceErr error
retrieveInstancesValues []cloudtypes.Instance
retrieveInstancesValues []metadata.InstanceMetadata
retrieveInstancesErr error
retrieveInstanceMetadaValues map[string]string
retrieveInstanceMetadataErr error
@ -388,11 +261,11 @@ type stubGCPClient struct {
unsetMetadataKeys []string
}
func (s *stubGCPClient) RetrieveInstances(ctx context.Context, project, zone string) ([]cloudtypes.Instance, error) {
func (s *stubGCPClient) RetrieveInstances(ctx context.Context, project, zone string) ([]metadata.InstanceMetadata, error) {
return s.retrieveInstancesValues, s.retrieveInstancesErr
}
func (s *stubGCPClient) RetrieveInstance(ctx context.Context, project, zone string, instanceName string) (cloudtypes.Instance, error) {
func (s *stubGCPClient) RetrieveInstance(ctx context.Context, project, zone string, instanceName string) (metadata.InstanceMetadata, error) {
return s.retrieveInstanceValue, s.retrieveInstanceErr
}

19
coordinator/cloudprovider/gcp/role.go Normal file
View File

@ -0,0 +1,19 @@
package gcp
import (
"github.com/edgelesssys/constellation/coordinator/role"
)
const roleMetadataKey = "constellation-role"
// extractRole extracts role from cloud provider metadata.
func extractRole(metadata map[string]string) role.Role {
switch metadata[roleMetadataKey] {
case role.Coordinator.String():
return role.Coordinator
case role.Node.String():
return role.Node
default:
return role.Unknown
}
}

11
coordinator/cloudprovider/role_test.go → coordinator/cloudprovider/gcp/role_test.go
View File

@ -1,9 +1,8 @@
package cloudprovider
package gcp
import (
"testing"
"github.com/edgelesssys/constellation/coordinator/core"
"github.com/edgelesssys/constellation/coordinator/role"
"github.com/stretchr/testify/assert"
"go.uber.org/goleak"
@ -23,19 +22,19 @@ func TestExtractRole(t *testing.T) {
}{
"coordinator role": {
metadata: map[string]string{
core.RoleMetadataKey: role.Coordinator.String(),
roleMetadataKey: role.Coordinator.String(),
},
wantRole: role.Coordinator,
},
"node role": {
metadata: map[string]string{
core.RoleMetadataKey: role.Node.String(),
roleMetadataKey: role.Node.String(),
},
wantRole: role.Node,
},
"unknown role": {
metadata: map[string]string{
core.RoleMetadataKey: "some-unknown-role",
roleMetadataKey: "some-unknown-role",
},
wantRole: role.Unknown,
},
@ -48,7 +47,7 @@ func TestExtractRole(t *testing.T) {
t.Run(name, func(t *testing.T) {
assert := assert.New(t)
role := ExtractRole(tc.metadata)
role := extractRole(tc.metadata)
assert.Equal(tc.wantRole, role)
})

4
coordinator/cloudprovider/qemu/autoscaler.go
View File

@ -1,8 +1,8 @@
package qemu
import (
"github.com/edgelesssys/constellation/coordinator/cloudprovider/cloudtypes"
"github.com/edgelesssys/constellation/coordinator/kubernetes/k8sapi/resources"
"github.com/edgelesssys/constellation/internal/cloud/metadata"
k8s "k8s.io/api/core/v1"
)
@ -15,7 +15,7 @@ func (a Autoscaler) Name() string {
}
// Secrets returns a list of secrets to deploy together with the k8s cluster-autoscaler.
func (a Autoscaler) Secrets(instance cloudtypes.Instance, cloudServiceAccountURI string) (resources.Secrets, error) {
func (a Autoscaler) Secrets(instance metadata.InstanceMetadata, cloudServiceAccountURI string) (resources.Secrets, error) {
return resources.Secrets{}, nil
}

8
coordinator/cloudprovider/qemu/ccm.go
View File

@ -3,8 +3,8 @@ package qemu
import (
"context"
"github.com/edgelesssys/constellation/coordinator/cloudprovider/cloudtypes"
"github.com/edgelesssys/constellation/coordinator/kubernetes/k8sapi/resources"
"github.com/edgelesssys/constellation/internal/cloud/metadata"
k8s "k8s.io/api/core/v1"
)
@ -33,13 +33,13 @@ func (c CloudControllerManager) ExtraArgs() []string {
// ConfigMaps returns a list of ConfigMaps to deploy together with the k8s cloud-controller-manager
// Reference: https://kubernetes.io/docs/concepts/configuration/configmap/ .
func (c CloudControllerManager) ConfigMaps(instance cloudtypes.Instance) (resources.ConfigMaps, error) {
func (c CloudControllerManager) ConfigMaps(instance metadata.InstanceMetadata) (resources.ConfigMaps, error) {
return resources.ConfigMaps{}, nil
}
// Secrets returns a list of secrets to deploy together with the k8s cloud-controller-manager.
// Reference: https://kubernetes.io/docs/concepts/configuration/secret/ .
func (c CloudControllerManager) Secrets(ctx context.Context, instance cloudtypes.Instance, cloudServiceAccountURI string) (resources.Secrets, error) {
func (c CloudControllerManager) Secrets(ctx context.Context, instance metadata.InstanceMetadata, cloudServiceAccountURI string) (resources.Secrets, error) {
return resources.Secrets{}, nil
}
@ -61,7 +61,7 @@ func (c CloudControllerManager) Env() []k8s.EnvVar {
// PrepareInstance is called on every instance before deploying the cloud-controller-manager.
// Allows for cloud-provider specific hooks.
func (c CloudControllerManager) PrepareInstance(instance cloudtypes.Instance, vpnIP string) error {
func (c CloudControllerManager) PrepareInstance(instance metadata.InstanceMetadata, vpnIP string) error {
// no specific hook required.
return nil
}

18
coordinator/cloudprovider/qemu/metadata.go
View File

@ -8,8 +8,8 @@ import (
"net/http"
"net/url"
"github.com/edgelesssys/constellation/coordinator/cloudprovider/cloudtypes"
"github.com/edgelesssys/constellation/coordinator/role"
"github.com/edgelesssys/constellation/internal/cloud/metadata"
)
const qemuMetadataEndpoint = "10.42.0.1:8080"
@ -23,34 +23,34 @@ func (m *Metadata) Supported() bool {
}
// List retrieves all instances belonging to the current constellation.
func (m *Metadata) List(ctx context.Context) ([]cloudtypes.Instance, error) {
func (m *Metadata) List(ctx context.Context) ([]metadata.InstanceMetadata, error) {
instancesRaw, err := m.retrieveMetadata(ctx, "/peers")
if err != nil {
return nil, err
}
var instances []cloudtypes.Instance
var instances []metadata.InstanceMetadata
err = json.Unmarshal(instancesRaw, &instances)
return instances, err
}
// Self retrieves the current instance.
func (m *Metadata) Self(ctx context.Context) (cloudtypes.Instance, error) {
func (m *Metadata) Self(ctx context.Context) (metadata.InstanceMetadata, error) {
instanceRaw, err := m.retrieveMetadata(ctx, "/self")
if err != nil {
return cloudtypes.Instance{}, err
return metadata.InstanceMetadata{}, err
}
var instance cloudtypes.Instance
var instance metadata.InstanceMetadata
err = json.Unmarshal(instanceRaw, &instance)
return instance, err
}
// GetInstance retrieves an instance using its providerID.
func (m Metadata) GetInstance(ctx context.Context, providerID string) (cloudtypes.Instance, error) {
func (m Metadata) GetInstance(ctx context.Context, providerID string) (metadata.InstanceMetadata, error) {
instances, err := m.List(ctx)
if err != nil {
return cloudtypes.Instance{}, err
return metadata.InstanceMetadata{}, err
}
for _, instance := range instances {
@ -58,7 +58,7 @@ func (m Metadata) GetInstance(ctx context.Context, providerID string) (cloudtype
return instance, nil
}
}
return cloudtypes.Instance{}, errors.New("instance not found")
return metadata.InstanceMetadata{}, errors.New("instance not found")
}
// SignalRole signals the constellation role via cloud provider metadata (if supported by the CSP and deployment type, otherwise does nothing).

18
coordinator/cloudprovider/role.go
View File

@ -1,18 +0,0 @@
package cloudprovider
import (
"github.com/edgelesssys/constellation/coordinator/core"
"github.com/edgelesssys/constellation/coordinator/role"
)
// ExtractRole extracts role from cloud provider metadata.
func ExtractRole(metadata map[string]string) role.Role {
switch metadata[core.RoleMetadataKey] {
case role.Coordinator.String():
return role.Coordinator
case role.Node.String():
return role.Node
default:
return role.Unknown
}
}

4
coordinator/kubernetes/kubernetes.go
View File

@ -282,7 +282,7 @@ func (k *KubeWrapper) setupCCM(ctx context.Context, subnetworkPodCIDR, cloudServ
if err != nil {
return fmt.Errorf("defining ConfigMaps for CCM failed: %w", err)
}
ccmSecrets, err := k.cloudControllerManager.Secrets(ctx, instance, cloudServiceAccountURI)
ccmSecrets, err := k.cloudControllerManager.Secrets(ctx, instance.ProviderID, cloudServiceAccountURI)
if err != nil {
return fmt.Errorf("defining Secrets for CCM failed: %w", err)
}
@ -316,7 +316,7 @@ func (k *KubeWrapper) setupClusterAutoscaler(instance cloudtypes.Instance, cloud
if !k.clusterAutoscaler.Supported() {
return nil
}
caSecrets, err := k.clusterAutoscaler.Secrets(instance, cloudServiceAccountURI)
caSecrets, err := k.clusterAutoscaler.Secrets(instance.ProviderID, cloudServiceAccountURI)
if err != nil {
return fmt.Errorf("defining Secrets for cluster-autoscaler failed: %w", err)
}

22
internal/azureshared/metadata.go
View File

@ -7,17 +7,10 @@ import (
"strings"
)
var (
azureVMProviderIDRegexp = regexp.MustCompile(`^azure:///subscriptions/([^/]+)/resourceGroups/([^/]+)/providers/Microsoft.Compute/virtualMachines/([^/]+)$`)
azureVMSSProviderIDRegexp = regexp.MustCompile(`^azure:///subscriptions/([^/]+)/resourceGroups/([^/]+)/providers/Microsoft.Compute/virtualMachineScaleSets/([^/]+)/virtualMachines/([^/]+)$`)
)
var azureVMSSProviderIDRegexp = regexp.MustCompile(`^azure:///subscriptions/([^/]+)/resourceGroups/([^/]+)/providers/Microsoft.Compute/virtualMachineScaleSets/([^/]+)/virtualMachines/([^/]+)$`)
// BasicsFromProviderID extracts subscriptionID and resourceGroup from both types of valid azure providerID.
func BasicsFromProviderID(providerID string) (subscriptionID, resourceGroup string, err error) {
subscriptionID, resourceGroup, _, err = VMInformationFromProviderID(providerID)
if err == nil {
return subscriptionID, resourceGroup, nil
}
subscriptionID, resourceGroup, _, _, err = ScaleSetInformationFromProviderID(providerID)
if err == nil {
return subscriptionID, resourceGroup, nil
@ -29,7 +22,7 @@ func BasicsFromProviderID(providerID string) (subscriptionID, resourceGroup stri
// suffix at the resource group, e.g., resource-group-J18dB
// J18dB is the UID.
func UIDFromProviderID(providerID string) (string, error) {
_, resourceGroup, err := BasicsFromProviderID(providerID)
_, resourceGroup, _, _, err := ScaleSetInformationFromProviderID(providerID)
if err != nil {
return "", err
}
@ -38,17 +31,6 @@ func UIDFromProviderID(providerID string) (string, error) {
return parts[len(parts)-1], nil
}
// VMInformationFromProviderID splits a provider's id belonging to a single azure instance into core components.
// A providerID for individual VMs is build after the following schema:
// - 'azure:///subscriptions/<subscription-id>/resourceGroups/<resource-group>/providers/Microsoft.Compute/virtualMachines/<instance-name>'
func VMInformationFromProviderID(providerID string) (subscriptionID, resourceGroup, instanceName string, err error) {
matches := azureVMProviderIDRegexp.FindStringSubmatch(providerID)
if len(matches) != 4 {
return "", "", "", errors.New("error splitting providerID")
}
return matches[1], matches[2], matches[3], nil
}
// ScaleSetInformationFromProviderID splits a provider's id belonging to an azure scaleset into core components.
// A providerID for scale set VMs is build after the following schema:
// - 'azure:///subscriptions/<subscription-id>/resourceGroups/<resource-group>/providers/Microsoft.Compute/virtualMachineScaleSets/<scale-set-name>/virtualMachines/<instance-id>'

25
internal/cloud/metadata/metadata.go
View File

@ -2,7 +2,6 @@ package metadata
import (
"context"
"errors"
"fmt"
"net"
"strconv"
@ -36,24 +35,30 @@ type metadataAPI interface {
Supported() bool
}
// TODO(katexochen): Rename to InitEndpoints
func CoordinatorEndpoints(ctx context.Context, api metadataAPI) ([]string, error) {
if !api.Supported() {
return nil, errors.New("retrieving instances list from cloud provider is not yet supported")
}
instances, err := api.List(ctx)
type InstanceSelfer interface {
// Self retrieves the current instance.
Self(ctx context.Context) (InstanceMetadata, error)
}
type InstanceLister interface {
// List retrieves all instances belonging to the current constellation.
List(ctx context.Context) ([]InstanceMetadata, error)
}
func InitServerEndpoints(ctx context.Context, lister InstanceLister) ([]string, error) {
instances, err := lister.List(ctx)
if err != nil {
return nil, fmt.Errorf("retrieving instances list from cloud provider: %w", err)
}
coordinatorEndpoints := []string{}
initServerEndpoints := []string{}
for _, instance := range instances {
// check if role of instance is "Coordinator"
if instance.Role == role.Coordinator {
for _, ip := range instance.PrivateIPs {
coordinatorEndpoints = append(coordinatorEndpoints, net.JoinHostPort(ip, strconv.Itoa(constants.CoordinatorPort)))
initServerEndpoints = append(initServerEndpoints, net.JoinHostPort(ip, strconv.Itoa(constants.CoordinatorPort)))
}
}
}
return coordinatorEndpoints, nil
return initServerEndpoints, nil
}

1
internal/constants/constants.go
View File

@ -50,7 +50,6 @@ const (
//
// Filenames.
//
StateFilename = "constellation-state.json"
ClusterIDsFileName = "constellation-id.json"
ConfigFilename = "constellation-conf.yaml"