diff --git a/coordinator/cloudprovider/aws/autoscaler.go b/coordinator/cloudprovider/aws/autoscaler.go deleted file mode 100644 index 3b9d65b1a..000000000 --- a/coordinator/cloudprovider/aws/autoscaler.go +++ /dev/null @@ -1,40 +0,0 @@ -package aws - -import ( - "github.com/edgelesssys/constellation/coordinator/cloudprovider/cloudtypes" - "github.com/edgelesssys/constellation/coordinator/kubernetes/k8sapi/resources" - k8s "k8s.io/api/core/v1" -) - -// Autoscaler holds the AWS cluster-autoscaler configuration. -type Autoscaler struct{} - -// Name returns the cloud-provider name as used by k8s cluster-autoscaler. -func (a Autoscaler) Name() string { - return "aws" -} - -// Secrets returns a list of secrets to deploy together with the k8s cluster-autoscaler. -func (a Autoscaler) Secrets(instance cloudtypes.Instance, cloudServiceAccountURI string) (resources.Secrets, error) { - return resources.Secrets{}, nil -} - -// Volumes returns a list of volumes to deploy together with the k8s cluster-autoscaler. -func (a Autoscaler) Volumes() []k8s.Volume { - return []k8s.Volume{} -} - -// VolumeMounts returns a list of volume mounts to deploy together with the k8s cluster-autoscaler. -func (a Autoscaler) VolumeMounts() []k8s.VolumeMount { - return []k8s.VolumeMount{} -} - -// Env returns a list of k8s environment key-value pairs to deploy together with the k8s cluster-autoscaler. -func (a Autoscaler) Env() []k8s.EnvVar { - return []k8s.EnvVar{} -} - -// Supported is used to determine if we support autoscaling for the cloud provider. -func (a Autoscaler) Supported() bool { - return false -} diff --git a/coordinator/cloudprovider/aws/ccm.go b/coordinator/cloudprovider/aws/ccm.go deleted file mode 100644 index 036288955..000000000 --- a/coordinator/cloudprovider/aws/ccm.go +++ /dev/null @@ -1,66 +0,0 @@ -package aws - -import ( - "context" - - "github.com/edgelesssys/constellation/coordinator/cloudprovider" - "github.com/edgelesssys/constellation/coordinator/cloudprovider/cloudtypes" - "github.com/edgelesssys/constellation/coordinator/kubernetes/k8sapi/resources" - k8s "k8s.io/api/core/v1" -) - -// CloudControllerManager holds the AWS cloud-controller-manager configuration. -type CloudControllerManager struct{} - -// Image returns the container image used to provide cloud-controller-manager for the cloud-provider. -func (c CloudControllerManager) Image() string { - return cloudprovider.CloudControllerManagerImageAWS -} - -// Path returns the path used by cloud-controller-manager executable within the container image. -func (c CloudControllerManager) Path() string { - return "/aws-cloud-controller-manager" -} - -// Name returns the cloud-provider name as used by k8s cloud-controller-manager (k8s.gcr.io/cloud-controller-manager). -func (c CloudControllerManager) Name() string { - return "aws" -} - -// ExtraArgs returns a list of arguments to append to the cloud-controller-manager command. -func (c CloudControllerManager) ExtraArgs() []string { - return []string{} -} - -// ConfigMaps returns a list of ConfigMaps to deploy together with the k8s cloud-controller-manager -// Reference: https://kubernetes.io/docs/concepts/configuration/configmap/ . -func (c CloudControllerManager) ConfigMaps(instance cloudtypes.Instance) (resources.ConfigMaps, error) { - return resources.ConfigMaps{}, nil -} - -// Secrets returns a list of secrets to deploy together with the k8s cloud-controller-manager. -// Reference: https://kubernetes.io/docs/concepts/configuration/secret/ . -func (c CloudControllerManager) Secrets(ctx context.Context, instance cloudtypes.Instance, cloudServiceAccountURI string) (resources.Secrets, error) { - return resources.Secrets{}, nil -} - -// Volumes returns a list of volumes to deploy together with the k8s cloud-controller-manager. -// Reference: https://kubernetes.io/docs/concepts/storage/volumes/ . -func (c CloudControllerManager) Volumes() []k8s.Volume { - return []k8s.Volume{} -} - -// VolumeMounts a list of of volume mounts to deploy together with the k8s cloud-controller-manager. -func (c CloudControllerManager) VolumeMounts() []k8s.VolumeMount { - return []k8s.VolumeMount{} -} - -// Env returns a list of k8s environment key-value pairs to deploy together with the k8s cloud-controller-manager. -func (c CloudControllerManager) Env() []k8s.EnvVar { - return []k8s.EnvVar{} -} - -// Supported is used to determine if cloud controller manager is implemented for this cloud provider. -func (c CloudControllerManager) Supported() bool { - return false -} diff --git a/coordinator/cloudprovider/aws/cloudnodemanager.go b/coordinator/cloudprovider/aws/cloudnodemanager.go deleted file mode 100644 index f62dc508f..000000000 --- a/coordinator/cloudprovider/aws/cloudnodemanager.go +++ /dev/null @@ -1,27 +0,0 @@ -package aws - -// CloudNodeManager holds the AWS cloud-node-manager configuration. -type CloudNodeManager struct{} - -// Image returns the container image used to provide cloud-node-manager for the cloud-provider. -// Not used on AWS. -func (c *CloudNodeManager) Image() string { - return "" -} - -// Path returns the path used by cloud-node-manager executable within the container image. -// Not used on AWS. -func (c *CloudNodeManager) Path() string { - return "" -} - -// ExtraArgs returns a list of arguments to append to the cloud-node-manager command. -// Not used on AWS. -func (c *CloudNodeManager) ExtraArgs() []string { - return []string{} -} - -// Supported is used to determine if cloud node manager is implemented for this cloud provider. -func (c *CloudNodeManager) Supported() bool { - return false -} diff --git a/coordinator/cloudprovider/aws/metadata.go b/coordinator/cloudprovider/aws/metadata.go deleted file mode 100644 index 29132ef57..000000000 --- a/coordinator/cloudprovider/aws/metadata.go +++ /dev/null @@ -1,79 +0,0 @@ -package aws - -import ( - "context" - "fmt" - - "github.com/aws/aws-sdk-go-v2/config" - "github.com/aws/aws-sdk-go-v2/feature/ec2/imds" - "github.com/edgelesssys/constellation/coordinator/cloudprovider/cloudtypes" - "github.com/edgelesssys/constellation/coordinator/role" -) - -// Metadata implements core.ProviderMetadata interface. -type Metadata struct{} - -// List retrieves all instances belonging to the current constellation. -func (m Metadata) List(ctx context.Context) ([]cloudtypes.Instance, error) { - // TODO: implement using https://pkg.go.dev/github.com/aws/aws-sdk-go-v2/service/ec2#Client.DescribeInstances - // And using AWS ec2 instance tags - panic("function *Metadata.List not implemented") -} - -// Self retrieves the current instance. -func (m Metadata) Self(ctx context.Context) (cloudtypes.Instance, error) { - identityDocument, err := retrieveIdentityDocument(ctx) - if err != nil { - return cloudtypes.Instance{}, err - } - // TODO: implement metadata using AWS ec2 instance tags - return cloudtypes.Instance{ - Name: identityDocument.InstanceID, - ProviderID: providerID(identityDocument), - PrivateIPs: []string{ - identityDocument.PrivateIP, - }, - }, nil -} - -// GetInstance retrieves an instance using its providerID. -func (m Metadata) GetInstance(ctx context.Context, providerID string) (cloudtypes.Instance, error) { - // TODO: implement using https://pkg.go.dev/github.com/aws/aws-sdk-go-v2/service/ec2#DescribeInstancesAPIClient.DescribeInstances - // And using AWS ec2 instance tags - // Filter request to only return info on this instance - panic("function *Metadata.GetInstance not implemented") -} - -// SignalRole signals the constellation role via cloud provider metadata (if supported by the CSP and deployment type, otherwise does nothing). -func (m Metadata) SignalRole(ctx context.Context, role role.Role) error { - panic("function *Metadata.SignalRole not implemented") -} - -// SetVPNIP stores the internally used VPN IP in cloud provider metadata (if supported and required for autoscaling by the CSP, otherwise does nothing). -func (m Metadata) SetVPNIP(ctx context.Context, vpnIP string) error { - panic("function *Metadata.SetVPNIP not implemented") -} - -// Supported is used to determine if metadata API is implemented for this cloud provider. -func (m Metadata) Supported() bool { - return false -} - -// retrieveIdentityDocument retrieves an AWS instance identity document. -func retrieveIdentityDocument(ctx context.Context) (*imds.GetInstanceIdentityDocumentOutput, error) { - cfg, err := config.LoadDefaultConfig(ctx) - if err != nil { - return nil, fmt.Errorf("loading default AWS configuration: %w", err) - } - client := imds.NewFromConfig(cfg) - identityDocument, err := client.GetInstanceIdentityDocument(ctx, &imds.GetInstanceIdentityDocumentInput{}) - if err != nil { - return nil, fmt.Errorf("retrieving AWS instance identity document: %w", err) - } - return identityDocument, nil -} - -func providerID(identityDocument *imds.GetInstanceIdentityDocumentOutput) string { - // On AWS, the ProviderID has the form "aws:////" - return fmt.Sprintf("aws://%v/%v", identityDocument.AvailabilityZone, identityDocument.InstanceID) -} diff --git a/coordinator/cloudprovider/azure/api.go b/coordinator/cloudprovider/azure/api.go index 7dde0241a..337ae7bcd 100644 --- a/coordinator/cloudprovider/azure/api.go +++ b/coordinator/cloudprovider/azure/api.go @@ -50,16 +50,6 @@ type publicIPAddressesAPI interface { options *armnetwork.PublicIPAddressesClientGetOptions) (armnetwork.PublicIPAddressesClientGetResponse, error) } -type virtualMachinesAPI interface { - Get(ctx context.Context, resourceGroupName string, vmName string, options *armcompute.VirtualMachinesClientGetOptions) (armcompute.VirtualMachinesClientGetResponse, error) - List(resourceGroupName string, options *armcompute.VirtualMachinesClientListOptions) virtualMachinesClientListPager -} - -type virtualMachinesClientListPager interface { - NextPage(ctx context.Context) bool - PageResponse() armcompute.VirtualMachinesClientListResponse -} - type virtualMachineScaleSetVMsAPI interface { Get(ctx context.Context, resourceGroupName string, vmScaleSetName string, instanceID string, options *armcompute.VirtualMachineScaleSetVMsClientGetOptions) (armcompute.VirtualMachineScaleSetVMsClientGetResponse, error) List(resourceGroupName string, virtualMachineScaleSetName string, options *armcompute.VirtualMachineScaleSetVMsClientListOptions) virtualMachineScaleSetVMsClientListPager diff --git a/coordinator/cloudprovider/azure/api_test.go b/coordinator/cloudprovider/azure/api_test.go index edfc39959..1514fa1ee 100644 --- a/coordinator/cloudprovider/azure/api_test.go +++ b/coordinator/cloudprovider/azure/api_test.go @@ -52,49 +52,6 @@ func (a *stubNetworkInterfacesAPI) Get(ctx context.Context, resourceGroupName st }, a.getErr } -type stubVirtualMachinesClientListPager struct { - pagesCounter int - pages [][]*armcompute.VirtualMachine -} - -func (p *stubVirtualMachinesClientListPager) NextPage(ctx context.Context) bool { - return p.pagesCounter < len(p.pages) -} - -func (p *stubVirtualMachinesClientListPager) PageResponse() armcompute.VirtualMachinesClientListResponse { - if p.pagesCounter >= len(p.pages) { - return armcompute.VirtualMachinesClientListResponse{} - } - p.pagesCounter = p.pagesCounter + 1 - return armcompute.VirtualMachinesClientListResponse{ - VirtualMachinesClientListResult: armcompute.VirtualMachinesClientListResult{ - VirtualMachineListResult: armcompute.VirtualMachineListResult{ - Value: p.pages[p.pagesCounter-1], - }, - }, - } -} - -type stubVirtualMachinesAPI struct { - getVM armcompute.VirtualMachine - getErr error - listPages [][]*armcompute.VirtualMachine -} - -func (a *stubVirtualMachinesAPI) Get(ctx context.Context, resourceGroupName string, vmName string, options *armcompute.VirtualMachinesClientGetOptions) (armcompute.VirtualMachinesClientGetResponse, error) { - return armcompute.VirtualMachinesClientGetResponse{ - VirtualMachinesClientGetResult: armcompute.VirtualMachinesClientGetResult{ - VirtualMachine: a.getVM, - }, - }, a.getErr -} - -func (a *stubVirtualMachinesAPI) List(resourceGroupName string, options *armcompute.VirtualMachinesClientListOptions) virtualMachinesClientListPager { - return &stubVirtualMachinesClientListPager{ - pages: a.listPages, - } -} - type stubVirtualMachineScaleSetVMsClientListPager struct { pagesCounter int pages [][]*armcompute.VirtualMachineScaleSetVM diff --git a/coordinator/cloudprovider/azure/autoscaler.go b/coordinator/cloudprovider/azure/autoscaler.go index 5f62f0614..8c0295915 100644 --- a/coordinator/cloudprovider/azure/autoscaler.go +++ b/coordinator/cloudprovider/azure/autoscaler.go @@ -1,7 +1,6 @@ package azure import ( - "github.com/edgelesssys/constellation/coordinator/cloudprovider/cloudtypes" "github.com/edgelesssys/constellation/coordinator/kubernetes/k8sapi/resources" "github.com/edgelesssys/constellation/internal/azureshared" k8s "k8s.io/api/core/v1" @@ -17,8 +16,8 @@ func (a *Autoscaler) Name() string { } // Secrets returns a list of secrets to deploy together with the k8s cluster-autoscaler. -func (a *Autoscaler) Secrets(instance cloudtypes.Instance, cloudServiceAccountURI string) (resources.Secrets, error) { - subscriptionID, resourceGroup, err := azureshared.BasicsFromProviderID(instance.ProviderID) +func (a *Autoscaler) Secrets(providerID string, cloudServiceAccountURI string) (resources.Secrets, error) { + subscriptionID, resourceGroup, err := azureshared.BasicsFromProviderID(providerID) if err != nil { return resources.Secrets{}, err } diff --git a/coordinator/cloudprovider/azure/autoscaler_test.go b/coordinator/cloudprovider/azure/autoscaler_test.go index 234d95531..0a6963d5d 100644 --- a/coordinator/cloudprovider/azure/autoscaler_test.go +++ b/coordinator/cloudprovider/azure/autoscaler_test.go @@ -3,7 +3,6 @@ package azure import ( "testing" - "github.com/edgelesssys/constellation/coordinator/cloudprovider/cloudtypes" "github.com/edgelesssys/constellation/coordinator/kubernetes/k8sapi/resources" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -13,13 +12,13 @@ import ( func TestAutoscalerSecrets(t *testing.T) { testCases := map[string]struct { - instance cloudtypes.Instance + providerID string cloudServiceAccountURI string wantSecrets resources.Secrets wantErr bool }{ "Secrets works": { - instance: cloudtypes.Instance{ProviderID: "azure:///subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/virtualMachines/instance-name"}, + providerID: "azure:///subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/virtualMachines/instance-name", cloudServiceAccountURI: "serviceaccount://azure?tenant_id=tenant-id&client_id=client-id&client_secret=client-secret", wantSecrets: resources.Secrets{ &k8s.Secret{ @@ -43,11 +42,11 @@ func TestAutoscalerSecrets(t *testing.T) { }, }, "invalid providerID fails": { - instance: cloudtypes.Instance{ProviderID: "invalid"}, - wantErr: true, + providerID: "invalid", + wantErr: true, }, "invalid cloudServiceAccountURI fails": { - instance: cloudtypes.Instance{ProviderID: "azure:///subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/virtualMachines/instance-name"}, + providerID: "azure:///subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/virtualMachines/instance-name", cloudServiceAccountURI: "invalid", wantErr: true, }, @@ -59,7 +58,7 @@ func TestAutoscalerSecrets(t *testing.T) { require := require.New(t) autoscaler := Autoscaler{} - secrets, err := autoscaler.Secrets(tc.instance, tc.cloudServiceAccountURI) + secrets, err := autoscaler.Secrets(tc.providerID, tc.cloudServiceAccountURI) if tc.wantErr { assert.Error(err) return diff --git a/coordinator/cloudprovider/azure/ccm.go b/coordinator/cloudprovider/azure/ccm.go index 464983116..5a39354c2 100644 --- a/coordinator/cloudprovider/azure/ccm.go +++ b/coordinator/cloudprovider/azure/ccm.go @@ -5,9 +5,9 @@ import ( "encoding/json" "github.com/edgelesssys/constellation/coordinator/cloudprovider" - "github.com/edgelesssys/constellation/coordinator/cloudprovider/cloudtypes" "github.com/edgelesssys/constellation/coordinator/kubernetes/k8sapi/resources" "github.com/edgelesssys/constellation/internal/azureshared" + "github.com/edgelesssys/constellation/internal/cloud/metadata" k8s "k8s.io/api/core/v1" meta "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -55,17 +55,17 @@ func (c *CloudControllerManager) ExtraArgs() []string { // ConfigMaps returns a list of ConfigMaps to deploy together with the k8s cloud-controller-manager // Reference: https://kubernetes.io/docs/concepts/configuration/configmap/ . -func (c *CloudControllerManager) ConfigMaps(instance cloudtypes.Instance) (resources.ConfigMaps, error) { +func (c *CloudControllerManager) ConfigMaps(instance metadata.InstanceMetadata) (resources.ConfigMaps, error) { return resources.ConfigMaps{}, nil } // Secrets returns a list of secrets to deploy together with the k8s cloud-controller-manager. // Reference: https://kubernetes.io/docs/concepts/configuration/secret/ . -func (c *CloudControllerManager) Secrets(ctx context.Context, instance cloudtypes.Instance, cloudServiceAccountURI string) (resources.Secrets, error) { +func (c *CloudControllerManager) Secrets(ctx context.Context, providerID string, cloudServiceAccountURI string) (resources.Secrets, error) { // Azure CCM expects cloud provider config to contain cluster configuration and service principal client secrets // reference: https://kubernetes-sigs.github.io/cloud-provider-azure/install/configs/ - subscriptionID, resourceGroup, err := azureshared.BasicsFromProviderID(instance.ProviderID) + subscriptionID, resourceGroup, err := azureshared.BasicsFromProviderID(providerID) if err != nil { return resources.Secrets{}, err } @@ -75,7 +75,7 @@ func (c *CloudControllerManager) Secrets(ctx context.Context, instance cloudtype } vmType := "standard" - if _, _, _, _, err := azureshared.ScaleSetInformationFromProviderID(instance.ProviderID); err == nil { + if _, _, _, _, err := azureshared.ScaleSetInformationFromProviderID(providerID); err == nil { vmType = "vmss" } diff --git a/coordinator/cloudprovider/azure/ccm_test.go b/coordinator/cloudprovider/azure/ccm_test.go index 39d0692c6..c38af2d73 100644 --- a/coordinator/cloudprovider/azure/ccm_test.go +++ b/coordinator/cloudprovider/azure/ccm_test.go @@ -5,8 +5,8 @@ import ( "errors" "testing" - "github.com/edgelesssys/constellation/coordinator/cloudprovider/cloudtypes" "github.com/edgelesssys/constellation/coordinator/kubernetes/k8sapi/resources" + "github.com/edgelesssys/constellation/internal/cloud/metadata" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" k8s "k8s.io/api/core/v1" @@ -16,14 +16,14 @@ import ( func TestSecrets(t *testing.T) { someErr := errors.New("some error") testCases := map[string]struct { - instance cloudtypes.Instance + providerID string metadata ccmMetadata cloudServiceAccountURI string wantSecrets resources.Secrets wantErr bool }{ "Secrets works": { - instance: cloudtypes.Instance{ProviderID: "azure:///subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/virtualMachines/instance-name"}, + providerID: "azure:///subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/virtualMachines/instance-name", cloudServiceAccountURI: "serviceaccount://azure?tenant_id=tenant-id&client_id=client-id&client_secret=client-secret&location=location", metadata: &ccmMetadataStub{loadBalancerName: "load-balancer-name", networkSecurityGroupName: "network-security-group-name"}, wantSecrets: resources.Secrets{ @@ -43,7 +43,7 @@ func TestSecrets(t *testing.T) { }, }, "Secrets works for scale sets": { - instance: cloudtypes.Instance{ProviderID: "azure:///subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/virtualMachineScaleSets/scale-set-name/virtualMachines/instance-id"}, + providerID: "azure:///subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/virtualMachineScaleSets/scale-set-name/virtualMachines/instance-id", cloudServiceAccountURI: "serviceaccount://azure?tenant_id=tenant-id&client_id=client-id&client_secret=client-secret&location=location", metadata: &ccmMetadataStub{loadBalancerName: "load-balancer-name", networkSecurityGroupName: "network-security-group-name"}, wantSecrets: resources.Secrets{ @@ -63,24 +63,24 @@ func TestSecrets(t *testing.T) { }, }, "cannot get load balancer Name": { - instance: cloudtypes.Instance{ProviderID: "azure:///subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/virtualMachineScaleSets/scale-set-name/virtualMachines/instance-id"}, + providerID: "azure:///subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/virtualMachineScaleSets/scale-set-name/virtualMachines/instance-id", cloudServiceAccountURI: "serviceaccount://azure?tenant_id=tenant-id&client_id=client-id&client_secret=client-secret&location=location", metadata: &ccmMetadataStub{getLoadBalancerNameErr: someErr}, wantErr: true, }, "cannot get network security group name": { - instance: cloudtypes.Instance{ProviderID: "azure:///subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/virtualMachineScaleSets/scale-set-name/virtualMachines/instance-id"}, + providerID: "azure:///subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/virtualMachineScaleSets/scale-set-name/virtualMachines/instance-id", cloudServiceAccountURI: "serviceaccount://azure?tenant_id=tenant-id&client_id=client-id&client_secret=client-secret&location=location", metadata: &ccmMetadataStub{getNetworkSecurityGroupNameErr: someErr}, wantErr: true, }, "invalid providerID fails": { - instance: cloudtypes.Instance{ProviderID: "invalid"}, - metadata: &ccmMetadataStub{}, - wantErr: true, + providerID: "invalid", + metadata: &ccmMetadataStub{}, + wantErr: true, }, "invalid cloudServiceAccountURI fails": { - instance: cloudtypes.Instance{ProviderID: "azure:///subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/virtualMachines/instance-name"}, + providerID: "azure:///subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/virtualMachines/instance-name", metadata: &ccmMetadataStub{}, cloudServiceAccountURI: "invalid", wantErr: true, @@ -93,7 +93,7 @@ func TestSecrets(t *testing.T) { require := require.New(t) cloud := NewCloudControllerManager(tc.metadata) - secrets, err := cloud.Secrets(context.Background(), tc.instance, tc.cloudServiceAccountURI) + secrets, err := cloud.Secrets(context.Background(), tc.providerID, tc.cloudServiceAccountURI) if tc.wantErr { assert.Error(err) return @@ -112,7 +112,7 @@ func TestTrivialCCMFunctions(t *testing.T) { assert.NotEmpty(cloud.Path()) assert.NotEmpty(cloud.Name()) assert.NotEmpty(cloud.ExtraArgs()) - assert.Empty(cloud.ConfigMaps(cloudtypes.Instance{})) + assert.Empty(cloud.ConfigMaps(metadata.InstanceMetadata{})) assert.NotEmpty(cloud.Volumes()) assert.NotEmpty(cloud.VolumeMounts()) assert.Empty(cloud.Env()) diff --git a/coordinator/cloudprovider/azure/metadata.go b/coordinator/cloudprovider/azure/metadata.go index 1d3e24179..dafcb7a26 100644 --- a/coordinator/cloudprovider/azure/metadata.go +++ b/coordinator/cloudprovider/azure/metadata.go @@ -11,10 +11,8 @@ import ( "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute" "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork" "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources" - "github.com/edgelesssys/constellation/coordinator/cloudprovider/cloudtypes" - "github.com/edgelesssys/constellation/coordinator/core" - "github.com/edgelesssys/constellation/coordinator/role" "github.com/edgelesssys/constellation/internal/azureshared" + "github.com/edgelesssys/constellation/internal/cloud/metadata" ) var ( @@ -31,7 +29,6 @@ type Metadata struct { publicIPAddressesAPI scaleSetsAPI loadBalancerAPI - virtualMachinesAPI virtualMachineScaleSetVMsAPI tagsAPI applicationInsightsAPI @@ -63,7 +60,6 @@ func NewMetadata(ctx context.Context) (*Metadata, error) { securityGroupsAPI := armnetwork.NewSecurityGroupsClient(subscriptionID, cred, nil) scaleSetsAPI := armcompute.NewVirtualMachineScaleSetsClient(subscriptionID, cred, nil) loadBalancerAPI := armnetwork.NewLoadBalancersClient(subscriptionID, cred, nil) - virtualMachinesAPI := armcompute.NewVirtualMachinesClient(subscriptionID, cred, nil) virtualMachineScaleSetVMsAPI := armcompute.NewVirtualMachineScaleSetVMsClient(subscriptionID, cred, nil) tagsAPI := armresources.NewTagsClient(subscriptionID, cred, nil) applicationInsightsAPI := armapplicationinsights.NewComponentsClient(subscriptionID, cred, nil) @@ -76,7 +72,6 @@ func NewMetadata(ctx context.Context) (*Metadata, error) { publicIPAddressesAPI: &publicIPAddressesClient{publicIPAddressesAPI}, loadBalancerAPI: &loadBalancersClient{loadBalancerAPI}, scaleSetsAPI: &scaleSetsClient{scaleSetsAPI}, - virtualMachinesAPI: &virtualMachinesClient{virtualMachinesAPI}, virtualMachineScaleSetVMsAPI: &virtualMachineScaleSetVMsClient{virtualMachineScaleSetVMsAPI}, tagsAPI: &tagsClient{tagsAPI}, applicationInsightsAPI: &applicationInsightsClient{applicationInsightsAPI}, @@ -84,7 +79,7 @@ func NewMetadata(ctx context.Context) (*Metadata, error) { } // List retrieves all instances belonging to the current constellation. -func (m *Metadata) List(ctx context.Context) ([]cloudtypes.Instance, error) { +func (m *Metadata) List(ctx context.Context) ([]metadata.InstanceMetadata, error) { providerID, err := m.providerID(ctx) if err != nil { return nil, err @@ -93,54 +88,29 @@ func (m *Metadata) List(ctx context.Context) ([]cloudtypes.Instance, error) { if err != nil { return nil, err } - singleInstances, err := m.listVMs(ctx, resourceGroup) - if err != nil { - return nil, err - } scaleSetInstances, err := m.listScaleSetVMs(ctx, resourceGroup) if err != nil { return nil, err } - instances := make([]cloudtypes.Instance, 0, len(singleInstances)+len(scaleSetInstances)) - instances = append(instances, singleInstances...) - instances = append(instances, scaleSetInstances...) - return instances, nil + return scaleSetInstances, nil } // Self retrieves the current instance. -func (m *Metadata) Self(ctx context.Context) (cloudtypes.Instance, error) { +func (m *Metadata) Self(ctx context.Context) (metadata.InstanceMetadata, error) { providerID, err := m.providerID(ctx) if err != nil { - return cloudtypes.Instance{}, err + return metadata.InstanceMetadata{}, err } return m.GetInstance(ctx, providerID) } // GetInstance retrieves an instance using its providerID. -func (m *Metadata) GetInstance(ctx context.Context, providerID string) (cloudtypes.Instance, error) { - instance, singleErr := m.getVM(ctx, providerID) - if singleErr == nil { - return instance, nil - } +func (m *Metadata) GetInstance(ctx context.Context, providerID string) (metadata.InstanceMetadata, error) { instance, scaleSetErr := m.getScaleSetVM(ctx, providerID) if scaleSetErr == nil { return instance, nil } - return cloudtypes.Instance{}, fmt.Errorf("retrieving instance given providerID %v as either single VM or scale set VM: %v; %v", providerID, singleErr, scaleSetErr) -} - -// SignalRole signals the constellation role via cloud provider metadata. -// On single VMs, the role is stored in tags, on scale set VMs, the role is inferred from the scale set and not signalied explicitly. -func (m *Metadata) SignalRole(ctx context.Context, role role.Role) error { - providerID, err := m.providerID(ctx) - if err != nil { - return err - } - if _, _, _, _, err := azureshared.ScaleSetInformationFromProviderID(providerID); err == nil { - // scale set instances cannot store tags and role can be inferred from scale set name. - return nil - } - return m.setTag(ctx, core.RoleMetadataKey, role.String()) + return metadata.InstanceMetadata{}, fmt.Errorf("retrieving instance given providerID %v: %w", providerID, scaleSetErr) } // GetNetworkSecurityGroupName returns the security group name of the resource group. diff --git a/coordinator/cloudprovider/azure/metadata_test.go b/coordinator/cloudprovider/azure/metadata_test.go index ed254796d..2128354fc 100644 --- a/coordinator/cloudprovider/azure/metadata_test.go +++ b/coordinator/cloudprovider/azure/metadata_test.go @@ -8,20 +8,13 @@ import ( "github.com/Azure/azure-sdk-for-go/sdk/azcore/to" "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute" "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork" - "github.com/edgelesssys/constellation/coordinator/cloudprovider/cloudtypes" - "github.com/edgelesssys/constellation/coordinator/role" + "github.com/edgelesssys/constellation/internal/cloud/metadata" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) func TestList(t *testing.T) { - wantInstances := []cloudtypes.Instance{ - { - Name: "instance-name", - ProviderID: "azure:///subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/virtualMachines/instance-name", - PrivateIPs: []string{"192.0.2.0"}, - SSHKeys: map[string][]string{"user": {"key-data"}}, - }, + wantInstances := []metadata.InstanceMetadata{ { Name: "scale-set-name-instance-id", ProviderID: "azure:///subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/virtualMachineScaleSets/scale-set-name/virtualMachines/instance-id", @@ -33,17 +26,15 @@ func TestList(t *testing.T) { imdsAPI imdsAPI networkInterfacesAPI networkInterfacesAPI scaleSetsAPI scaleSetsAPI - virtualMachinesAPI virtualMachinesAPI virtualMachineScaleSetVMsAPI virtualMachineScaleSetVMsAPI tagsAPI tagsAPI wantErr bool - wantInstances []cloudtypes.Instance + wantInstances []metadata.InstanceMetadata }{ "List works": { imdsAPI: newIMDSStub(), networkInterfacesAPI: newNetworkInterfacesStub(), scaleSetsAPI: newScaleSetsStub(), - virtualMachinesAPI: newVirtualMachinesStub(), virtualMachineScaleSetVMsAPI: newVirtualMachineScaleSetsVMsStub(), tagsAPI: newTagsStub(), wantInstances: wantInstances, @@ -56,16 +47,10 @@ func TestList(t *testing.T) { imdsAPI: newInvalidIMDSStub(), wantErr: true, }, - "listVMs fails": { - imdsAPI: newIMDSStub(), - virtualMachinesAPI: newFailingListsVirtualMachinesStub(), - wantErr: true, - }, "listScaleSetVMs fails": { imdsAPI: newIMDSStub(), networkInterfacesAPI: newNetworkInterfacesStub(), scaleSetsAPI: newScaleSetsStub(), - virtualMachinesAPI: newVirtualMachinesStub(), virtualMachineScaleSetVMsAPI: newFailingListsVirtualMachineScaleSetsVMsStub(), tagsAPI: newTagsStub(), wantErr: true, @@ -77,15 +62,14 @@ func TestList(t *testing.T) { assert := assert.New(t) require := require.New(t) - metadata := Metadata{ + azureMetadata := Metadata{ imdsAPI: tc.imdsAPI, networkInterfacesAPI: tc.networkInterfacesAPI, scaleSetsAPI: tc.scaleSetsAPI, - virtualMachinesAPI: tc.virtualMachinesAPI, virtualMachineScaleSetVMsAPI: tc.virtualMachineScaleSetVMsAPI, tagsAPI: tc.tagsAPI, } - instances, err := metadata.List(context.Background()) + instances, err := azureMetadata.List(context.Background()) if tc.wantErr { assert.Error(err) @@ -98,13 +82,13 @@ func TestList(t *testing.T) { } func TestSelf(t *testing.T) { - wantVMInstance := cloudtypes.Instance{ + wantVMInstance := metadata.InstanceMetadata{ Name: "instance-name", ProviderID: "azure:///subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/virtualMachines/instance-name", PrivateIPs: []string{"192.0.2.0"}, SSHKeys: map[string][]string{"user": {"key-data"}}, } - wantScaleSetInstance := cloudtypes.Instance{ + wantScaleSetInstance := metadata.InstanceMetadata{ Name: "scale-set-name-instance-id", ProviderID: "azure:///subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/virtualMachineScaleSets/scale-set-name/virtualMachines/instance-id", PrivateIPs: []string{"192.0.2.0"}, @@ -113,22 +97,19 @@ func TestSelf(t *testing.T) { testCases := map[string]struct { imdsAPI imdsAPI networkInterfacesAPI networkInterfacesAPI - virtualMachinesAPI virtualMachinesAPI virtualMachineScaleSetVMsAPI virtualMachineScaleSetVMsAPI wantErr bool - wantInstance cloudtypes.Instance + wantInstance metadata.InstanceMetadata }{ "self for individual instance works": { imdsAPI: newIMDSStub(), networkInterfacesAPI: newNetworkInterfacesStub(), - virtualMachinesAPI: newVirtualMachinesStub(), virtualMachineScaleSetVMsAPI: newVirtualMachineScaleSetsVMsStub(), wantInstance: wantVMInstance, }, "self for scale set instance works": { imdsAPI: newScaleSetIMDSStub(), networkInterfacesAPI: newNetworkInterfacesStub(), - virtualMachinesAPI: newVirtualMachinesStub(), virtualMachineScaleSetVMsAPI: newVirtualMachineScaleSetsVMsStub(), wantInstance: wantScaleSetInstance, }, @@ -137,9 +118,8 @@ func TestSelf(t *testing.T) { wantErr: true, }, "GetInstance fails": { - imdsAPI: newIMDSStub(), - virtualMachinesAPI: newFailingGetVirtualMachinesStub(), - wantErr: true, + imdsAPI: newIMDSStub(), + wantErr: true, }, } @@ -151,7 +131,6 @@ func TestSelf(t *testing.T) { metadata := Metadata{ imdsAPI: tc.imdsAPI, networkInterfacesAPI: tc.networkInterfacesAPI, - virtualMachinesAPI: tc.virtualMachinesAPI, virtualMachineScaleSetVMsAPI: tc.virtualMachineScaleSetVMsAPI, } instance, err := metadata.Self(context.Background()) @@ -166,50 +145,6 @@ func TestSelf(t *testing.T) { } } -func TestSignalRole(t *testing.T) { - testCases := map[string]struct { - imdsAPI imdsAPI - tagsAPI tagsAPI - wantErr bool - }{ - "SignalRole works": { - imdsAPI: newIMDSStub(), - tagsAPI: newTagsStub(), - }, - "SignalRole is not attempted on scale set vm": { - imdsAPI: newScaleSetIMDSStub(), - }, - "providerID cannot be retrieved": { - imdsAPI: &stubIMDSAPI{retrieveErr: errors.New("imds err")}, - wantErr: true, - }, - "setting tag fails": { - imdsAPI: newIMDSStub(), - tagsAPI: newFailingTagsStub(), - wantErr: true, - }, - } - - for name, tc := range testCases { - t.Run(name, func(t *testing.T) { - assert := assert.New(t) - require := require.New(t) - - metadata := Metadata{ - imdsAPI: tc.imdsAPI, - tagsAPI: tc.tagsAPI, - } - err := metadata.SignalRole(context.Background(), role.Coordinator) - - if tc.wantErr { - assert.Error(err) - return - } - require.NoError(err) - }) - } -} - func TestGetNetworkSecurityGroupName(t *testing.T) { name := "network-security-group-name" testCases := map[string]struct { @@ -719,12 +654,6 @@ func newInvalidIMDSStub() *stubIMDSAPI { } } -func newFailingIMDSStub() *stubIMDSAPI { - return &stubIMDSAPI{ - retrieveErr: errors.New("imds retrieve error"), - } -} - func newNetworkInterfacesStub() *stubNetworkInterfacesAPI { return &stubNetworkInterfacesAPI{ getInterface: armnetwork.Interface{ @@ -754,81 +683,6 @@ func newScaleSetsStub() *stubScaleSetsAPI { } } -func newVirtualMachinesStub() *stubVirtualMachinesAPI { - return &stubVirtualMachinesAPI{ - getVM: armcompute.VirtualMachine{ - Name: to.StringPtr("instance-name"), - ID: to.StringPtr("/subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/virtualMachines/instance-name"), - Properties: &armcompute.VirtualMachineProperties{ - NetworkProfile: &armcompute.NetworkProfile{ - NetworkInterfaces: []*armcompute.NetworkInterfaceReference{ - { - ID: to.StringPtr("/subscriptions/subscription/resourceGroups/resource-group/providers/Microsoft.Network/networkInterfaces/interface-name"), - }, - }, - }, - OSProfile: &armcompute.OSProfile{ - LinuxConfiguration: &armcompute.LinuxConfiguration{ - SSH: &armcompute.SSHConfiguration{ - PublicKeys: []*armcompute.SSHPublicKey{ - { - KeyData: to.StringPtr("key-data"), - Path: to.StringPtr("/home/user/.ssh/authorized_keys"), - }, - }, - }, - }, - }, - }, - }, - listPages: [][]*armcompute.VirtualMachine{ - { - { - Name: to.StringPtr("instance-name"), - ID: to.StringPtr("/subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/virtualMachines/instance-name"), - Properties: &armcompute.VirtualMachineProperties{ - NetworkProfile: &armcompute.NetworkProfile{ - NetworkInterfaces: []*armcompute.NetworkInterfaceReference{ - { - ID: to.StringPtr("/subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Network/networkInterfaces/interface-name"), - }, - }, - }, - OSProfile: &armcompute.OSProfile{ - LinuxConfiguration: &armcompute.LinuxConfiguration{ - SSH: &armcompute.SSHConfiguration{ - PublicKeys: []*armcompute.SSHPublicKey{ - { - KeyData: to.StringPtr("key-data"), - Path: to.StringPtr("/home/user/.ssh/authorized_keys"), - }, - }, - }, - }, - }, - }, - }, - }, - }, - } -} - -func newFailingListsVirtualMachinesStub() *stubVirtualMachinesAPI { - return &stubVirtualMachinesAPI{ - listPages: [][]*armcompute.VirtualMachine{ - { - {}, - }, - }, - } -} - -func newFailingGetVirtualMachinesStub() *stubVirtualMachinesAPI { - return &stubVirtualMachinesAPI{ - getErr: errors.New("get err"), - } -} - func newVirtualMachineScaleSetsVMsStub() *stubVirtualMachineScaleSetVMsAPI { return &stubVirtualMachineScaleSetVMsAPI{ getVM: armcompute.VirtualMachineScaleSetVM{ @@ -907,10 +761,3 @@ func newFailingListsVirtualMachineScaleSetsVMsStub() *stubVirtualMachineScaleSet func newTagsStub() *stubTagsAPI { return &stubTagsAPI{} } - -func newFailingTagsStub() *stubTagsAPI { - return &stubTagsAPI{ - createOrUpdateAtScopeErr: errors.New("createOrUpdateErr"), - updateAtScopeErr: errors.New("updateErr"), - } -} diff --git a/coordinator/cloudprovider/azure/scaleset.go b/coordinator/cloudprovider/azure/scaleset.go index b43864685..e0a6978d2 100644 --- a/coordinator/cloudprovider/azure/scaleset.go +++ b/coordinator/cloudprovider/azure/scaleset.go @@ -7,9 +7,9 @@ import ( "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute" "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork" - "github.com/edgelesssys/constellation/coordinator/cloudprovider/cloudtypes" "github.com/edgelesssys/constellation/coordinator/role" "github.com/edgelesssys/constellation/internal/azureshared" + "github.com/edgelesssys/constellation/internal/cloud/metadata" ) var ( @@ -18,30 +18,30 @@ var ( ) // getScaleSetVM tries to get an azure vm belonging to a scale set. -func (m *Metadata) getScaleSetVM(ctx context.Context, providerID string) (cloudtypes.Instance, error) { +func (m *Metadata) getScaleSetVM(ctx context.Context, providerID string) (metadata.InstanceMetadata, error) { _, resourceGroup, scaleSet, instanceID, err := azureshared.ScaleSetInformationFromProviderID(providerID) if err != nil { - return cloudtypes.Instance{}, err + return metadata.InstanceMetadata{}, err } vmResp, err := m.virtualMachineScaleSetVMsAPI.Get(ctx, resourceGroup, scaleSet, instanceID, nil) if err != nil { - return cloudtypes.Instance{}, err + return metadata.InstanceMetadata{}, err } networkInterfaces, err := m.getScaleSetVMInterfaces(ctx, vmResp.VirtualMachineScaleSetVM, resourceGroup, scaleSet, instanceID) if err != nil { - return cloudtypes.Instance{}, err + return metadata.InstanceMetadata{}, err } publicIPAddresses, err := m.getScaleSetVMPublicIPAddresses(ctx, resourceGroup, scaleSet, instanceID, networkInterfaces) if err != nil { - return cloudtypes.Instance{}, err + return metadata.InstanceMetadata{}, err } return convertScaleSetVMToCoreInstance(scaleSet, vmResp.VirtualMachineScaleSetVM, networkInterfaces, publicIPAddresses) } // listScaleSetVMs lists all scale set VMs in the current resource group. -func (m *Metadata) listScaleSetVMs(ctx context.Context, resourceGroup string) ([]cloudtypes.Instance, error) { - instances := []cloudtypes.Instance{} +func (m *Metadata) listScaleSetVMs(ctx context.Context, resourceGroup string) ([]metadata.InstanceMetadata, error) { + instances := []metadata.InstanceMetadata{} scaleSetPager := m.scaleSetsAPI.List(resourceGroup, nil) for scaleSetPager.NextPage(ctx) { for _, scaleSet := range scaleSetPager.PageResponse().Value { @@ -71,12 +71,12 @@ func (m *Metadata) listScaleSetVMs(ctx context.Context, resourceGroup string) ([ } // convertScaleSetVMToCoreInstance converts an azure scale set virtual machine with interface configurations into a core.Instance. -func convertScaleSetVMToCoreInstance(scaleSet string, vm armcompute.VirtualMachineScaleSetVM, networkInterfaces []armnetwork.Interface, publicIPAddresses []string) (cloudtypes.Instance, error) { +func convertScaleSetVMToCoreInstance(scaleSet string, vm armcompute.VirtualMachineScaleSetVM, networkInterfaces []armnetwork.Interface, publicIPAddresses []string) (metadata.InstanceMetadata, error) { if vm.ID == nil { - return cloudtypes.Instance{}, errors.New("retrieving instance from armcompute API client returned no instance ID") + return metadata.InstanceMetadata{}, errors.New("retrieving instance from armcompute API client returned no instance ID") } if vm.Properties == nil || vm.Properties.OSProfile == nil || vm.Properties.OSProfile.ComputerName == nil { - return cloudtypes.Instance{}, errors.New("retrieving instance from armcompute API client returned no computer name") + return metadata.InstanceMetadata{}, errors.New("retrieving instance from armcompute API client returned no computer name") } var sshKeys map[string][]string if vm.Properties.OSProfile.LinuxConfiguration == nil || vm.Properties.OSProfile.LinuxConfiguration.SSH == nil { @@ -84,7 +84,7 @@ func convertScaleSetVMToCoreInstance(scaleSet string, vm armcompute.VirtualMachi } else { sshKeys = extractSSHKeys(*vm.Properties.OSProfile.LinuxConfiguration.SSH) } - return cloudtypes.Instance{ + return metadata.InstanceMetadata{ Name: *vm.Properties.OSProfile.ComputerName, ProviderID: "azure://" + *vm.ID, Role: extractScaleSetVMRole(scaleSet), diff --git a/coordinator/cloudprovider/azure/scaleset_test.go b/coordinator/cloudprovider/azure/scaleset_test.go index c06ce5723..8d95d7d77 100644 --- a/coordinator/cloudprovider/azure/scaleset_test.go +++ b/coordinator/cloudprovider/azure/scaleset_test.go @@ -8,14 +8,14 @@ import ( "github.com/Azure/azure-sdk-for-go/sdk/azcore/to" "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute" "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork" - "github.com/edgelesssys/constellation/coordinator/cloudprovider/cloudtypes" "github.com/edgelesssys/constellation/coordinator/role" + "github.com/edgelesssys/constellation/internal/cloud/metadata" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) func TestGetScaleSetVM(t *testing.T) { - wantInstance := cloudtypes.Instance{ + wantInstance := metadata.InstanceMetadata{ Name: "scale-set-name-instance-id", ProviderID: "azure:///subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/virtualMachineScaleSets/scale-set-name/virtualMachines/instance-id", PrivateIPs: []string{"192.0.2.0"}, @@ -26,7 +26,7 @@ func TestGetScaleSetVM(t *testing.T) { networkInterfacesAPI networkInterfacesAPI virtualMachineScaleSetVMsAPI virtualMachineScaleSetVMsAPI wantErr bool - wantInstance cloudtypes.Instance + wantInstance metadata.InstanceMetadata }{ "getVM for scale set instance works": { providerID: "azure:///subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/virtualMachineScaleSets/scale-set-name/virtualMachines/instance-id", @@ -43,12 +43,6 @@ func TestGetScaleSetVM(t *testing.T) { virtualMachineScaleSetVMsAPI: newFailingGetScaleSetVirtualMachinesStub(), wantErr: true, }, - "retrieving interfaces fails": { - providerID: "azure:///subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/virtualMachineScaleSets/scale-set-name/virtualMachines/instance-id", - virtualMachineScaleSetVMsAPI: newVirtualMachineScaleSetsVMsStub(), - networkInterfacesAPI: newFailingNetworkInterfacesStub(), - wantErr: true, - }, "conversion fails": { providerID: "azure:///subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/virtualMachineScaleSets/scale-set-name/virtualMachines/instance-id", virtualMachineScaleSetVMsAPI: newGetInvalidScaleSetVirtualMachinesStub(), @@ -79,7 +73,7 @@ func TestGetScaleSetVM(t *testing.T) { } func TestListScaleSetVMs(t *testing.T) { - wantInstances := []cloudtypes.Instance{ + wantInstances := []metadata.InstanceMetadata{ { Name: "scale-set-name-instance-id", ProviderID: "azure:///subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/virtualMachineScaleSets/scale-set-name/virtualMachines/instance-id", @@ -93,7 +87,7 @@ func TestListScaleSetVMs(t *testing.T) { virtualMachineScaleSetVMsAPI virtualMachineScaleSetVMsAPI scaleSetsAPI scaleSetsAPI wantErr bool - wantInstances []cloudtypes.Instance + wantInstances []metadata.InstanceMetadata }{ "listVMs works": { imdsAPI: newIMDSStub(), @@ -114,7 +108,7 @@ func TestListScaleSetVMs(t *testing.T) { networkInterfacesAPI: newNetworkInterfacesStub(), virtualMachineScaleSetVMsAPI: &stubVirtualMachineScaleSetVMsAPI{}, scaleSetsAPI: newScaleSetsStub(), - wantInstances: []cloudtypes.Instance{}, + wantInstances: []metadata.InstanceMetadata{}, }, "can skip nil in VM list": { imdsAPI: newIMDSStub(), @@ -123,13 +117,6 @@ func TestListScaleSetVMs(t *testing.T) { scaleSetsAPI: newScaleSetsStub(), wantInstances: wantInstances, }, - "retrieving network interfaces fails": { - imdsAPI: newIMDSStub(), - networkInterfacesAPI: newFailingNetworkInterfacesStub(), - virtualMachineScaleSetVMsAPI: newVirtualMachineScaleSetsVMsStub(), - scaleSetsAPI: newScaleSetsStub(), - wantErr: true, - }, "converting instance fails": { imdsAPI: newIMDSStub(), networkInterfacesAPI: newNetworkInterfacesStub(), @@ -168,7 +155,7 @@ func TestConvertScaleSetVMToCoreInstance(t *testing.T) { inInterface []armnetwork.Interface inPublicIPs []string wantErr bool - wantInstance cloudtypes.Instance + wantInstance metadata.InstanceMetadata }{ "conversion works": { inVM: armcompute.VirtualMachineScaleSetVM{ @@ -197,7 +184,7 @@ func TestConvertScaleSetVMToCoreInstance(t *testing.T) { }, }, inPublicIPs: []string{"192.0.2.100", "192.0.2.101"}, - wantInstance: cloudtypes.Instance{ + wantInstance: metadata.InstanceMetadata{ Name: "scale-set-name-instance-id", ProviderID: "azure:///subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/virtualMachineScaleSets/scale-set-name/virtualMachines/instance-id", PrivateIPs: []string{"192.0.2.0"}, diff --git a/coordinator/cloudprovider/azure/virtualmachine.go b/coordinator/cloudprovider/azure/virtualmachine.go deleted file mode 100644 index 470de0f0b..000000000 --- a/coordinator/cloudprovider/azure/virtualmachine.go +++ /dev/null @@ -1,91 +0,0 @@ -package azure - -import ( - "context" - "fmt" - - "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute" - "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork" - "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources" - "github.com/Azure/go-autorest/autorest/to" - "github.com/edgelesssys/constellation/coordinator/cloudprovider" - "github.com/edgelesssys/constellation/coordinator/cloudprovider/cloudtypes" - "github.com/edgelesssys/constellation/internal/azureshared" -) - -func (m *Metadata) getVM(ctx context.Context, providerID string) (cloudtypes.Instance, error) { - _, resourceGroup, instanceName, err := azureshared.VMInformationFromProviderID(providerID) - if err != nil { - return cloudtypes.Instance{}, err - } - vmResp, err := m.virtualMachinesAPI.Get(ctx, resourceGroup, instanceName, nil) - if err != nil { - return cloudtypes.Instance{}, err - } - interfaceIPConfigurations, err := m.getVMInterfaces(ctx, vmResp.VirtualMachine, resourceGroup) - if err != nil { - return cloudtypes.Instance{}, err - } - return convertVMToCoreInstance(vmResp.VirtualMachine, interfaceIPConfigurations) -} - -// listVMs lists all individual VMs in the current resource group. -func (m *Metadata) listVMs(ctx context.Context, resourceGroup string) ([]cloudtypes.Instance, error) { - instances := []cloudtypes.Instance{} - pager := m.virtualMachinesAPI.List(resourceGroup, nil) - for pager.NextPage(ctx) { - for _, vm := range pager.PageResponse().Value { - if vm == nil { - continue - } - interfaces, err := m.getVMInterfaces(ctx, *vm, resourceGroup) - if err != nil { - return nil, err - } - instance, err := convertVMToCoreInstance(*vm, interfaces) - if err != nil { - return nil, err - } - instances = append(instances, instance) - } - } - return instances, nil -} - -// setTag merges key-value pair into VM tags. -func (m *Metadata) setTag(ctx context.Context, key, value string) error { - instanceMetadata, err := m.imdsAPI.Retrieve(ctx) - if err != nil { - return err - } - _, err = m.tagsAPI.UpdateAtScope(ctx, instanceMetadata.Compute.ResourceID, armresources.TagsPatchResource{ - Operation: armresources.TagsPatchOperationMerge.ToPtr(), - Properties: &armresources.Tags{ - Tags: map[string]*string{ - key: to.StringPtr(value), - }, - }, - }, nil) - return err -} - -// convertVMToCoreInstance converts an azure virtual machine with interface configurations into a cloudtypes.Instance. -func convertVMToCoreInstance(vm armcompute.VirtualMachine, networkInterfaces []armnetwork.Interface) (cloudtypes.Instance, error) { - if vm.Name == nil || vm.ID == nil { - return cloudtypes.Instance{}, fmt.Errorf("retrieving instance from armcompute API client returned invalid instance Name (%v) or ID (%v)", vm.Name, vm.ID) - } - var sshKeys map[string][]string - if vm.Properties == nil || vm.Properties.OSProfile == nil || vm.Properties.OSProfile.LinuxConfiguration == nil || vm.Properties.OSProfile.LinuxConfiguration.SSH == nil { - sshKeys = map[string][]string{} - } else { - sshKeys = extractSSHKeys(*vm.Properties.OSProfile.LinuxConfiguration.SSH) - } - metadata := extractInstanceTags(vm.Tags) - return cloudtypes.Instance{ - Name: *vm.Name, - ProviderID: "azure://" + *vm.ID, - Role: cloudprovider.ExtractRole(metadata), - PrivateIPs: extractPrivateIPs(networkInterfaces), - SSHKeys: sshKeys, - }, nil -} diff --git a/coordinator/cloudprovider/azure/virtualmachine_test.go b/coordinator/cloudprovider/azure/virtualmachine_test.go deleted file mode 100644 index e0dd0970e..000000000 --- a/coordinator/cloudprovider/azure/virtualmachine_test.go +++ /dev/null @@ -1,374 +0,0 @@ -package azure - -import ( - "context" - "errors" - "testing" - - "github.com/Azure/azure-sdk-for-go/sdk/azcore/to" - "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute" - "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork" - "github.com/edgelesssys/constellation/coordinator/cloudprovider/cloudtypes" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestGetVM(t *testing.T) { - wantInstance := cloudtypes.Instance{ - Name: "instance-name", - ProviderID: "azure:///subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/virtualMachines/instance-name", - PrivateIPs: []string{"192.0.2.0"}, - SSHKeys: map[string][]string{"user": {"key-data"}}, - } - testCases := map[string]struct { - providerID string - networkInterfacesAPI networkInterfacesAPI - virtualMachinesAPI virtualMachinesAPI - wantErr bool - wantInstance cloudtypes.Instance - }{ - "getVM for individual instance works": { - providerID: "azure:///subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/virtualMachines/instance-name", - networkInterfacesAPI: newNetworkInterfacesStub(), - virtualMachinesAPI: newVirtualMachinesStub(), - wantInstance: wantInstance, - }, - "getVM for scale set instance must fail": { - providerID: "azure:///subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/virtualMachineScaleSets/scale-set-name/virtualMachines/instance-id", - wantErr: true, - }, - "Get fails": { - providerID: "azure:///subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/virtualMachines/instance-name", - virtualMachinesAPI: newFailingGetVirtualMachinesStub(), - wantErr: true, - }, - "retrieving interfaces fails": { - providerID: "azure:///subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/virtualMachines/instance-name", - virtualMachinesAPI: newVirtualMachinesStub(), - networkInterfacesAPI: newFailingNetworkInterfacesStub(), - wantErr: true, - }, - "conversion fails": { - providerID: "azure:///subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/virtualMachines/instance-name", - virtualMachinesAPI: newGetInvalidVirtualMachinesStub(), - networkInterfacesAPI: newNetworkInterfacesStub(), - wantErr: true, - }, - } - - for name, tc := range testCases { - t.Run(name, func(t *testing.T) { - assert := assert.New(t) - require := require.New(t) - - metadata := Metadata{ - networkInterfacesAPI: tc.networkInterfacesAPI, - virtualMachinesAPI: tc.virtualMachinesAPI, - } - instance, err := metadata.getVM(context.Background(), tc.providerID) - - if tc.wantErr { - assert.Error(err) - return - } - require.NoError(err) - assert.Equal(tc.wantInstance, instance) - }) - } -} - -func TestListVMs(t *testing.T) { - wantInstances := []cloudtypes.Instance{ - { - Name: "instance-name", - ProviderID: "azure:///subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/virtualMachines/instance-name", - PrivateIPs: []string{"192.0.2.0"}, - SSHKeys: map[string][]string{"user": {"key-data"}}, - }, - } - testCases := map[string]struct { - imdsAPI imdsAPI - networkInterfacesAPI networkInterfacesAPI - virtualMachinesAPI virtualMachinesAPI - wantErr bool - wantInstances []cloudtypes.Instance - }{ - "listVMs works": { - imdsAPI: newIMDSStub(), - networkInterfacesAPI: newNetworkInterfacesStub(), - virtualMachinesAPI: newVirtualMachinesStub(), - wantInstances: wantInstances, - }, - "listVMs can return 0 VMs": { - imdsAPI: newIMDSStub(), - networkInterfacesAPI: newNetworkInterfacesStub(), - virtualMachinesAPI: &stubVirtualMachinesAPI{}, - wantInstances: []cloudtypes.Instance{}, - }, - "can skip nil in VM list": { - imdsAPI: newIMDSStub(), - networkInterfacesAPI: newNetworkInterfacesStub(), - virtualMachinesAPI: newListContainingNilVirtualMachinesStub(), - wantInstances: wantInstances, - }, - "retrieving network interfaces fails": { - imdsAPI: newIMDSStub(), - networkInterfacesAPI: newFailingNetworkInterfacesStub(), - virtualMachinesAPI: newVirtualMachinesStub(), - wantErr: true, - }, - "converting instance fails": { - imdsAPI: newIMDSStub(), - networkInterfacesAPI: newNetworkInterfacesStub(), - virtualMachinesAPI: newListContainingInvalidVirtualMachinesStub(), - wantErr: true, - }, - } - - for name, tc := range testCases { - t.Run(name, func(t *testing.T) { - assert := assert.New(t) - require := require.New(t) - - metadata := Metadata{ - imdsAPI: tc.imdsAPI, - networkInterfacesAPI: tc.networkInterfacesAPI, - virtualMachinesAPI: tc.virtualMachinesAPI, - } - instances, err := metadata.listVMs(context.Background(), "resource-group") - - if tc.wantErr { - assert.Error(err) - return - } - require.NoError(err) - assert.ElementsMatch(tc.wantInstances, instances) - }) - } -} - -func TestSetTag(t *testing.T) { - testCases := map[string]struct { - imdsAPI imdsAPI - tagsAPI tagsAPI - wantErr bool - }{ - "setTag works": { - imdsAPI: newIMDSStub(), - tagsAPI: newTagsStub(), - }, - "retrieving resource ID fails": { - imdsAPI: newFailingIMDSStub(), - tagsAPI: newTagsStub(), - wantErr: true, - }, - "updating tags fails": { - imdsAPI: newIMDSStub(), - tagsAPI: newFailingTagsStub(), - wantErr: true, - }, - } - - for name, tc := range testCases { - t.Run(name, func(t *testing.T) { - assert := assert.New(t) - require := require.New(t) - - metadata := Metadata{ - imdsAPI: tc.imdsAPI, - tagsAPI: tc.tagsAPI, - } - err := metadata.setTag(context.Background(), "key", "value") - - if tc.wantErr { - assert.Error(err) - return - } - require.NoError(err) - }) - } -} - -func TestConvertVMToCoreInstance(t *testing.T) { - testCases := map[string]struct { - inVM armcompute.VirtualMachine - inInterface []armnetwork.Interface - wantErr bool - wantInstance cloudtypes.Instance - }{ - "conversion works": { - inVM: armcompute.VirtualMachine{ - Name: to.StringPtr("instance-name"), - ID: to.StringPtr("/subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/virtualMachines/instance-name"), - Tags: map[string]*string{"tag-key": to.StringPtr("tag-value")}, - Properties: &armcompute.VirtualMachineProperties{ - OSProfile: &armcompute.OSProfile{ - LinuxConfiguration: &armcompute.LinuxConfiguration{ - SSH: &armcompute.SSHConfiguration{ - PublicKeys: []*armcompute.SSHPublicKey{ - { - Path: to.StringPtr("/home/user/.ssh/authorized_keys"), - KeyData: to.StringPtr("key-data"), - }, - }, - }, - }, - }, - }, - }, - inInterface: []armnetwork.Interface{ - { - Name: to.StringPtr("interface-name"), - ID: to.StringPtr("/subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Network/networkInterfaces/interface-name"), - Properties: &armnetwork.InterfacePropertiesFormat{ - IPConfigurations: []*armnetwork.InterfaceIPConfiguration{ - { - Properties: &armnetwork.InterfaceIPConfigurationPropertiesFormat{ - PrivateIPAddress: to.StringPtr("192.0.2.0"), - }, - }, - }, - }, - }, - }, - wantInstance: cloudtypes.Instance{ - Name: "instance-name", - ProviderID: "azure:///subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/virtualMachines/instance-name", - PrivateIPs: []string{"192.0.2.0"}, - SSHKeys: map[string][]string{"user": {"key-data"}}, - }, - }, - "conversion without SSH keys works": { - inVM: armcompute.VirtualMachine{ - Name: to.StringPtr("instance-name"), - ID: to.StringPtr("/subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/virtualMachines/instance-name"), - Tags: map[string]*string{"tag-key": to.StringPtr("tag-value")}, - }, - inInterface: []armnetwork.Interface{ - { - Name: to.StringPtr("interface-name"), - ID: to.StringPtr("/subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Network/networkInterfaces/interface-name"), - Properties: &armnetwork.InterfacePropertiesFormat{ - IPConfigurations: []*armnetwork.InterfaceIPConfiguration{ - { - Properties: &armnetwork.InterfaceIPConfigurationPropertiesFormat{ - PrivateIPAddress: to.StringPtr("192.0.2.0"), - }, - }, - }, - }, - }, - }, - wantInstance: cloudtypes.Instance{ - Name: "instance-name", - ProviderID: "azure:///subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/virtualMachines/instance-name", - PrivateIPs: []string{"192.0.2.0"}, - SSHKeys: map[string][]string{}, - }, - }, - "invalid instance": { - inVM: armcompute.VirtualMachine{}, - wantErr: true, - }, - } - - for name, tc := range testCases { - t.Run(name, func(t *testing.T) { - assert := assert.New(t) - require := require.New(t) - - instance, err := convertVMToCoreInstance(tc.inVM, tc.inInterface) - - if tc.wantErr { - assert.Error(err) - return - } - require.NoError(err) - assert.Equal(tc.wantInstance, instance) - }) - } -} - -func newFailingNetworkInterfacesStub() *stubNetworkInterfacesAPI { - return &stubNetworkInterfacesAPI{ - getErr: errors.New("get err"), - } -} - -func newGetInvalidVirtualMachinesStub() *stubVirtualMachinesAPI { - return &stubVirtualMachinesAPI{ - getVM: armcompute.VirtualMachine{}, - } -} - -func newListContainingNilVirtualMachinesStub() *stubVirtualMachinesAPI { - return &stubVirtualMachinesAPI{ - listPages: [][]*armcompute.VirtualMachine{ - { - nil, - { - Name: to.StringPtr("instance-name"), - ID: to.StringPtr("/subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/virtualMachines/instance-name"), - Tags: map[string]*string{ - "tag-key": to.StringPtr("tag-value"), - }, - Properties: &armcompute.VirtualMachineProperties{ - NetworkProfile: &armcompute.NetworkProfile{ - NetworkInterfaces: []*armcompute.NetworkInterfaceReference{ - { - ID: to.StringPtr("/subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Network/networkInterfaces/interface-name"), - }, - }, - }, - OSProfile: &armcompute.OSProfile{ - LinuxConfiguration: &armcompute.LinuxConfiguration{ - SSH: &armcompute.SSHConfiguration{ - PublicKeys: []*armcompute.SSHPublicKey{ - { - KeyData: to.StringPtr("key-data"), - Path: to.StringPtr("/home/user/.ssh/authorized_keys"), - }, - }, - }, - }, - }, - }, - }, - }, - }, - } -} - -func newListContainingInvalidVirtualMachinesStub() *stubVirtualMachinesAPI { - return &stubVirtualMachinesAPI{ - listPages: [][]*armcompute.VirtualMachine{ - { - { - Name: nil, - ID: nil, - Properties: &armcompute.VirtualMachineProperties{ - NetworkProfile: &armcompute.NetworkProfile{ - NetworkInterfaces: []*armcompute.NetworkInterfaceReference{ - { - ID: to.StringPtr("/subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Network/networkInterfaces/interface-name"), - }, - }, - }, - OSProfile: &armcompute.OSProfile{ - LinuxConfiguration: &armcompute.LinuxConfiguration{ - SSH: &armcompute.SSHConfiguration{ - PublicKeys: []*armcompute.SSHPublicKey{ - { - KeyData: to.StringPtr("key-data"), - Path: to.StringPtr("/home/user/.ssh/authorized_keys"), - }, - }, - }, - }, - }, - }, - }, - }, - }, - } -} diff --git a/coordinator/cloudprovider/azure/wrappers.go b/coordinator/cloudprovider/azure/wrappers.go index 22eb04861..b8ac0333a 100644 --- a/coordinator/cloudprovider/azure/wrappers.go +++ b/coordinator/cloudprovider/azure/wrappers.go @@ -68,18 +68,6 @@ func (c *loadBalancersClient) List(resourceGroupName string, options *armnetwork return c.LoadBalancersClient.List(resourceGroupName, options) } -type virtualMachinesClient struct { - *armcompute.VirtualMachinesClient -} - -func (c *virtualMachinesClient) Get(ctx context.Context, resourceGroupName, vmName string, options *armcompute.VirtualMachinesClientGetOptions) (armcompute.VirtualMachinesClientGetResponse, error) { - return c.VirtualMachinesClient.Get(ctx, resourceGroupName, vmName, options) -} - -func (c *virtualMachinesClient) List(resourceGroupName string, options *armcompute.VirtualMachinesClientListOptions) virtualMachinesClientListPager { - return c.VirtualMachinesClient.List(resourceGroupName, options) -} - type virtualMachineScaleSetVMsClient struct { *armcompute.VirtualMachineScaleSetVMsClient } diff --git a/coordinator/cloudprovider/cloudtypes/instance.go b/coordinator/cloudprovider/cloudtypes/instance.go deleted file mode 100644 index 90391bde3..000000000 --- a/coordinator/cloudprovider/cloudtypes/instance.go +++ /dev/null @@ -1,15 +0,0 @@ -package cloudtypes - -import "github.com/edgelesssys/constellation/coordinator/role" - -// Instance describes metadata of a peer. -type Instance struct { - Name string - ProviderID string - Role role.Role - PrivateIPs []string - PublicIPs []string - AliasIPRanges []string - // SSHKeys maps usernames to ssh public keys. - SSHKeys map[string][]string -} diff --git a/coordinator/cloudprovider/gcp/autoscaler.go b/coordinator/cloudprovider/gcp/autoscaler.go index 1dc1a34e8..d54b4fb97 100644 --- a/coordinator/cloudprovider/gcp/autoscaler.go +++ b/coordinator/cloudprovider/gcp/autoscaler.go @@ -1,8 +1,8 @@ package gcp import ( - "github.com/edgelesssys/constellation/coordinator/cloudprovider/cloudtypes" "github.com/edgelesssys/constellation/coordinator/kubernetes/k8sapi/resources" + "github.com/edgelesssys/constellation/internal/cloud/metadata" k8s "k8s.io/api/core/v1" ) @@ -15,7 +15,7 @@ func (a *Autoscaler) Name() string { } // Secrets returns a list of secrets to deploy together with the k8s cluster-autoscaler. -func (a *Autoscaler) Secrets(instance cloudtypes.Instance, cloudServiceAccountURI string) (resources.Secrets, error) { +func (a *Autoscaler) Secrets(instance metadata.InstanceMetadata, cloudServiceAccountURI string) (resources.Secrets, error) { return resources.Secrets{}, nil } diff --git a/coordinator/cloudprovider/gcp/autoscaler_test.go b/coordinator/cloudprovider/gcp/autoscaler_test.go index 0c9d8702f..5be1d1eff 100644 --- a/coordinator/cloudprovider/gcp/autoscaler_test.go +++ b/coordinator/cloudprovider/gcp/autoscaler_test.go @@ -3,7 +3,7 @@ package gcp import ( "testing" - "github.com/edgelesssys/constellation/coordinator/cloudprovider/cloudtypes" + "github.com/edgelesssys/constellation/internal/cloud/metadata" "github.com/stretchr/testify/assert" ) @@ -12,7 +12,7 @@ func TestTrivialAutoscalerFunctions(t *testing.T) { autoscaler := Autoscaler{} assert.NotEmpty(autoscaler.Name()) - assert.Empty(autoscaler.Secrets(cloudtypes.Instance{}, "")) + assert.Empty(autoscaler.Secrets(metadata.InstanceMetadata{}, "")) assert.NotEmpty(autoscaler.Volumes()) assert.NotEmpty(autoscaler.VolumeMounts()) assert.NotEmpty(autoscaler.Env()) diff --git a/coordinator/cloudprovider/gcp/ccm.go b/coordinator/cloudprovider/gcp/ccm.go index 783781fc5..7eaeaa02d 100644 --- a/coordinator/cloudprovider/gcp/ccm.go +++ b/coordinator/cloudprovider/gcp/ccm.go @@ -7,8 +7,8 @@ import ( "strings" "github.com/edgelesssys/constellation/coordinator/cloudprovider" - "github.com/edgelesssys/constellation/coordinator/cloudprovider/cloudtypes" "github.com/edgelesssys/constellation/coordinator/kubernetes/k8sapi/resources" + "github.com/edgelesssys/constellation/internal/cloud/metadata" "github.com/edgelesssys/constellation/internal/gcpshared" k8s "k8s.io/api/core/v1" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -46,7 +46,7 @@ func (c *CloudControllerManager) ExtraArgs() []string { // ConfigMaps returns a list of ConfigMaps to deploy together with the k8s cloud-controller-manager // Reference: https://kubernetes.io/docs/concepts/configuration/configmap/ . -func (c *CloudControllerManager) ConfigMaps(instance cloudtypes.Instance) (resources.ConfigMaps, error) { +func (c *CloudControllerManager) ConfigMaps(instance metadata.InstanceMetadata) (resources.ConfigMaps, error) { // GCP CCM expects cloud config to contain the GCP project-id and other configuration. // reference: https://github.com/kubernetes/cloud-provider-gcp/blob/master/cluster/gce/gci/configure-helper.sh#L791-L892 var config strings.Builder @@ -80,7 +80,7 @@ func (c *CloudControllerManager) ConfigMaps(instance cloudtypes.Instance) (resou // Secrets returns a list of secrets to deploy together with the k8s cloud-controller-manager. // Reference: https://kubernetes.io/docs/concepts/configuration/secret/ . -func (c *CloudControllerManager) Secrets(ctx context.Context, instance cloudtypes.Instance, cloudServiceAccountURI string) (resources.Secrets, error) { +func (c *CloudControllerManager) Secrets(ctx context.Context, _ string, cloudServiceAccountURI string) (resources.Secrets, error) { serviceAccountKey, err := gcpshared.ServiceAccountKeyFromURI(cloudServiceAccountURI) if err != nil { return resources.Secrets{}, err diff --git a/coordinator/cloudprovider/gcp/ccm_test.go b/coordinator/cloudprovider/gcp/ccm_test.go index bce2a0e32..25880ad54 100644 --- a/coordinator/cloudprovider/gcp/ccm_test.go +++ b/coordinator/cloudprovider/gcp/ccm_test.go @@ -5,8 +5,8 @@ import ( "encoding/json" "testing" - "github.com/edgelesssys/constellation/coordinator/cloudprovider/cloudtypes" "github.com/edgelesssys/constellation/coordinator/kubernetes/k8sapi/resources" + "github.com/edgelesssys/constellation/internal/cloud/metadata" "github.com/edgelesssys/constellation/internal/gcpshared" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -16,12 +16,12 @@ import ( func TestConfigMaps(t *testing.T) { testCases := map[string]struct { - instance cloudtypes.Instance + instance metadata.InstanceMetadata wantConfigMaps resources.ConfigMaps wantErr bool }{ "ConfigMaps works": { - instance: cloudtypes.Instance{ProviderID: "gce://project-id/zone/instanceName-UID-0", Name: "instanceName-UID-0"}, + instance: metadata.InstanceMetadata{ProviderID: "gce://project-id/zone/instanceName-UID-0", Name: "instanceName-UID-0"}, wantConfigMaps: resources.ConfigMaps{ &k8s.ConfigMap{ TypeMeta: v1.TypeMeta{ @@ -43,7 +43,7 @@ node-tags = constellation-UID }, }, "invalid providerID fails": { - instance: cloudtypes.Instance{ProviderID: "invalid"}, + instance: metadata.InstanceMetadata{ProviderID: "invalid"}, wantErr: true, }, } @@ -82,7 +82,7 @@ func TestSecrets(t *testing.T) { rawKey, err := json.Marshal(serviceAccountKey) require.NoError(t, err) testCases := map[string]struct { - instance cloudtypes.Instance + instance metadata.InstanceMetadata cloudServiceAccountURI string wantSecrets resources.Secrets wantErr bool @@ -117,7 +117,7 @@ func TestSecrets(t *testing.T) { require := require.New(t) cloud := CloudControllerManager{} - secrets, err := cloud.Secrets(context.Background(), tc.instance, tc.cloudServiceAccountURI) + secrets, err := cloud.Secrets(context.Background(), tc.instance.ProviderID, tc.cloudServiceAccountURI) if tc.wantErr { assert.Error(err) return diff --git a/coordinator/cloudprovider/gcp/client.go b/coordinator/cloudprovider/gcp/client.go index 3a2600db7..8a795cc45 100644 --- a/coordinator/cloudprovider/gcp/client.go +++ b/coordinator/cloudprovider/gcp/client.go @@ -7,16 +7,17 @@ import ( "strings" compute "cloud.google.com/go/compute/apiv1" - "github.com/edgelesssys/constellation/coordinator/cloudprovider" - "github.com/edgelesssys/constellation/coordinator/cloudprovider/cloudtypes" - "github.com/edgelesssys/constellation/coordinator/core" + "github.com/edgelesssys/constellation/internal/cloud/metadata" "github.com/edgelesssys/constellation/internal/gcpshared" "google.golang.org/api/iterator" computepb "google.golang.org/genproto/googleapis/cloud/compute/v1" "google.golang.org/protobuf/proto" ) -const gcpSSHMetadataKey = "ssh-keys" +const ( + gcpSSHMetadataKey = "ssh-keys" + constellationUIDMetadataKey = "constellation-uid" +) var zoneFromRegionRegex = regexp.MustCompile("([a-z]*-[a-z]*[0-9])") @@ -51,7 +52,7 @@ func NewClient(ctx context.Context) (*Client, error) { } // RetrieveInstances returns list of instances including their ips and metadata. -func (c *Client) RetrieveInstances(ctx context.Context, project, zone string) ([]cloudtypes.Instance, error) { +func (c *Client) RetrieveInstances(ctx context.Context, project, zone string) ([]metadata.InstanceMetadata, error) { uid, err := c.uid() if err != nil { return nil, err @@ -62,7 +63,7 @@ func (c *Client) RetrieveInstances(ctx context.Context, project, zone string) ([ } instanceIterator := c.instanceAPI.List(ctx, req) - instances := []cloudtypes.Instance{} + instances := []metadata.InstanceMetadata{} for { resp, err := instanceIterator.Next() if err == iterator.Done { @@ -73,7 +74,7 @@ func (c *Client) RetrieveInstances(ctx context.Context, project, zone string) ([ } metadata := extractInstanceMetadata(resp.Metadata, "", false) // skip instances not belonging to the current constellation - if instanceUID, ok := metadata[core.ConstellationUIDMetadataKey]; !ok || instanceUID != uid { + if instanceUID, ok := metadata[constellationUIDMetadataKey]; !ok || instanceUID != uid { continue } instance, err := convertToCoreInstance(resp, project, zone) @@ -87,10 +88,10 @@ func (c *Client) RetrieveInstances(ctx context.Context, project, zone string) ([ } // RetrieveInstance returns a an instance including ips and metadata. -func (c *Client) RetrieveInstance(ctx context.Context, project, zone, instanceName string) (cloudtypes.Instance, error) { +func (c *Client) RetrieveInstance(ctx context.Context, project, zone, instanceName string) (metadata.InstanceMetadata, error) { instance, err := c.getComputeInstance(ctx, project, zone, instanceName) if err != nil { - return cloudtypes.Instance{}, err + return metadata.InstanceMetadata{}, err } return convertToCoreInstance(instance, project, zone) @@ -286,7 +287,7 @@ func (c *Client) updateInstanceMetadata(ctx context.Context, project, zone, inst // uid retrieves the current instances uid. func (c *Client) uid() (string, error) { // API endpoint: http://metadata.google.internal/computeMetadata/v1/instance/attributes/constellation-uid - uid, err := c.RetrieveInstanceMetadata(core.ConstellationUIDMetadataKey) + uid, err := c.RetrieveInstanceMetadata(constellationUIDMetadataKey) if err != nil { return "", fmt.Errorf("retrieving constellation uid: %w", err) } @@ -367,19 +368,19 @@ func extractSSHKeys(metadata map[string]string) map[string][]string { } // convertToCoreInstance converts a *computepb.Instance to a core.Instance. -func convertToCoreInstance(in *computepb.Instance, project string, zone string) (cloudtypes.Instance, error) { +func convertToCoreInstance(in *computepb.Instance, project string, zone string) (metadata.InstanceMetadata, error) { if in.Name == nil { - return cloudtypes.Instance{}, fmt.Errorf("retrieving instance from compute API client returned invalid instance Name: %v", in.Name) + return metadata.InstanceMetadata{}, fmt.Errorf("retrieving instance from compute API client returned invalid instance Name: %v", in.Name) } - metadata := extractInstanceMetadata(in.Metadata, "", false) - return cloudtypes.Instance{ + mdata := extractInstanceMetadata(in.Metadata, "", false) + return metadata.InstanceMetadata{ Name: *in.Name, ProviderID: gcpshared.JoinProviderID(project, zone, *in.Name), - Role: cloudprovider.ExtractRole(metadata), + Role: extractRole(mdata), PrivateIPs: extractPrivateIPs(in.NetworkInterfaces), PublicIPs: extractPublicIPs(in.NetworkInterfaces), AliasIPRanges: extractAliasIPRanges(in.NetworkInterfaces), - SSHKeys: extractSSHKeys(metadata), + SSHKeys: extractSSHKeys(mdata), }, nil } diff --git a/coordinator/cloudprovider/gcp/client_test.go b/coordinator/cloudprovider/gcp/client_test.go index c7680e8b9..ff4864a26 100644 --- a/coordinator/cloudprovider/gcp/client_test.go +++ b/coordinator/cloudprovider/gcp/client_test.go @@ -6,9 +6,8 @@ import ( "testing" compute "cloud.google.com/go/compute/apiv1" - "github.com/edgelesssys/constellation/coordinator/cloudprovider/cloudtypes" - "github.com/edgelesssys/constellation/coordinator/core" "github.com/edgelesssys/constellation/coordinator/role" + "github.com/edgelesssys/constellation/internal/cloud/metadata" gax "github.com/googleapis/gax-go/v2" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -44,11 +43,11 @@ func TestRetrieveInstances(t *testing.T) { Value: proto.String("value-2"), }, { - Key: proto.String(core.ConstellationUIDMetadataKey), + Key: proto.String(constellationUIDMetadataKey), Value: proto.String(uid), }, { - Key: proto.String(core.RoleMetadataKey), + Key: proto.String(roleMetadataKey), Value: proto.String(role.Coordinator.String()), }, }, @@ -70,14 +69,14 @@ func TestRetrieveInstances(t *testing.T) { metadata stubMetadataClient instanceIter *stubInstanceIterator instanceIterMutator func(*stubInstanceIterator) - wantInstances []cloudtypes.Instance + wantInstances []metadata.InstanceMetadata wantErr bool }{ "retrieve works": { client: stubInstancesClient{}, metadata: stubMetadataClient{InstanceValue: uid}, instanceIter: newTestIter(), - wantInstances: []cloudtypes.Instance{ + wantInstances: []metadata.InstanceMetadata{ { Name: "someInstance", ProviderID: "gce://someProject/someZone/someInstance", @@ -101,7 +100,7 @@ func TestRetrieveInstances(t *testing.T) { metadata: stubMetadataClient{InstanceValue: uid}, instanceIter: newTestIter(), instanceIterMutator: func(sii *stubInstanceIterator) { sii.instances[0].NetworkInterfaces = nil }, - wantInstances: []cloudtypes.Instance{ + wantInstances: []metadata.InstanceMetadata{ { Name: "someInstance", ProviderID: "gce://someProject/someZone/someInstance", @@ -118,7 +117,7 @@ func TestRetrieveInstances(t *testing.T) { metadata: stubMetadataClient{InstanceValue: uid}, instanceIter: newTestIter(), instanceIterMutator: func(sii *stubInstanceIterator) { sii.instances[0].NetworkInterfaces[0].NetworkIP = nil }, - wantInstances: []cloudtypes.Instance{ + wantInstances: []metadata.InstanceMetadata{ { Name: "someInstance", ProviderID: "gce://someProject/someZone/someInstance", @@ -135,7 +134,7 @@ func TestRetrieveInstances(t *testing.T) { metadata: stubMetadataClient{InstanceValue: uid}, instanceIter: newTestIter(), instanceIterMutator: func(sii *stubInstanceIterator) { sii.instances[0].Metadata.Items[2].Key = proto.String("") }, - wantInstances: []cloudtypes.Instance{}, + wantInstances: []metadata.InstanceMetadata{}, }, "constellation retrieval fails": { client: stubInstancesClient{}, @@ -148,7 +147,7 @@ func TestRetrieveInstances(t *testing.T) { metadata: stubMetadataClient{InstanceValue: uid}, instanceIter: newTestIter(), instanceIterMutator: func(sii *stubInstanceIterator) { sii.instances[0].Metadata.Items[3].Key = proto.String("") }, - wantInstances: []cloudtypes.Instance{ + wantInstances: []metadata.InstanceMetadata{ { Name: "someInstance", ProviderID: "gce://someProject/someZone/someInstance", @@ -224,13 +223,13 @@ func TestRetrieveInstance(t *testing.T) { client stubInstancesClient clientInstance *computepb.Instance clientInstanceMutator func(*computepb.Instance) - wantInstance cloudtypes.Instance + wantInstance metadata.InstanceMetadata wantErr bool }{ "retrieve works": { client: stubInstancesClient{}, clientInstance: newTestInstance(), - wantInstance: cloudtypes.Instance{ + wantInstance: metadata.InstanceMetadata{ Name: "someInstance", ProviderID: "gce://someProject/someZone/someInstance", AliasIPRanges: []string{"192.0.2.0/16"}, @@ -246,7 +245,7 @@ func TestRetrieveInstance(t *testing.T) { i.Metadata.Items[0].Key = proto.String("ssh-keys") i.Metadata.Items[0].Value = proto.String("bob:ssh-rsa bobskey") }, - wantInstance: cloudtypes.Instance{ + wantInstance: metadata.InstanceMetadata{ Name: "someInstance", ProviderID: "gce://someProject/someZone/someInstance", AliasIPRanges: []string{"192.0.2.0/16"}, @@ -259,10 +258,10 @@ func TestRetrieveInstance(t *testing.T) { client: stubInstancesClient{}, clientInstance: newTestInstance(), clientInstanceMutator: func(i *computepb.Instance) { - i.Metadata.Items[0].Key = proto.String(core.RoleMetadataKey) + i.Metadata.Items[0].Key = proto.String(roleMetadataKey) i.Metadata.Items[0].Value = proto.String(role.Coordinator.String()) }, - wantInstance: cloudtypes.Instance{ + wantInstance: metadata.InstanceMetadata{ Name: "someInstance", ProviderID: "gce://someProject/someZone/someInstance", AliasIPRanges: []string{"192.0.2.0/16"}, @@ -283,7 +282,7 @@ func TestRetrieveInstance(t *testing.T) { client: stubInstancesClient{}, clientInstance: newTestInstance(), clientInstanceMutator: func(i *computepb.Instance) { i.Metadata.Items[0] = nil }, - wantInstance: cloudtypes.Instance{ + wantInstance: metadata.InstanceMetadata{ Name: "someInstance", ProviderID: "gce://someProject/someZone/someInstance", AliasIPRanges: []string{"192.0.2.0/16"}, @@ -296,7 +295,7 @@ func TestRetrieveInstance(t *testing.T) { client: stubInstancesClient{}, clientInstance: newTestInstance(), clientInstanceMutator: func(i *computepb.Instance) { i.Metadata.Items[0].Key = nil }, - wantInstance: cloudtypes.Instance{ + wantInstance: metadata.InstanceMetadata{ Name: "someInstance", ProviderID: "gce://someProject/someZone/someInstance", AliasIPRanges: []string{"192.0.2.0/16"}, @@ -309,7 +308,7 @@ func TestRetrieveInstance(t *testing.T) { client: stubInstancesClient{}, clientInstance: newTestInstance(), clientInstanceMutator: func(i *computepb.Instance) { i.Metadata.Items[0].Value = nil }, - wantInstance: cloudtypes.Instance{ + wantInstance: metadata.InstanceMetadata{ Name: "someInstance", ProviderID: "gce://someProject/someZone/someInstance", AliasIPRanges: []string{"192.0.2.0/16"}, @@ -322,7 +321,7 @@ func TestRetrieveInstance(t *testing.T) { client: stubInstancesClient{}, clientInstance: newTestInstance(), clientInstanceMutator: func(i *computepb.Instance) { i.NetworkInterfaces[0] = nil }, - wantInstance: cloudtypes.Instance{ + wantInstance: metadata.InstanceMetadata{ Name: "someInstance", ProviderID: "gce://someProject/someZone/someInstance", AliasIPRanges: []string{}, @@ -335,7 +334,7 @@ func TestRetrieveInstance(t *testing.T) { client: stubInstancesClient{}, clientInstance: newTestInstance(), clientInstanceMutator: func(i *computepb.Instance) { i.NetworkInterfaces[0].NetworkIP = nil }, - wantInstance: cloudtypes.Instance{ + wantInstance: metadata.InstanceMetadata{ Name: "someInstance", ProviderID: "gce://someProject/someZone/someInstance", AliasIPRanges: []string{"192.0.2.0/16"}, @@ -348,7 +347,7 @@ func TestRetrieveInstance(t *testing.T) { client: stubInstancesClient{}, clientInstance: newTestInstance(), clientInstanceMutator: func(i *computepb.Instance) { i.NetworkInterfaces[0].AliasIpRanges[0].IpCidrRange = nil }, - wantInstance: cloudtypes.Instance{ + wantInstance: metadata.InstanceMetadata{ Name: "someInstance", ProviderID: "gce://someProject/someZone/someInstance", AliasIPRanges: []string{}, @@ -361,7 +360,7 @@ func TestRetrieveInstance(t *testing.T) { client: stubInstancesClient{}, clientInstance: newTestInstance(), clientInstanceMutator: func(i *computepb.Instance) { i.NetworkInterfaces[0].AccessConfigs[0].NatIP = nil }, - wantInstance: cloudtypes.Instance{ + wantInstance: metadata.InstanceMetadata{ Name: "someInstance", ProviderID: "gce://someProject/someZone/someInstance", AliasIPRanges: []string{"192.0.2.0/16"}, diff --git a/coordinator/cloudprovider/gcp/metadata.go b/coordinator/cloudprovider/gcp/metadata.go index e2f54eb15..d02905f4b 100644 --- a/coordinator/cloudprovider/gcp/metadata.go +++ b/coordinator/cloudprovider/gcp/metadata.go @@ -4,18 +4,16 @@ import ( "context" "fmt" - "github.com/edgelesssys/constellation/coordinator/cloudprovider/cloudtypes" - "github.com/edgelesssys/constellation/coordinator/core" - "github.com/edgelesssys/constellation/coordinator/role" + "github.com/edgelesssys/constellation/internal/cloud/metadata" "github.com/edgelesssys/constellation/internal/gcpshared" ) // API handles all GCP API requests. type API interface { // RetrieveInstances retrieves a list of all accessible GCP instances with their metadata. - RetrieveInstances(ctx context.Context, project, zone string) ([]cloudtypes.Instance, error) + RetrieveInstances(ctx context.Context, project, zone string) ([]metadata.InstanceMetadata, error) // RetrieveInstances retrieves a single GCP instances with its metadata. - RetrieveInstance(ctx context.Context, project, zone, instanceName string) (cloudtypes.Instance, error) + RetrieveInstance(ctx context.Context, project, zone, instanceName string) (metadata.InstanceMetadata, error) // RetrieveInstanceMetadata retrieves the GCP instance metadata of the current instance. RetrieveInstanceMetadata(attr string) (string, error) // RetrieveProjectID retrieves the GCP projectID containing the current instance. @@ -47,7 +45,7 @@ func New(api API) *Metadata { } // List retrieves all instances belonging to the current constellation. -func (m *Metadata) List(ctx context.Context) ([]cloudtypes.Instance, error) { +func (m *Metadata) List(ctx context.Context) ([]metadata.InstanceMetadata, error) { project, err := m.api.RetrieveProjectID() if err != nil { return nil, err @@ -64,65 +62,31 @@ func (m *Metadata) List(ctx context.Context) ([]cloudtypes.Instance, error) { } // Self retrieves the current instance. -func (m *Metadata) Self(ctx context.Context) (cloudtypes.Instance, error) { +func (m *Metadata) Self(ctx context.Context) (metadata.InstanceMetadata, error) { project, err := m.api.RetrieveProjectID() if err != nil { - return cloudtypes.Instance{}, err + return metadata.InstanceMetadata{}, err } zone, err := m.api.RetrieveZone() if err != nil { - return cloudtypes.Instance{}, err + return metadata.InstanceMetadata{}, err } instanceName, err := m.api.RetrieveInstanceName() if err != nil { - return cloudtypes.Instance{}, err + return metadata.InstanceMetadata{}, err } return m.api.RetrieveInstance(ctx, project, zone, instanceName) } // GetInstance retrieves an instance using its providerID. -func (m *Metadata) GetInstance(ctx context.Context, providerID string) (cloudtypes.Instance, error) { +func (m *Metadata) GetInstance(ctx context.Context, providerID string) (metadata.InstanceMetadata, error) { project, zone, instanceName, err := gcpshared.SplitProviderID(providerID) if err != nil { - return cloudtypes.Instance{}, fmt.Errorf("invalid providerID: %w", err) + return metadata.InstanceMetadata{}, fmt.Errorf("invalid providerID: %w", err) } return m.api.RetrieveInstance(ctx, project, zone, instanceName) } -// SignalRole signals the constellation role via cloud provider metadata. -func (m *Metadata) SignalRole(ctx context.Context, role role.Role) error { - project, err := m.api.RetrieveProjectID() - if err != nil { - return err - } - zone, err := m.api.RetrieveZone() - if err != nil { - return err - } - instanceName, err := m.api.RetrieveInstanceName() - if err != nil { - return err - } - return m.api.SetInstanceMetadata(ctx, project, zone, instanceName, core.RoleMetadataKey, role.String()) -} - -// SetVPNIP stores the internally used VPN IP in cloud provider metadata. -func (m *Metadata) SetVPNIP(ctx context.Context, vpnIP string) error { - project, err := m.api.RetrieveProjectID() - if err != nil { - return err - } - zone, err := m.api.RetrieveZone() - if err != nil { - return err - } - instanceName, err := m.api.RetrieveInstanceName() - if err != nil { - return err - } - return m.api.SetInstanceMetadata(ctx, project, zone, instanceName, core.VPNIPMetadataKey, vpnIP) -} - // GetSubnetworkCIDR returns the subnetwork CIDR of the current instance. func (m *Metadata) GetSubnetworkCIDR(ctx context.Context) (string, error) { project, err := m.api.RetrieveProjectID() @@ -140,11 +104,6 @@ func (m *Metadata) GetSubnetworkCIDR(ctx context.Context) (string, error) { return m.api.RetrieveSubnetworkAliasCIDR(ctx, project, zone, instanceName) } -// SupportsLoadBalancer returns true if the cloud provider supports load balancers. -func (m *Metadata) SupportsLoadBalancer() bool { - return true -} - // GetLoadBalancerIP returns the IP of the load balancer. func (m *Metadata) GetLoadBalancerIP(ctx context.Context) (string, error) { project, err := m.api.RetrieveProjectID() @@ -157,8 +116,3 @@ func (m *Metadata) GetLoadBalancerIP(ctx context.Context) (string, error) { } return m.api.RetrieveLoadBalancerIP(ctx, project, zone) } - -// Supported is used to determine if metadata API is implemented for this cloud provider. -func (m *Metadata) Supported() bool { - return true -} diff --git a/coordinator/cloudprovider/gcp/metadata_test.go b/coordinator/cloudprovider/gcp/metadata_test.go index 1cc858e86..9da5e1c91 100644 --- a/coordinator/cloudprovider/gcp/metadata_test.go +++ b/coordinator/cloudprovider/gcp/metadata_test.go @@ -5,9 +5,7 @@ import ( "errors" "testing" - "github.com/edgelesssys/constellation/coordinator/cloudprovider/cloudtypes" - "github.com/edgelesssys/constellation/coordinator/core" - "github.com/edgelesssys/constellation/coordinator/role" + "github.com/edgelesssys/constellation/internal/cloud/metadata" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -15,8 +13,8 @@ import ( func TestList(t *testing.T) { err := errors.New("some err") uid := "1234" - instancesGenerator := func() *[]cloudtypes.Instance { - return &[]cloudtypes.Instance{ + instancesGenerator := func() *[]metadata.InstanceMetadata { + return &[]metadata.InstanceMetadata{ { Name: "someInstance", ProviderID: "gce://someProject/someZone/someInstance", @@ -27,10 +25,10 @@ func TestList(t *testing.T) { testCases := map[string]struct { client stubGCPClient - instancesGenerator func() *[]cloudtypes.Instance - instancesMutator func(*[]cloudtypes.Instance) + instancesGenerator func() *[]metadata.InstanceMetadata + instancesMutator func(*[]metadata.InstanceMetadata) wantErr bool - wantInstances []cloudtypes.Instance + wantInstances []metadata.InstanceMetadata }{ "retrieve works": { client: stubGCPClient{ @@ -41,7 +39,7 @@ func TestList(t *testing.T) { }, }, instancesGenerator: instancesGenerator, - wantInstances: []cloudtypes.Instance{ + wantInstances: []metadata.InstanceMetadata{ { Name: "someInstance", ProviderID: "gce://someProject/someZone/someInstance", @@ -106,19 +104,19 @@ func TestSelf(t *testing.T) { testCases := map[string]struct { client stubGCPClient wantErr bool - wantInstance cloudtypes.Instance + wantInstance metadata.InstanceMetadata }{ "retrieve works": { client: stubGCPClient{ projectID: "someProjectID", zone: "someZone", - retrieveInstanceValue: cloudtypes.Instance{ + retrieveInstanceValue: metadata.InstanceMetadata{ Name: "someInstance", ProviderID: "gce://someProject/someZone/someInstance", PrivateIPs: []string{"192.0.2.0"}, }, }, - wantInstance: cloudtypes.Instance{ + wantInstance: metadata.InstanceMetadata{ Name: "someInstance", ProviderID: "gce://someProject/someZone/someInstance", PrivateIPs: []string{"192.0.2.0"}, @@ -180,18 +178,18 @@ func TestGetInstance(t *testing.T) { providerID string client stubGCPClient wantErr bool - wantInstance cloudtypes.Instance + wantInstance metadata.InstanceMetadata }{ "retrieve works": { providerID: "gce://someProject/someZone/someInstance", client: stubGCPClient{ - retrieveInstanceValue: cloudtypes.Instance{ + retrieveInstanceValue: metadata.InstanceMetadata{ Name: "someInstance", ProviderID: "gce://someProject/someZone/someInstance", PrivateIPs: []string{"192.0.2.0"}, }, }, - wantInstance: cloudtypes.Instance{ + wantInstance: metadata.InstanceMetadata{ Name: "someInstance", ProviderID: "gce://someProject/someZone/someInstance", PrivateIPs: []string{"192.0.2.0"}, @@ -232,135 +230,10 @@ func TestGetInstance(t *testing.T) { } } -func TestSignalRole(t *testing.T) { - err := errors.New("some err") - - testCases := map[string]struct { - client stubGCPClient - wantErr bool - wantRole role.Role - }{ - "signaling role works": { - client: stubGCPClient{ - projectID: "someProjectID", - zone: "someZone", - instanceName: "someName", - }, - wantRole: role.Coordinator, - }, - "project metadata retrieval error is detected": { - client: stubGCPClient{ - retrieveProjectIDErr: err, - }, - wantErr: true, - }, - "instance zone retrieval error is detected": { - client: stubGCPClient{ - retrieveZoneErr: err, - }, - wantErr: true, - }, - "instance name retrieval error is detected": { - client: stubGCPClient{ - retrieveInstanceNameErr: err, - }, - wantErr: true, - }, - } - - for name, tc := range testCases { - t.Run(name, func(t *testing.T) { - assert := assert.New(t) - require := require.New(t) - - cloud := New(&tc.client) - err := cloud.SignalRole(context.Background(), tc.wantRole) - - if tc.wantErr { - assert.Error(err) - return - } - require.NoError(err) - - assert.ElementsMatch([]string{"someProjectID"}, tc.client.instanceMetadataProjects) - assert.ElementsMatch([]string{"someZone"}, tc.client.instanceMetadataZones) - assert.ElementsMatch([]string{"someName"}, tc.client.instanceMetadataInstanceNames) - assert.ElementsMatch([]string{core.RoleMetadataKey}, tc.client.instanceMetadataKeys) - assert.ElementsMatch([]string{tc.wantRole.String()}, tc.client.instanceMetadataValues) - }) - } -} - -func TestSetVPNIP(t *testing.T) { - err := errors.New("some err") - - testCases := map[string]struct { - client stubGCPClient - wantErr bool - wantVPNIP string - }{ - "signaling role works": { - client: stubGCPClient{ - projectID: "someProjectID", - zone: "someZone", - instanceName: "someName", - }, - wantVPNIP: "192.0.2.0", - }, - "project metadata retrieval error is detected": { - client: stubGCPClient{ - retrieveProjectIDErr: err, - }, - wantErr: true, - }, - "instance zone retrieval error is detected": { - client: stubGCPClient{ - retrieveZoneErr: err, - }, - wantErr: true, - }, - "instance name retrieval error is detected": { - client: stubGCPClient{ - retrieveInstanceNameErr: err, - }, - wantErr: true, - }, - } - - for name, tc := range testCases { - t.Run(name, func(t *testing.T) { - assert := assert.New(t) - require := require.New(t) - - cloud := New(&tc.client) - err := cloud.SetVPNIP(context.Background(), tc.wantVPNIP) - - if tc.wantErr { - assert.Error(err) - return - } - require.NoError(err) - - assert.ElementsMatch([]string{"someProjectID"}, tc.client.instanceMetadataProjects) - assert.ElementsMatch([]string{"someZone"}, tc.client.instanceMetadataZones) - assert.ElementsMatch([]string{"someName"}, tc.client.instanceMetadataInstanceNames) - assert.ElementsMatch([]string{core.VPNIPMetadataKey}, tc.client.instanceMetadataKeys) - assert.ElementsMatch([]string{tc.wantVPNIP}, tc.client.instanceMetadataValues) - }) - } -} - -func TestTrivialMetadataFunctions(t *testing.T) { - assert := assert.New(t) - metadata := Metadata{} - - assert.True(metadata.Supported()) -} - type stubGCPClient struct { - retrieveInstanceValue cloudtypes.Instance + retrieveInstanceValue metadata.InstanceMetadata retrieveInstanceErr error - retrieveInstancesValues []cloudtypes.Instance + retrieveInstancesValues []metadata.InstanceMetadata retrieveInstancesErr error retrieveInstanceMetadaValues map[string]string retrieveInstanceMetadataErr error @@ -388,11 +261,11 @@ type stubGCPClient struct { unsetMetadataKeys []string } -func (s *stubGCPClient) RetrieveInstances(ctx context.Context, project, zone string) ([]cloudtypes.Instance, error) { +func (s *stubGCPClient) RetrieveInstances(ctx context.Context, project, zone string) ([]metadata.InstanceMetadata, error) { return s.retrieveInstancesValues, s.retrieveInstancesErr } -func (s *stubGCPClient) RetrieveInstance(ctx context.Context, project, zone string, instanceName string) (cloudtypes.Instance, error) { +func (s *stubGCPClient) RetrieveInstance(ctx context.Context, project, zone string, instanceName string) (metadata.InstanceMetadata, error) { return s.retrieveInstanceValue, s.retrieveInstanceErr } diff --git a/coordinator/cloudprovider/gcp/role.go b/coordinator/cloudprovider/gcp/role.go new file mode 100644 index 000000000..48724a950 --- /dev/null +++ b/coordinator/cloudprovider/gcp/role.go @@ -0,0 +1,19 @@ +package gcp + +import ( + "github.com/edgelesssys/constellation/coordinator/role" +) + +const roleMetadataKey = "constellation-role" + +// extractRole extracts role from cloud provider metadata. +func extractRole(metadata map[string]string) role.Role { + switch metadata[roleMetadataKey] { + case role.Coordinator.String(): + return role.Coordinator + case role.Node.String(): + return role.Node + default: + return role.Unknown + } +} diff --git a/coordinator/cloudprovider/role_test.go b/coordinator/cloudprovider/gcp/role_test.go similarity index 78% rename from coordinator/cloudprovider/role_test.go rename to coordinator/cloudprovider/gcp/role_test.go index bb2187f4e..e5c7c1f34 100644 --- a/coordinator/cloudprovider/role_test.go +++ b/coordinator/cloudprovider/gcp/role_test.go @@ -1,9 +1,8 @@ -package cloudprovider +package gcp import ( "testing" - "github.com/edgelesssys/constellation/coordinator/core" "github.com/edgelesssys/constellation/coordinator/role" "github.com/stretchr/testify/assert" "go.uber.org/goleak" @@ -23,19 +22,19 @@ func TestExtractRole(t *testing.T) { }{ "coordinator role": { metadata: map[string]string{ - core.RoleMetadataKey: role.Coordinator.String(), + roleMetadataKey: role.Coordinator.String(), }, wantRole: role.Coordinator, }, "node role": { metadata: map[string]string{ - core.RoleMetadataKey: role.Node.String(), + roleMetadataKey: role.Node.String(), }, wantRole: role.Node, }, "unknown role": { metadata: map[string]string{ - core.RoleMetadataKey: "some-unknown-role", + roleMetadataKey: "some-unknown-role", }, wantRole: role.Unknown, }, @@ -48,7 +47,7 @@ func TestExtractRole(t *testing.T) { t.Run(name, func(t *testing.T) { assert := assert.New(t) - role := ExtractRole(tc.metadata) + role := extractRole(tc.metadata) assert.Equal(tc.wantRole, role) }) diff --git a/coordinator/cloudprovider/qemu/autoscaler.go b/coordinator/cloudprovider/qemu/autoscaler.go index 58533febf..84b7d34df 100644 --- a/coordinator/cloudprovider/qemu/autoscaler.go +++ b/coordinator/cloudprovider/qemu/autoscaler.go @@ -1,8 +1,8 @@ package qemu import ( - "github.com/edgelesssys/constellation/coordinator/cloudprovider/cloudtypes" "github.com/edgelesssys/constellation/coordinator/kubernetes/k8sapi/resources" + "github.com/edgelesssys/constellation/internal/cloud/metadata" k8s "k8s.io/api/core/v1" ) @@ -15,7 +15,7 @@ func (a Autoscaler) Name() string { } // Secrets returns a list of secrets to deploy together with the k8s cluster-autoscaler. -func (a Autoscaler) Secrets(instance cloudtypes.Instance, cloudServiceAccountURI string) (resources.Secrets, error) { +func (a Autoscaler) Secrets(instance metadata.InstanceMetadata, cloudServiceAccountURI string) (resources.Secrets, error) { return resources.Secrets{}, nil } diff --git a/coordinator/cloudprovider/qemu/ccm.go b/coordinator/cloudprovider/qemu/ccm.go index b01515aa0..d8b76a9b2 100644 --- a/coordinator/cloudprovider/qemu/ccm.go +++ b/coordinator/cloudprovider/qemu/ccm.go @@ -3,8 +3,8 @@ package qemu import ( "context" - "github.com/edgelesssys/constellation/coordinator/cloudprovider/cloudtypes" "github.com/edgelesssys/constellation/coordinator/kubernetes/k8sapi/resources" + "github.com/edgelesssys/constellation/internal/cloud/metadata" k8s "k8s.io/api/core/v1" ) @@ -33,13 +33,13 @@ func (c CloudControllerManager) ExtraArgs() []string { // ConfigMaps returns a list of ConfigMaps to deploy together with the k8s cloud-controller-manager // Reference: https://kubernetes.io/docs/concepts/configuration/configmap/ . -func (c CloudControllerManager) ConfigMaps(instance cloudtypes.Instance) (resources.ConfigMaps, error) { +func (c CloudControllerManager) ConfigMaps(instance metadata.InstanceMetadata) (resources.ConfigMaps, error) { return resources.ConfigMaps{}, nil } // Secrets returns a list of secrets to deploy together with the k8s cloud-controller-manager. // Reference: https://kubernetes.io/docs/concepts/configuration/secret/ . -func (c CloudControllerManager) Secrets(ctx context.Context, instance cloudtypes.Instance, cloudServiceAccountURI string) (resources.Secrets, error) { +func (c CloudControllerManager) Secrets(ctx context.Context, instance metadata.InstanceMetadata, cloudServiceAccountURI string) (resources.Secrets, error) { return resources.Secrets{}, nil } @@ -61,7 +61,7 @@ func (c CloudControllerManager) Env() []k8s.EnvVar { // PrepareInstance is called on every instance before deploying the cloud-controller-manager. // Allows for cloud-provider specific hooks. -func (c CloudControllerManager) PrepareInstance(instance cloudtypes.Instance, vpnIP string) error { +func (c CloudControllerManager) PrepareInstance(instance metadata.InstanceMetadata, vpnIP string) error { // no specific hook required. return nil } diff --git a/coordinator/cloudprovider/qemu/metadata.go b/coordinator/cloudprovider/qemu/metadata.go index e269ee173..0ba4530b1 100644 --- a/coordinator/cloudprovider/qemu/metadata.go +++ b/coordinator/cloudprovider/qemu/metadata.go @@ -8,8 +8,8 @@ import ( "net/http" "net/url" - "github.com/edgelesssys/constellation/coordinator/cloudprovider/cloudtypes" "github.com/edgelesssys/constellation/coordinator/role" + "github.com/edgelesssys/constellation/internal/cloud/metadata" ) const qemuMetadataEndpoint = "10.42.0.1:8080" @@ -23,34 +23,34 @@ func (m *Metadata) Supported() bool { } // List retrieves all instances belonging to the current constellation. -func (m *Metadata) List(ctx context.Context) ([]cloudtypes.Instance, error) { +func (m *Metadata) List(ctx context.Context) ([]metadata.InstanceMetadata, error) { instancesRaw, err := m.retrieveMetadata(ctx, "/peers") if err != nil { return nil, err } - var instances []cloudtypes.Instance + var instances []metadata.InstanceMetadata err = json.Unmarshal(instancesRaw, &instances) return instances, err } // Self retrieves the current instance. -func (m *Metadata) Self(ctx context.Context) (cloudtypes.Instance, error) { +func (m *Metadata) Self(ctx context.Context) (metadata.InstanceMetadata, error) { instanceRaw, err := m.retrieveMetadata(ctx, "/self") if err != nil { - return cloudtypes.Instance{}, err + return metadata.InstanceMetadata{}, err } - var instance cloudtypes.Instance + var instance metadata.InstanceMetadata err = json.Unmarshal(instanceRaw, &instance) return instance, err } // GetInstance retrieves an instance using its providerID. -func (m Metadata) GetInstance(ctx context.Context, providerID string) (cloudtypes.Instance, error) { +func (m Metadata) GetInstance(ctx context.Context, providerID string) (metadata.InstanceMetadata, error) { instances, err := m.List(ctx) if err != nil { - return cloudtypes.Instance{}, err + return metadata.InstanceMetadata{}, err } for _, instance := range instances { @@ -58,7 +58,7 @@ func (m Metadata) GetInstance(ctx context.Context, providerID string) (cloudtype return instance, nil } } - return cloudtypes.Instance{}, errors.New("instance not found") + return metadata.InstanceMetadata{}, errors.New("instance not found") } // SignalRole signals the constellation role via cloud provider metadata (if supported by the CSP and deployment type, otherwise does nothing). diff --git a/coordinator/cloudprovider/role.go b/coordinator/cloudprovider/role.go deleted file mode 100644 index a8284b463..000000000 --- a/coordinator/cloudprovider/role.go +++ /dev/null @@ -1,18 +0,0 @@ -package cloudprovider - -import ( - "github.com/edgelesssys/constellation/coordinator/core" - "github.com/edgelesssys/constellation/coordinator/role" -) - -// ExtractRole extracts role from cloud provider metadata. -func ExtractRole(metadata map[string]string) role.Role { - switch metadata[core.RoleMetadataKey] { - case role.Coordinator.String(): - return role.Coordinator - case role.Node.String(): - return role.Node - default: - return role.Unknown - } -} diff --git a/coordinator/kubernetes/kubernetes.go b/coordinator/kubernetes/kubernetes.go index ed8fc3238..7b52c5b11 100644 --- a/coordinator/kubernetes/kubernetes.go +++ b/coordinator/kubernetes/kubernetes.go @@ -282,7 +282,7 @@ func (k *KubeWrapper) setupCCM(ctx context.Context, subnetworkPodCIDR, cloudServ if err != nil { return fmt.Errorf("defining ConfigMaps for CCM failed: %w", err) } - ccmSecrets, err := k.cloudControllerManager.Secrets(ctx, instance, cloudServiceAccountURI) + ccmSecrets, err := k.cloudControllerManager.Secrets(ctx, instance.ProviderID, cloudServiceAccountURI) if err != nil { return fmt.Errorf("defining Secrets for CCM failed: %w", err) } @@ -316,7 +316,7 @@ func (k *KubeWrapper) setupClusterAutoscaler(instance cloudtypes.Instance, cloud if !k.clusterAutoscaler.Supported() { return nil } - caSecrets, err := k.clusterAutoscaler.Secrets(instance, cloudServiceAccountURI) + caSecrets, err := k.clusterAutoscaler.Secrets(instance.ProviderID, cloudServiceAccountURI) if err != nil { return fmt.Errorf("defining Secrets for cluster-autoscaler failed: %w", err) } diff --git a/internal/azureshared/metadata.go b/internal/azureshared/metadata.go index 7003fed1c..9a2e9ab45 100644 --- a/internal/azureshared/metadata.go +++ b/internal/azureshared/metadata.go @@ -7,17 +7,10 @@ import ( "strings" ) -var ( - azureVMProviderIDRegexp = regexp.MustCompile(`^azure:///subscriptions/([^/]+)/resourceGroups/([^/]+)/providers/Microsoft.Compute/virtualMachines/([^/]+)$`) - azureVMSSProviderIDRegexp = regexp.MustCompile(`^azure:///subscriptions/([^/]+)/resourceGroups/([^/]+)/providers/Microsoft.Compute/virtualMachineScaleSets/([^/]+)/virtualMachines/([^/]+)$`) -) +var azureVMSSProviderIDRegexp = regexp.MustCompile(`^azure:///subscriptions/([^/]+)/resourceGroups/([^/]+)/providers/Microsoft.Compute/virtualMachineScaleSets/([^/]+)/virtualMachines/([^/]+)$`) // BasicsFromProviderID extracts subscriptionID and resourceGroup from both types of valid azure providerID. func BasicsFromProviderID(providerID string) (subscriptionID, resourceGroup string, err error) { - subscriptionID, resourceGroup, _, err = VMInformationFromProviderID(providerID) - if err == nil { - return subscriptionID, resourceGroup, nil - } subscriptionID, resourceGroup, _, _, err = ScaleSetInformationFromProviderID(providerID) if err == nil { return subscriptionID, resourceGroup, nil @@ -29,7 +22,7 @@ func BasicsFromProviderID(providerID string) (subscriptionID, resourceGroup stri // suffix at the resource group, e.g., resource-group-J18dB // J18dB is the UID. func UIDFromProviderID(providerID string) (string, error) { - _, resourceGroup, err := BasicsFromProviderID(providerID) + _, resourceGroup, _, _, err := ScaleSetInformationFromProviderID(providerID) if err != nil { return "", err } @@ -38,17 +31,6 @@ func UIDFromProviderID(providerID string) (string, error) { return parts[len(parts)-1], nil } -// VMInformationFromProviderID splits a provider's id belonging to a single azure instance into core components. -// A providerID for individual VMs is build after the following schema: -// - 'azure:///subscriptions//resourceGroups//providers/Microsoft.Compute/virtualMachines/' -func VMInformationFromProviderID(providerID string) (subscriptionID, resourceGroup, instanceName string, err error) { - matches := azureVMProviderIDRegexp.FindStringSubmatch(providerID) - if len(matches) != 4 { - return "", "", "", errors.New("error splitting providerID") - } - return matches[1], matches[2], matches[3], nil -} - // ScaleSetInformationFromProviderID splits a provider's id belonging to an azure scaleset into core components. // A providerID for scale set VMs is build after the following schema: // - 'azure:///subscriptions//resourceGroups//providers/Microsoft.Compute/virtualMachineScaleSets//virtualMachines/' diff --git a/internal/cloud/metadata/metadata.go b/internal/cloud/metadata/metadata.go index 7f8905aae..3d9af6e15 100644 --- a/internal/cloud/metadata/metadata.go +++ b/internal/cloud/metadata/metadata.go @@ -2,7 +2,6 @@ package metadata import ( "context" - "errors" "fmt" "net" "strconv" @@ -36,24 +35,30 @@ type metadataAPI interface { Supported() bool } -// TODO(katexochen): Rename to InitEndpoints -func CoordinatorEndpoints(ctx context.Context, api metadataAPI) ([]string, error) { - if !api.Supported() { - return nil, errors.New("retrieving instances list from cloud provider is not yet supported") - } - instances, err := api.List(ctx) +type InstanceSelfer interface { + // Self retrieves the current instance. + Self(ctx context.Context) (InstanceMetadata, error) +} + +type InstanceLister interface { + // List retrieves all instances belonging to the current constellation. + List(ctx context.Context) ([]InstanceMetadata, error) +} + +func InitServerEndpoints(ctx context.Context, lister InstanceLister) ([]string, error) { + instances, err := lister.List(ctx) if err != nil { return nil, fmt.Errorf("retrieving instances list from cloud provider: %w", err) } - coordinatorEndpoints := []string{} + initServerEndpoints := []string{} for _, instance := range instances { // check if role of instance is "Coordinator" if instance.Role == role.Coordinator { for _, ip := range instance.PrivateIPs { - coordinatorEndpoints = append(coordinatorEndpoints, net.JoinHostPort(ip, strconv.Itoa(constants.CoordinatorPort))) + initServerEndpoints = append(initServerEndpoints, net.JoinHostPort(ip, strconv.Itoa(constants.CoordinatorPort))) } } } - return coordinatorEndpoints, nil + return initServerEndpoints, nil } diff --git a/internal/constants/constants.go b/internal/constants/constants.go index e40eec799..576fd17f5 100644 --- a/internal/constants/constants.go +++ b/internal/constants/constants.go @@ -50,7 +50,6 @@ const ( // // Filenames. // - StateFilename = "constellation-state.json" ClusterIDsFileName = "constellation-id.json" ConfigFilename = "constellation-conf.yaml"