Use tags for UID and role parsing (#242)

* Apply tags to all applicable GCP resources

* Move GCP UID and role from VM metadata to labels

* Adjust Azure tags to be in line with GCP and AWS

* Dont rely on resource name to find resources

Signed-off-by: Daniel Weiße <dw@edgeless.systems>

internal/cloud/gcp/api.go

@@ -35,6 +35,7 @@ type forwardingRulesAPI interface {
 
 type metadataAPI interface {
 	InstanceAttributeValue(attr string) (string, error)
+	InstanceID() (string, error)
 	ProjectID() (string, error)
 	Zone() (string, error)
 	InstanceName() (string, error)

internal/cloud/gcp/ccm.go

@@ -26,8 +26,8 @@ type CloudControllerManager struct {
 }
 
 // NewCloudControllerManager returns an initialized cloud controller manager configuration struct for GCP.
-func NewCloudControllerManager(metadata *Metadata) (*CloudControllerManager, error) {
-	uid, err := metadata.api.UID()
+func NewCloudControllerManager(ctx context.Context, metadata *Metadata) (*CloudControllerManager, error) {
+	uid, err := metadata.api.UID(ctx)
 	if err != nil {
 		return nil, fmt.Errorf("getting uid from metadata: %w", err)
 	}

internal/cloud/gcp/client.go

@@ -15,16 +15,17 @@ import (
 	"strings"
 
 	compute "cloud.google.com/go/compute/apiv1"
+	"github.com/edgelesssys/constellation/v2/internal/cloud"
 	"github.com/edgelesssys/constellation/v2/internal/cloud/metadata"
 	"github.com/edgelesssys/constellation/v2/internal/gcpshared"
+	"github.com/edgelesssys/constellation/v2/internal/role"
 	"google.golang.org/api/iterator"
 	computepb "google.golang.org/genproto/googleapis/cloud/compute/v1"
 	"google.golang.org/protobuf/proto"
 )
 
 const (
-	gcpSSHMetadataKey           = "ssh-keys"
-	constellationUIDMetadataKey = "constellation-uid"
+	gcpSSHMetadataKey = "ssh-keys"
 )
 
 var zoneFromRegionRegex = regexp.MustCompile("([a-z]*-[a-z]*[0-9])")
@@ -61,11 +62,12 @@ func NewClient(ctx context.Context) (*Client, error) {
 
 // RetrieveInstances returns list of instances including their ips and metadata.
 func (c *Client) RetrieveInstances(ctx context.Context, project, zone string) ([]metadata.InstanceMetadata, error) {
-	uid, err := c.UID()
+	uid, err := c.UID(ctx)
 	if err != nil {
 		return nil, err
 	}
 	req := &computepb.ListInstancesRequest{
+		Filter:  proto.String(fmt.Sprintf("labels.%s:%s", cloud.TagUID, uid)),
 		Project: project,
 		Zone:    zone,
 	}
@@ -80,11 +82,6 @@ func (c *Client) RetrieveInstances(ctx context.Context, project, zone string) ([
 		if err != nil {
 			return nil, fmt.Errorf("retrieving instance list from compute API client: %w", err)
 		}
-		metadata := extractInstanceMetadata(resp.Metadata, "", false)
-		// skip instances not belonging to the current constellation
-		if instanceUID, ok := metadata[constellationUIDMetadataKey]; !ok || instanceUID != uid {
-			continue
-		}
 		instance, err := convertToCoreInstance(resp, project, zone)
 		if err != nil {
 			return nil, err
@@ -223,7 +220,7 @@ func (c *Client) RetrieveSubnetworkAliasCIDR(ctx context.Context, project, zone,
 
 // RetrieveLoadBalancerEndpoint returns the endpoint of the load balancer with the constellation-uid tag.
 func (c *Client) RetrieveLoadBalancerEndpoint(ctx context.Context, project string) (string, error) {
-	uid, err := c.UID()
+	uid, err := c.UID(ctx)
 	if err != nil {
 		return "", err
 	}
@@ -240,7 +237,7 @@ func (c *Client) RetrieveLoadBalancerEndpoint(ctx context.Context, project strin
 		if err != nil {
 			return "", fmt.Errorf("retrieving load balancer IP failed: %w", err)
 		}
-		if resp.Labels["constellation-uid"] == uid {
+		if resp.Labels[cloud.TagUID] == uid && resp.Labels["constellation-use"] == "kubernetes" {
 			if resp.PortRange == nil {
 				return "", errors.New("load balancer with searched UID has no ports")
 			}
@@ -292,13 +289,30 @@ func (c *Client) updateInstanceMetadata(ctx context.Context, project, zone, inst
 }
 
 // UID retrieves the current instances uid.
-func (c *Client) UID() (string, error) {
+func (c *Client) UID(ctx context.Context) (string, error) {
 	// API endpoint: http://metadata.google.internal/computeMetadata/v1/instance/attributes/constellation-uid
-	uid, err := c.RetrieveInstanceMetadata(constellationUIDMetadataKey)
+	instanceID, err := c.InstanceID()
 	if err != nil {
-		return "", fmt.Errorf("retrieving constellation uid: %w", err)
+		return "", fmt.Errorf("retrieving instance ID: %w", err)
 	}
-	return uid, nil
+	project, err := c.ProjectID()
+	if err != nil {
+		return "", fmt.Errorf("retrieving project ID: %w", err)
+	}
+	zone, err := c.Zone()
+	if err != nil {
+		return "", fmt.Errorf("retrieving zone: %w", err)
+	}
+
+	instance, err := c.instanceAPI.Get(ctx, &computepb.GetInstanceRequest{
+		Project:  project,
+		Zone:     zone,
+		Instance: instanceID,
+	})
+	if err != nil {
+		return "", fmt.Errorf("retrieving instance labels: %w", err)
+	}
+	return instance.Labels[cloud.TagUID], nil
 }
 
 // extractVPCIP extracts the primary private IP from a list of interfaces.
@@ -385,7 +399,7 @@ func convertToCoreInstance(in *computepb.Instance, project string, zone string)
 	return metadata.InstanceMetadata{
 		Name:          *in.Name,
 		ProviderID:    gcpshared.JoinProviderID(project, zone, *in.Name),
-		Role:          extractRole(mdata),
+		Role:          role.FromString(in.Labels[cloud.TagRole]),
 		VPCIP:         extractVPCIP(in.NetworkInterfaces),
 		PublicIP:      extractPublicIP(in.NetworkInterfaces),
 		AliasIPRanges: extractAliasIPRanges(in.NetworkInterfaces),

internal/cloud/gcp/client_test.go

@@ -12,6 +12,7 @@ import (
 	"testing"
 
 	compute "cloud.google.com/go/compute/apiv1"
+	"github.com/edgelesssys/constellation/v2/internal/cloud"
 	"github.com/edgelesssys/constellation/v2/internal/cloud/metadata"
 	"github.com/edgelesssys/constellation/v2/internal/role"
 	gax "github.com/googleapis/gax-go/v2"
@@ -38,6 +39,10 @@ func TestRetrieveInstances(t *testing.T) {
 			instances: []*computepb.Instance{
 				{
 					Name: proto.String("someInstance"),
+					Labels: map[string]string{
+						cloud.TagRole: role.ControlPlane.String(),
+						cloud.TagUID:  uid,
+					},
 					Metadata: &computepb.Metadata{
 						Items: []*computepb.Items{
 							{
@@ -48,14 +53,6 @@ func TestRetrieveInstances(t *testing.T) {
 								Key:   proto.String("key-2"),
 								Value: proto.String("value-2"),
 							},
-							{
-								Key:   proto.String(constellationUIDMetadataKey),
-								Value: proto.String(uid),
-							},
-							{
-								Key:   proto.String(roleMetadataKey),
-								Value: proto.String(role.ControlPlane.String()),
-							},
 						},
 					},
 					NetworkInterfaces: []*computepb.NetworkInterface{
@@ -70,6 +67,13 @@ func TestRetrieveInstances(t *testing.T) {
 			},
 		}
 	}
+	instance := &computepb.Instance{
+		Name: proto.String("instance"),
+		Labels: map[string]string{
+			cloud.TagRole: role.ControlPlane.String(),
+			cloud.TagUID:  uid,
+		},
+	}
 
 	testCases := map[string]struct {
 		client              stubInstancesClient
@@ -80,7 +84,7 @@ func TestRetrieveInstances(t *testing.T) {
 		wantErr             bool
 	}{
 		"retrieve works": {
-			client:       stubInstancesClient{},
+			client:       stubInstancesClient{GetInstance: instance},
 			metadata:     stubMetadataClient{InstanceValue: uid},
 			instanceIter: newTestIter(),
 			wantInstances: []metadata.InstanceMetadata{
@@ -96,14 +100,14 @@ func TestRetrieveInstances(t *testing.T) {
 			},
 		},
 		"instance name is null": {
-			client:              stubInstancesClient{},
+			client:              stubInstancesClient{GetInstance: instance},
 			metadata:            stubMetadataClient{InstanceValue: uid},
 			instanceIter:        newTestIter(),
 			instanceIterMutator: func(sii *stubInstanceIterator) { sii.instances[0].Name = nil },
 			wantErr:             true,
 		},
 		"no instance with network ip": {
-			client:              stubInstancesClient{},
+			client:              stubInstancesClient{GetInstance: instance},
 			metadata:            stubMetadataClient{InstanceValue: uid},
 			instanceIter:        newTestIter(),
 			instanceIterMutator: func(sii *stubInstanceIterator) { sii.instances[0].NetworkInterfaces = nil },
@@ -120,7 +124,7 @@ func TestRetrieveInstances(t *testing.T) {
 			},
 		},
 		"network ip is nil": {
-			client:              stubInstancesClient{},
+			client:              stubInstancesClient{GetInstance: instance},
 			metadata:            stubMetadataClient{InstanceValue: uid},
 			instanceIter:        newTestIter(),
 			instanceIterMutator: func(sii *stubInstanceIterator) { sii.instances[0].NetworkInterfaces[0].NetworkIP = nil },
@@ -136,24 +140,17 @@ func TestRetrieveInstances(t *testing.T) {
 				},
 			},
 		},
-		"constellation id is not set": {
-			client:              stubInstancesClient{},
-			metadata:            stubMetadataClient{InstanceValue: uid},
-			instanceIter:        newTestIter(),
-			instanceIterMutator: func(sii *stubInstanceIterator) { sii.instances[0].Metadata.Items[2].Key = proto.String("") },
-			wantInstances:       []metadata.InstanceMetadata{},
-		},
 		"constellation retrieval fails": {
-			client:       stubInstancesClient{},
-			metadata:     stubMetadataClient{InstanceErr: someErr},
+			client:       stubInstancesClient{GetInstance: instance},
+			metadata:     stubMetadataClient{instanceIDErr: someErr},
 			instanceIter: newTestIter(),
 			wantErr:      true,
 		},
 		"role is not set": {
-			client:              stubInstancesClient{},
+			client:              stubInstancesClient{GetInstance: instance},
 			metadata:            stubMetadataClient{InstanceValue: uid},
 			instanceIter:        newTestIter(),
-			instanceIterMutator: func(sii *stubInstanceIterator) { sii.instances[0].Metadata.Items[3].Key = proto.String("") },
+			instanceIterMutator: func(sii *stubInstanceIterator) { delete(sii.instances[0].Labels, cloud.TagRole) },
 			wantInstances: []metadata.InstanceMetadata{
 				{
 					Name:          "someInstance",
@@ -167,7 +164,7 @@ func TestRetrieveInstances(t *testing.T) {
 			},
 		},
 		"instance iterator Next() errors": {
-			client:       stubInstancesClient{},
+			client:       stubInstancesClient{GetInstance: instance},
 			metadata:     stubMetadataClient{InstanceValue: uid},
 			instanceIter: &stubInstanceIterator{nextErr: someErr},
 			wantErr:      true,
@@ -203,7 +200,8 @@ func TestRetrieveInstances(t *testing.T) {
 func TestRetrieveInstance(t *testing.T) {
 	newTestInstance := func() *computepb.Instance {
 		return &computepb.Instance{
-			Name: proto.String("someInstance"),
+			Name:   proto.String("someInstance"),
+			Labels: map[string]string{},
 			Metadata: &computepb.Metadata{
 				Items: []*computepb.Items{
 					{
@@ -266,8 +264,7 @@ func TestRetrieveInstance(t *testing.T) {
 			client:         stubInstancesClient{},
 			clientInstance: newTestInstance(),
 			clientInstanceMutator: func(i *computepb.Instance) {
-				i.Metadata.Items[0].Key = proto.String(roleMetadataKey)
-				i.Metadata.Items[0].Value = proto.String(role.ControlPlane.String())
+				i.Labels[cloud.TagRole] = role.ControlPlane.String()
 			},
 			wantInstance: metadata.InstanceMetadata{
 				Name:          "someInstance",
@@ -782,22 +779,31 @@ func TestRetrieveSubnetworkAliasCIDR(t *testing.T) {
 func TestRetrieveLoadBalancerEndpoint(t *testing.T) {
 	loadBalancerIP := "192.0.2.1"
 	uid := "uid"
+	use := "kubernetes"
 	someErr := errors.New("some error")
+	instance := &computepb.Instance{
+		Labels: map[string]string{
+			cloud.TagUID: uid,
+		},
+	}
+
 	testCases := map[string]struct {
+		instanceAPI               stubInstancesClient
 		stubForwardingRulesClient stubForwardingRulesClient
 		stubMetadataClient        stubMetadataClient
 		wantLoadBalancerIP        string
 		wantErr                   bool
 	}{
 		"works": {
-			stubMetadataClient: stubMetadataClient{InstanceValue: uid},
+			instanceAPI:        stubInstancesClient{GetInstance: instance},
+			stubMetadataClient: stubMetadataClient{},
 			stubForwardingRulesClient: stubForwardingRulesClient{
 				ForwardingRuleIterator: &stubForwardingRuleIterator{
 					rules: []*computepb.ForwardingRule{
 						{
 							IPAddress: proto.String(loadBalancerIP),
 							PortRange: proto.String("100-100"),
-							Labels:    map[string]string{"constellation-uid": uid},
+							Labels:    map[string]string{cloud.TagUID: uid, "constellation-use": use},
 						},
 					},
 				},
@@ -805,7 +811,8 @@ func TestRetrieveLoadBalancerEndpoint(t *testing.T) {
 			wantLoadBalancerIP: loadBalancerIP,
 		},
 		"fails when no matching load balancers exists": {
-			stubMetadataClient: stubMetadataClient{InstanceValue: uid},
+			instanceAPI:        stubInstancesClient{GetInstance: instance},
+			stubMetadataClient: stubMetadataClient{},
 			stubForwardingRulesClient: stubForwardingRulesClient{
 				ForwardingRuleIterator: &stubForwardingRuleIterator{
 					rules: []*computepb.ForwardingRule{
@@ -819,14 +826,15 @@ func TestRetrieveLoadBalancerEndpoint(t *testing.T) {
 			wantErr: true,
 		},
 		"fails when retrieving uid": {
-			stubMetadataClient: stubMetadataClient{InstanceErr: someErr},
+			instanceAPI:        stubInstancesClient{GetInstance: instance},
+			stubMetadataClient: stubMetadataClient{instanceIDErr: someErr},
 			stubForwardingRulesClient: stubForwardingRulesClient{
 				ForwardingRuleIterator: &stubForwardingRuleIterator{
 					rules: []*computepb.ForwardingRule{
 						{
 							IPAddress: proto.String(loadBalancerIP),
 							PortRange: proto.String("100-100"),
-							Labels:    map[string]string{"constellation-uid": uid},
+							Labels:    map[string]string{cloud.TagUID: uid, "constellation-use": use},
 						},
 					},
 				},
@@ -834,13 +842,14 @@ func TestRetrieveLoadBalancerEndpoint(t *testing.T) {
 			wantErr: true,
 		},
 		"fails when answer has empty port range": {
-			stubMetadataClient: stubMetadataClient{InstanceErr: someErr},
+			instanceAPI:        stubInstancesClient{GetInstance: instance},
+			stubMetadataClient: stubMetadataClient{},
 			stubForwardingRulesClient: stubForwardingRulesClient{
 				ForwardingRuleIterator: &stubForwardingRuleIterator{
 					rules: []*computepb.ForwardingRule{
 						{
 							IPAddress: proto.String(loadBalancerIP),
-							Labels:    map[string]string{"constellation-uid": uid},
+							Labels:    map[string]string{cloud.TagUID: uid, "constellation-use": use},
 						},
 					},
 				},
@@ -848,6 +857,7 @@ func TestRetrieveLoadBalancerEndpoint(t *testing.T) {
 			wantErr: true,
 		},
 		"fails when retrieving loadbalancer IP": {
+			instanceAPI:        stubInstancesClient{GetInstance: instance},
 			stubMetadataClient: stubMetadataClient{},
 			stubForwardingRulesClient: stubForwardingRulesClient{
 				ForwardingRuleIterator: &stubForwardingRuleIterator{
@@ -856,7 +866,23 @@ func TestRetrieveLoadBalancerEndpoint(t *testing.T) {
 						{
 							IPAddress: proto.String(loadBalancerIP),
 							PortRange: proto.String("100-100"),
-							Labels:    map[string]string{"constellation-uid": uid},
+							Labels:    map[string]string{cloud.TagUID: uid, "constellation-use": use},
+						},
+					},
+				},
+			},
+			wantErr: true,
+		},
+		"fails on incorrect use label": {
+			instanceAPI:        stubInstancesClient{GetInstance: instance},
+			stubMetadataClient: stubMetadataClient{InstanceValue: uid},
+			stubForwardingRulesClient: stubForwardingRulesClient{
+				ForwardingRuleIterator: &stubForwardingRuleIterator{
+					rules: []*computepb.ForwardingRule{
+						{
+							IPAddress: proto.String(loadBalancerIP),
+							PortRange: proto.String("100-100"),
+							Labels:    map[string]string{cloud.TagUID: uid, "constellation-use": "bootstrapper"},
 						},
 					},
 				},
@@ -869,7 +895,7 @@ func TestRetrieveLoadBalancerEndpoint(t *testing.T) {
 			assert := assert.New(t)
 			require := require.New(t)
 
-			client := Client{forwardingRulesAPI: tc.stubForwardingRulesClient, metadataAPI: tc.stubMetadataClient}
+			client := Client{instanceAPI: tc.instanceAPI, forwardingRulesAPI: tc.stubForwardingRulesClient, metadataAPI: tc.stubMetadataClient}
 			aliasCIDR, err := client.RetrieveLoadBalancerEndpoint(context.Background(), "project")
 
 			if tc.wantErr {
@@ -1049,6 +1075,8 @@ func (s stubForwardingRulesClient) Close() error {
 type stubMetadataClient struct {
 	InstanceValue     string
 	InstanceErr       error
+	instanceIDValue   string
+	instanceIDErr     error
 	ProjectIDValue    string
 	ProjectIDErr      error
 	ZoneValue         string
@@ -1061,6 +1089,10 @@ func (s stubMetadataClient) InstanceAttributeValue(attr string) (string, error)
 	return s.InstanceValue, s.InstanceErr
 }
 
+func (s stubMetadataClient) InstanceID() (string, error) {
+	return s.instanceIDValue, s.instanceIDErr
+}
+
 func (s stubMetadataClient) ProjectID() (string, error) {
 	return s.ProjectIDValue, s.ProjectIDErr
 }

internal/cloud/gcp/metadata.go

@@ -17,7 +17,7 @@ import (
 // API handles all GCP API requests.
 type API interface {
 	// UID retrieves the current instances uid.
-	UID() (string, error)
+	UID(context.Context) (string, error)
 	// RetrieveInstances retrieves a list of all accessible GCP instances with their metadata.
 	RetrieveInstances(ctx context.Context, project, zone string) ([]metadata.InstanceMetadata, error)
 	// RetrieveInstances retrieves a single GCP instances with its metadata.
@@ -128,7 +128,7 @@ func (m *Metadata) GetLoadBalancerEndpoint(ctx context.Context) (string, error)
 
 // UID retrieves the UID of the constellation.
 func (m *Metadata) UID(ctx context.Context) (string, error) {
-	return m.api.UID()
+	return m.api.UID(ctx)
 }
 
 // Supported is used to determine if metadata API is implemented for this cloud provider.

internal/cloud/gcp/metadata_test.go

@@ -11,6 +11,7 @@ import (
 	"errors"
 	"testing"
 
+	"github.com/edgelesssys/constellation/v2/internal/cloud"
 	"github.com/edgelesssys/constellation/v2/internal/cloud/metadata"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -41,7 +42,7 @@ func TestList(t *testing.T) {
 				projectID: "someProjectID",
 				zone:      "someZone",
 				retrieveInstanceMetadaValues: map[string]string{
-					"constellation-uid": uid,
+					cloud.TagUID: uid,
 				},
 			},
 			instancesGenerator: instancesGenerator,
@@ -58,7 +59,7 @@ func TestList(t *testing.T) {
 				projectID: "someProjectID",
 				zone:      "someZone",
 				retrieveInstanceMetadaValues: map[string]string{
-					"constellation-uid": uid,
+					cloud.TagUID: uid,
 				},
 				retrieveInstancesErr: err,
 			},
@@ -133,7 +134,7 @@ func TestSelf(t *testing.T) {
 				projectID: "someProjectID",
 				zone:      "someZone",
 				retrieveInstanceMetadaValues: map[string]string{
-					"constellation-uid": uid,
+					cloud.TagUID: uid,
 				},
 				retrieveInstanceErr: err,
 			},
@@ -297,7 +298,7 @@ func (s *stubGCPClient) RetrieveLoadBalancerEndpoint(ctx context.Context, projec
 	return s.loadBalancerIP, s.retrieveLoadBalancerErr
 }
 
-func (s *stubGCPClient) UID() (string, error) {
+func (s *stubGCPClient) UID(context.Context) (string, error) {
 	return s.retrieveUIDValue, s.retrieveUIDErr
 }
 

internal/cloud/gcp/role.go

@@ -1,25 +0,0 @@
-/*
-Copyright (c) Edgeless Systems GmbH
-
-SPDX-License-Identifier: AGPL-3.0-only
-*/
-
-package gcp
-
-import (
-	"github.com/edgelesssys/constellation/v2/internal/role"
-)
-
-const roleMetadataKey = "constellation-role"
-
-// extractRole extracts role from cloud provider metadata.
-func extractRole(metadata map[string]string) role.Role {
-	switch metadata[roleMetadataKey] {
-	case role.ControlPlane.String():
-		return role.ControlPlane
-	case role.Worker.String():
-		return role.Worker
-	default:
-		return role.Unknown
-	}
-}

internal/cloud/gcp/role_test.go

@@ -1,53 +0,0 @@
-/*
-Copyright (c) Edgeless Systems GmbH
-
-SPDX-License-Identifier: AGPL-3.0-only
-*/
-
-package gcp
-
-import (
-	"testing"
-
-	"github.com/edgelesssys/constellation/v2/internal/role"
-	"github.com/stretchr/testify/assert"
-)
-
-func TestExtractRole(t *testing.T) {
-	testCases := map[string]struct {
-		metadata map[string]string
-		wantRole role.Role
-	}{
-		"bootstrapper role": {
-			metadata: map[string]string{
-				roleMetadataKey: role.ControlPlane.String(),
-			},
-			wantRole: role.ControlPlane,
-		},
-		"node role": {
-			metadata: map[string]string{
-				roleMetadataKey: role.Worker.String(),
-			},
-			wantRole: role.Worker,
-		},
-		"unknown role": {
-			metadata: map[string]string{
-				roleMetadataKey: "some-unknown-role",
-			},
-			wantRole: role.Unknown,
-		},
-		"no role": {
-			wantRole: role.Unknown,
-		},
-	}
-
-	for name, tc := range testCases {
-		t.Run(name, func(t *testing.T) {
-			assert := assert.New(t)
-
-			role := extractRole(tc.metadata)
-
-			assert.Equal(tc.wantRole, role)
-		})
-	}
-}

internal/cloud/gcp/wrappers.go

@@ -69,6 +69,10 @@ func (c *metadataClient) InstanceAttributeValue(attr string) (string, error) {
 	return metadata.InstanceAttributeValue(attr)
 }
 
+func (c *metadataClient) InstanceID() (string, error) {
+	return metadata.InstanceID()
+}
+
 func (c *metadataClient) ProjectID() (string, error) {
 	return metadata.ProjectID()
 }