Git-Mirrors/constellation
1
0
Fork 0
mirror of https://github.com/edgelesssys/constellation.git synced 2024-10-01 01:36:09 -04:00
Code Issues Packages Projects Releases Wiki Activity

Use tags for UID and role parsing (#242)

Browse Source 
* Apply tags to all applicable GCP resources

* Move GCP UID and role from VM metadata to labels

* Adjust Azure tags to be in line with GCP and AWS

* Dont rely on resource name to find resources

Signed-off-by: Daniel Weiße <dw@edgeless.systems>
This commit is contained in:
Daniel Weiße 2022-10-24 16:58:21 +02:00 committed by GitHub
parent c2814aeddb
commit b35b74b772
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
34 changed files with 344 additions and 360 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
bootstrapper/cmd/bootstrapper/main.go
View File

@ -135,7 +135,7 @@ func main() {
if err != nil {
log.With(zap.Error(err)).Fatalf("Failed to marshal PCRs")
}
cloudControllerManager, err := gcpcloud.NewCloudControllerManager(metadata)
cloudControllerManager, err := gcpcloud.NewCloudControllerManager(ctx, metadata)
if err != nil {
log.With(zap.Error(err)).Fatalf("Failed to create cloud controller manager")
}

2
bootstrapper/internal/kubernetes/kubernetes.go
View File

@ -144,7 +144,7 @@ func (k *KubeWrapper) InitCluster(
zap.String("nodeName", nodeName),
zap.String("providerID", providerID),
zap.String("nodeIP", nodeIP),
zap.String("controlPlaneEndpointEndpoint", controlPlaneEndpoint),
zap.String("controlPlaneEndpoint", controlPlaneEndpoint),
zap.String("podCIDR", subnetworkPodCIDR),
).Infof("Setting information for node")

4
cli/internal/terraform/terraform/azure/main.tf
View File

@ -194,7 +194,7 @@ module "scale_set_control_plane" {
instance_type = var.instance_type
confidential_vm = var.confidential_vm
secure_boot = var.secure_boot
tags = merge(local.tags, { role = "control-plane" })
tags = merge(local.tags, { constellation-role = "control-plane" })
image_id = var.image_id
user_assigned_identity = var.user_assigned_identity
network_security_group_id = azurerm_network_security_group.security_group.id
@ -217,7 +217,7 @@ module "scale_set_worker" {
instance_type = var.instance_type
confidential_vm = var.confidential_vm
secure_boot = var.secure_boot
tags = merge(local.tags, { role = "worker" })
tags = merge(local.tags, { constellation-role = "worker" })
image_id = var.image_id
user_assigned_identity = var.user_assigned_identity
network_security_group_id = azurerm_network_security_group.security_group.id

13
cli/internal/terraform/terraform/gcp/main.tf
View File

@ -22,7 +22,7 @@ provider "google" {
locals {
uid = random_id.uid.hex
name = "${var.name}-${local.uid}"
tag = "constellation-${local.uid}"
labels = { constellation-uid = local.uid }
ports_node_range = "30000-32767"
ports_kubernetes = "6443"
ports_bootstrapper = "9000"
@ -138,6 +138,7 @@ module "instance_group_control_plane" {
{ name = "recovery", port = local.ports_recovery },
var.debug ? [{ name = "debugd", port = local.ports_debugd }] : [],
])
labels = local.labels
}
module "instance_group_worker" {
@ -154,6 +155,7 @@ module "instance_group_worker" {
subnetwork = google_compute_subnetwork.vpc_subnetwork.id
kube_env = local.kube_env
debug = var.debug
labels = local.labels
}
resource "google_compute_global_address" "loadbalancer_ip" {
@ -168,9 +170,7 @@ module "loadbalancer_kube" {
backend_instance_group = module.instance_group_control_plane.instance_group
ip_address = google_compute_global_address.loadbalancer_ip.self_link
port = local.ports_kubernetes
frontend_labels = {
constellation-uid = local.uid
}
frontend_labels = merge(local.labels, { constellation-use = "kubernetes" })
}
module "loadbalancer_boot" {
@ -181,6 +181,7 @@ module "loadbalancer_boot" {
backend_instance_group = module.instance_group_control_plane.instance_group
ip_address = google_compute_global_address.loadbalancer_ip.self_link
port = local.ports_bootstrapper
frontend_labels = merge(local.labels, { constellation-use = "bootstrapper" })
}
module "loadbalancer_verify" {
@ -191,6 +192,7 @@ module "loadbalancer_verify" {
backend_instance_group = module.instance_group_control_plane.instance_group
ip_address = google_compute_global_address.loadbalancer_ip.self_link
port = local.ports_verify
frontend_labels = merge(local.labels, { constellation-use = "verify" })
}
module "loadbalancer_konnectivity" {
@ -201,6 +203,7 @@ module "loadbalancer_konnectivity" {
backend_instance_group = module.instance_group_control_plane.instance_group
ip_address = google_compute_global_address.loadbalancer_ip.self_link
port = local.ports_konnectivity
frontend_labels = merge(local.labels, { constellation-use = "konnectivity" })
}
module "loadbalancer_recovery" {
@ -211,6 +214,7 @@ module "loadbalancer_recovery" {
backend_instance_group = module.instance_group_control_plane.instance_group
ip_address = google_compute_global_address.loadbalancer_ip.self_link
port = local.ports_recovery
frontend_labels = merge(local.labels, { constellation-use = "recovery" })
}
module "loadbalancer_debugd" {
@ -222,4 +226,5 @@ module "loadbalancer_debugd" {
backend_instance_group = module.instance_group_control_plane.instance_group
ip_address = google_compute_global_address.loadbalancer_ip.self_link
port = local.ports_debugd
frontend_labels = merge(local.labels, { constellation-use = "debugd" })
}

3
cli/internal/terraform/terraform/gcp/modules/instance_group/main.tf
View File

@ -16,6 +16,7 @@ resource "google_compute_instance_template" "template" {
name = local.name
machine_type = var.instance_type
tags = ["constellation-${var.uid}"]
labels = merge(var.labels, { constellation-role = local.role_dashed })
confidential_instance_config {
enable_confidential_compute = true
@ -41,8 +42,6 @@ resource "google_compute_instance_template" "template" {
metadata = {
kube-env = var.kube_env
constellation-uid = var.uid
constellation-role = var.role
serial-port-enable = var.debug ? "TRUE" : "FALSE"
}

6
cli/internal/terraform/terraform/gcp/modules/instance_group/variables.tf
View File

@ -13,6 +13,12 @@ variable "uid" {
description = "UID of the cluster. This is used for tags."
}
variable "labels" {
type = map(string)
default = {}
description = "Labels to apply to the instance group."
}
variable "instance_type" {
type = string
description = "Instance type for the nodes."

5
internal/cloud/aws/logger.go
View File

@ -18,6 +18,7 @@ import (
"github.com/aws/aws-sdk-go-v2/feature/ec2/imds"
logs "github.com/aws/aws-sdk-go-v2/service/cloudwatchlogs"
"github.com/aws/aws-sdk-go-v2/service/cloudwatchlogs/types"
"github.com/edgelesssys/constellation/v2/internal/cloud"
"k8s.io/utils/clock"
)
@ -147,7 +148,7 @@ func (l *Logger) createStream(ctx context.Context, imds imdsAPI) error {
l.streamName = name
// find log group with matching Constellation UID
uid, err := readInstanceTag(ctx, imds, tagUID)
uid, err := readInstanceTag(ctx, imds, cloud.TagUID)
if err != nil {
return err
}
@ -162,7 +163,7 @@ func (l *Logger) createStream(ctx context.Context, imds imdsAPI) error {
if err != nil {
continue // we may not have permission to read the tags of a log group outside the Constellation scope
}
if tags.Tags[tagUID] == uid {
if tags.Tags[cloud.TagUID] == uid {
l.groupName = *group.LogGroupName
res.NextToken = nil // stop pagination
break

45
internal/cloud/aws/logger_test.go
View File

@ -17,6 +17,7 @@ import (
"github.com/aws/aws-sdk-go-v2/aws"
logs "github.com/aws/aws-sdk-go-v2/service/cloudwatchlogs"
"github.com/aws/aws-sdk-go-v2/service/cloudwatchlogs/types"
"github.com/edgelesssys/constellation/v2/internal/cloud"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"go.uber.org/goleak"
@ -40,8 +41,8 @@ func TestCreateStream(t *testing.T) {
"success new stream minimal": {
imds: &stubIMDS{
tags: map[string]string{
tagName: "test-instance",
tagUID: "uid",
tagName: "test-instance",
cloud.TagUID: "uid",
},
},
logs: &stubLogs{
@ -50,7 +51,7 @@ func TestCreateStream(t *testing.T) {
{LogGroupName: aws.String("test-group")},
},
},
listTags: map[string]map[string]string{"test-group": {tagUID: "uid"}},
listTags: map[string]map[string]string{"test-group": {cloud.TagUID: "uid"}},
},
wantStream: "test-instance",
wantGroup: "test-group",
@ -58,8 +59,8 @@ func TestCreateStream(t *testing.T) {
"success one group of many": {
imds: &stubIMDS{
tags: map[string]string{
tagName: "test-instance",
tagUID: "uid",
tagName: "test-instance",
cloud.TagUID: "uid",
},
},
logs: &stubLogs{
@ -89,13 +90,13 @@ func TestCreateStream(t *testing.T) {
"some-tag": "random-tag",
},
"other-group": {
tagUID: "other-uid",
cloud.TagUID: "other-uid",
},
"another-group": {
"some-tag": "uid",
},
"test-group": {
tagUID: "uid",
cloud.TagUID: "uid",
},
},
},
@ -105,8 +106,8 @@ func TestCreateStream(t *testing.T) {
"success stream exists": {
imds: &stubIMDS{
tags: map[string]string{
tagName: "test-instance",
tagUID: "uid",
tagName: "test-instance",
cloud.TagUID: "uid",
},
},
logs: &stubLogs{
@ -115,7 +116,7 @@ func TestCreateStream(t *testing.T) {
{LogGroupName: aws.String("test-group")},
},
},
listTags: map[string]map[string]string{"test-group": {tagUID: "uid"}},
listTags: map[string]map[string]string{"test-group": {cloud.TagUID: "uid"}},
createErr: &types.ResourceAlreadyExistsException{},
},
wantStream: "test-instance",
@ -124,8 +125,8 @@ func TestCreateStream(t *testing.T) {
"create stream error": {
imds: &stubIMDS{
tags: map[string]string{
tagName: "test-instance",
tagUID: "uid",
tagName: "test-instance",
cloud.TagUID: "uid",
},
},
logs: &stubLogs{
@ -134,7 +135,7 @@ func TestCreateStream(t *testing.T) {
{LogGroupName: aws.String("test-group")},
},
},
listTags: map[string]map[string]string{"test-group": {tagUID: "uid"}},
listTags: map[string]map[string]string{"test-group": {cloud.TagUID: "uid"}},
createErr: someErr,
},
wantErr: true,
@ -151,14 +152,14 @@ func TestCreateStream(t *testing.T) {
{LogGroupName: aws.String("test-group")},
},
},
listTags: map[string]map[string]string{"test-group": {tagUID: "uid"}},
listTags: map[string]map[string]string{"test-group": {cloud.TagUID: "uid"}},
},
wantErr: true,
},
"missing name tag": {
imds: &stubIMDS{
tags: map[string]string{
tagUID: "uid",
cloud.TagUID: "uid",
},
},
logs: &stubLogs{
@ -167,33 +168,33 @@ func TestCreateStream(t *testing.T) {
{LogGroupName: aws.String("test-group")},
},
},
listTags: map[string]map[string]string{"test-group": {tagUID: "uid"}},
listTags: map[string]map[string]string{"test-group": {cloud.TagUID: "uid"}},
},
wantErr: true,
},
"describe groups error": {
imds: &stubIMDS{
tags: map[string]string{
tagName: "test-instance",
tagUID: "uid",
tagName: "test-instance",
cloud.TagUID: "uid",
},
},
logs: &stubLogs{
describeErr: someErr,
listTags: map[string]map[string]string{"test-group": {tagUID: "uid"}},
listTags: map[string]map[string]string{"test-group": {cloud.TagUID: "uid"}},
},
wantErr: true,
},
"no matching groups": {
imds: &stubIMDS{
tags: map[string]string{
tagName: "test-instance",
tagUID: "uid",
tagName: "test-instance",
cloud.TagUID: "uid",
},
},
logs: &stubLogs{
describeRes1: &logs.DescribeLogGroupsOutput{},
listTags: map[string]map[string]string{"test-group": {tagUID: "uid"}},
listTags: map[string]map[string]string{"test-group": {cloud.TagUID: "uid"}},
},
wantErr: true,
},

13
internal/cloud/aws/metadata.go
View File

@ -17,14 +17,13 @@ import (
"github.com/aws/aws-sdk-go-v2/feature/ec2/imds"
"github.com/aws/aws-sdk-go-v2/service/ec2"
"github.com/aws/aws-sdk-go-v2/service/ec2/types"
"github.com/edgelesssys/constellation/v2/internal/cloud"
"github.com/edgelesssys/constellation/v2/internal/cloud/metadata"
"github.com/edgelesssys/constellation/v2/internal/role"
)
const (
tagName = "Name"
tagRole = "constellation-role"
tagUID = "constellation-uid"
)
type ec2API interface {
@ -62,7 +61,7 @@ func (m *Metadata) Supported() bool {
// List retrieves all instances belonging to the current Constellation.
func (m *Metadata) List(ctx context.Context) ([]metadata.InstanceMetadata, error) {
uid, err := readInstanceTag(ctx, m.imds, tagUID)
uid, err := readInstanceTag(ctx, m.imds, cloud.TagUID)
if err != nil {
return nil, fmt.Errorf("retrieving uid tag: %w", err)
}
@ -85,7 +84,7 @@ func (m *Metadata) Self(ctx context.Context) (metadata.InstanceMetadata, error)
if err != nil {
return metadata.InstanceMetadata{}, fmt.Errorf("retrieving name tag: %w", err)
}
instanceRole, err := readInstanceTag(ctx, m.imds, tagRole)
instanceRole, err := readInstanceTag(ctx, m.imds, cloud.TagRole)
if err != nil {
return metadata.InstanceMetadata{}, fmt.Errorf("retrieving role tag: %w", err)
}
@ -128,7 +127,7 @@ func (m *Metadata) GetInstance(ctx context.Context, providerID string) (metadata
// UID returns the UID of the Constellation.
func (m *Metadata) UID(ctx context.Context) (string, error) {
return readInstanceTag(ctx, m.imds, tagUID)
return readInstanceTag(ctx, m.imds, cloud.TagUID)
}
// SupportsLoadBalancer returns true if the cloud provider supports load balancers.
@ -151,7 +150,7 @@ func (m *Metadata) getAllInstancesInGroup(ctx context.Context, uid string) ([]ty
instanceReq := &ec2.DescribeInstancesInput{
Filters: []types.Filter{
{
Name: aws.String("tag:" + tagUID),
Name: aws.String("tag:" + cloud.TagUID),
Values: []string{uid},
},
},
@ -199,7 +198,7 @@ func (m *Metadata) convertToMetadataInstance(ec2Instances []types.Instance) ([]m
}
newInstance.Name = name
instanceRole, err := findTag(ec2Instance.Tags, tagRole)
instanceRole, err := findTag(ec2Instance.Tags, cloud.TagRole)
if err != nil {
return nil, fmt.Errorf("retrieving tag for instance %s: %w", *ec2Instance.InstanceId, err)
}

43
internal/cloud/aws/metadata_test.go
View File

@ -17,6 +17,7 @@ import (
"github.com/aws/aws-sdk-go-v2/feature/ec2/imds"
"github.com/aws/aws-sdk-go-v2/service/ec2"
"github.com/aws/aws-sdk-go-v2/service/ec2/types"
"github.com/edgelesssys/constellation/v2/internal/cloud"
"github.com/edgelesssys/constellation/v2/internal/cloud/metadata"
"github.com/edgelesssys/constellation/v2/internal/role"
"github.com/stretchr/testify/assert"
@ -41,8 +42,8 @@ func TestSelf(t *testing.T) {
},
},
tags: map[string]string{
tagName: "test-instance",
tagRole: "controlplane",
tagName: "test-instance",
cloud.TagRole: "controlplane",
},
},
wantSelf: metadata.InstanceMetadata{
@ -62,8 +63,8 @@ func TestSelf(t *testing.T) {
},
},
tags: map[string]string{
tagName: "test-instance",
tagRole: "worker",
tagName: "test-instance",
cloud.TagRole: "worker",
},
},
wantSelf: metadata.InstanceMetadata{
@ -77,8 +78,8 @@ func TestSelf(t *testing.T) {
imds: &stubIMDS{
getInstanceIdentityDocumentErr: someErr,
tags: map[string]string{
tagName: "test-instance",
tagRole: "controlplane",
tagName: "test-instance",
cloud.TagRole: "controlplane",
},
},
wantErr: true,
@ -106,7 +107,7 @@ func TestSelf(t *testing.T) {
},
},
tags: map[string]string{
tagRole: "controlplane",
cloud.TagRole: "controlplane",
},
},
wantErr: true,
@ -165,11 +166,11 @@ func TestList(t *testing.T) {
Value: aws.String("name-1"),
},
{
Key: aws.String(tagRole),
Key: aws.String(cloud.TagRole),
Value: aws.String("controlplane"),
},
{
Key: aws.String(tagUID),
Key: aws.String(cloud.TagUID),
Value: aws.String("uid"),
},
},
@ -187,11 +188,11 @@ func TestList(t *testing.T) {
Value: aws.String("name-2"),
},
{
Key: aws.String(tagRole),
Key: aws.String(cloud.TagRole),
Value: aws.String("worker"),
},
{
Key: aws.String(tagUID),
Key: aws.String(cloud.TagUID),
Value: aws.String("uid"),
},
},
@ -210,7 +211,7 @@ func TestList(t *testing.T) {
"success single page": {
imds: &stubIMDS{
tags: map[string]string{
tagUID: "uid",
cloud.TagUID: "uid",
},
},
ec2: &stubEC2{
@ -234,7 +235,7 @@ func TestList(t *testing.T) {
"success multiple pages": {
imds: &stubIMDS{
tags: map[string]string{
tagUID: "uid",
cloud.TagUID: "uid",
},
},
ec2: &stubEC2{
@ -255,11 +256,11 @@ func TestList(t *testing.T) {
Value: aws.String("name-3"),
},
{
Key: aws.String(tagRole),
Key: aws.String(cloud.TagRole),
Value: aws.String("worker"),
},
{
Key: aws.String(tagUID),
Key: aws.String(cloud.TagUID),
Value: aws.String("uid"),
},
},
@ -302,7 +303,7 @@ func TestList(t *testing.T) {
"describe instances fails": {
imds: &stubIMDS{
tags: map[string]string{
tagUID: "uid",
cloud.TagUID: "uid",
},
},
ec2: &stubEC2{
@ -350,7 +351,7 @@ func TestConvertToMetadataInstance(t *testing.T) {
Value: aws.String("name-1"),
},
{
Key: aws.String(tagRole),
Key: aws.String(cloud.TagRole),
Value: aws.String("controlplane"),
},
},
@ -377,7 +378,7 @@ func TestConvertToMetadataInstance(t *testing.T) {
Value: aws.String("name-1"),
},
{
Key: aws.String(tagRole),
Key: aws.String(cloud.TagRole),
Value: aws.String("controlplane"),
},
},
@ -417,7 +418,7 @@ func TestConvertToMetadataInstance(t *testing.T) {
Value: aws.String("name-1"),
},
{
Key: aws.String(tagRole),
Key: aws.String(cloud.TagRole),
Value: aws.String("controlplane"),
},
},
@ -439,7 +440,7 @@ func TestConvertToMetadataInstance(t *testing.T) {
Value: aws.String("name-1"),
},
{
Key: aws.String(tagRole),
Key: aws.String(cloud.TagRole),
Value: aws.String("controlplane"),
},
},
@ -458,7 +459,7 @@ func TestConvertToMetadataInstance(t *testing.T) {
},
Tags: []types.Tag{
{
Key: aws.String(tagRole),
Key: aws.String(cloud.TagRole),
Value: aws.String("controlplane"),
},
},

5
internal/cloud/azure/imds.go
View File

@ -15,6 +15,7 @@ import (
"net/http"
"time"
"github.com/edgelesssys/constellation/v2/internal/cloud"
"github.com/edgelesssys/constellation/v2/internal/role"
)
@ -91,7 +92,7 @@ func (c *imdsClient) UID(ctx context.Context) (string, error) {
}
for _, tag := range c.cache.Compute.Tags {
if tag.Name == "constellation-uid" {
if tag.Name == cloud.TagUID {
return tag.Value, nil
}
}
@ -107,7 +108,7 @@ func (c *imdsClient) Role(ctx context.Context) (role.Role, error) {
}
for _, tag := range c.cache.Compute.Tags {
if tag.Name == "role" {
if tag.Name == cloud.TagRole {
return role.FromString(tag.Value), nil
}
}

9
internal/cloud/azure/imds_test.go
View File

@ -15,6 +15,7 @@ import (
"net/http/httptest"
"testing"
"github.com/edgelesssys/constellation/v2/internal/cloud"
"github.com/edgelesssys/constellation/v2/internal/role"
"github.com/stretchr/testify/assert"
"google.golang.org/grpc/test/bufconn"
@ -22,8 +23,8 @@ import (
func TestIMDSClient(t *testing.T) {
uidTags := []metadataTag{
{Name: "constellation-uid", Value: "uid"},
{Name: "role", Value: "worker"},
{Name: cloud.TagUID, Value: "uid"},
{Name: cloud.TagRole, Value: "worker"},
}
response := metadataResponse{
Compute: metadataResponseCompute{
@ -48,14 +49,14 @@ func TestIMDSClient(t *testing.T) {
Compute: metadataResponseCompute{
ResourceID: "resource-id",
ResourceGroup: "resource-group",
Tags: []metadataTag{{Name: "role", Value: "worker"}},
Tags: []metadataTag{{Name: cloud.TagRole, Value: "worker"}},
},
}
responseWithoutRole := metadataResponse{
Compute: metadataResponseCompute{
ResourceID: "resource-id",
ResourceGroup: "resource-group",
Tags: []metadataTag{{Name: "constellation-uid", Value: "uid"}},
Tags: []metadataTag{{Name: cloud.TagUID, Value: "uid"}},
},
}

3
internal/cloud/azure/metadata.go
View File

@ -17,6 +17,7 @@ import (
armcomputev2 "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v2"
"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork"
"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources"
"github.com/edgelesssys/constellation/v2/internal/cloud"
"github.com/edgelesssys/constellation/v2/internal/cloud/metadata"
)
@ -301,7 +302,7 @@ func (m *Metadata) getAppInsights(ctx context.Context) (*armapplicationinsights.
continue
}
tag, ok := component.Tags["constellation-uid"]
tag, ok := component.Tags[cloud.TagUID]
if !ok || tag == nil {
continue
}

13
internal/cloud/azure/metadata_test.go
View File

@ -14,6 +14,7 @@ import (
"github.com/Azure/azure-sdk-for-go/sdk/azcore/to"
armcomputev2 "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v2"
"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork"
"github.com/edgelesssys/constellation/v2/internal/cloud"
"github.com/edgelesssys/constellation/v2/internal/cloud/metadata"
"github.com/edgelesssys/constellation/v2/internal/role"
"github.com/stretchr/testify/assert"
@ -654,8 +655,8 @@ func newScaleSetsStub() *stubScaleSetsAPI {
list: []armcomputev2.VirtualMachineScaleSet{{
Name: to.Ptr("scale-set-name"),
Tags: map[string]*string{
"constellation-uid": to.Ptr("uid"),
"role": to.Ptr("worker"),
cloud.TagUID: to.Ptr("uid"),
cloud.TagRole: to.Ptr("worker"),
},
}},
},
@ -691,8 +692,8 @@ func newVirtualMachineScaleSetsVMsStub() *stubVirtualMachineScaleSetVMsAPI {
},
},
Tags: map[string]*string{
"constellation-uid": to.Ptr("uid"),
"role": to.Ptr("worker"),
cloud.TagUID: to.Ptr("uid"),
cloud.TagRole: to.Ptr("worker"),
},
},
pager: &stubVirtualMachineScaleSetVMPager{
@ -724,8 +725,8 @@ func newVirtualMachineScaleSetsVMsStub() *stubVirtualMachineScaleSetVMsAPI {
},
},
Tags: map[string]*string{
"constellation-uid": to.Ptr("uid"),
"role": to.Ptr("worker"),
cloud.TagUID: to.Ptr("uid"),
cloud.TagRole: to.Ptr("worker"),
},
},
},

3
internal/cloud/azure/scaleset.go
View File

@ -16,6 +16,7 @@ import (
armcomputev2 "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v2"
"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork"
"github.com/edgelesssys/constellation/v2/internal/azureshared"
"github.com/edgelesssys/constellation/v2/internal/cloud"
"github.com/edgelesssys/constellation/v2/internal/cloud/metadata"
"github.com/edgelesssys/constellation/v2/internal/role"
)
@ -117,7 +118,7 @@ func extractScaleSetVMRole(tags map[string]*string) role.Role {
if tags == nil {
return role.Unknown
}
roleStr, ok := tags["role"]
roleStr, ok := tags[cloud.TagRole]
if !ok {
return role.Unknown
}

11
internal/cloud/azure/scaleset_test.go
View File

@ -14,6 +14,7 @@ import (
"github.com/Azure/azure-sdk-for-go/sdk/azcore/to"
armcomputev2 "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v2"
"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork"
"github.com/edgelesssys/constellation/v2/internal/cloud"
"github.com/edgelesssys/constellation/v2/internal/cloud/metadata"
"github.com/edgelesssys/constellation/v2/internal/role"
"github.com/stretchr/testify/assert"
@ -223,15 +224,15 @@ func TestExtractScaleSetVMRole(t *testing.T) {
wantRole role.Role
}{
"control-plane role": {
tags: map[string]*string{"role": to.Ptr("control-plane")},
tags: map[string]*string{cloud.TagRole: to.Ptr("control-plane")},
wantRole: role.ControlPlane,
},
"worker role": {
tags: map[string]*string{"role": to.Ptr("worker")},
tags: map[string]*string{cloud.TagRole: to.Ptr("worker")},
wantRole: role.Worker,
},
"unknown role": {
tags: map[string]*string{"role": to.Ptr("foo")},
tags: map[string]*string{cloud.TagRole: to.Ptr("foo")},
wantRole: role.Unknown,
},
"no role": {
@ -239,7 +240,7 @@ func TestExtractScaleSetVMRole(t *testing.T) {
wantRole: role.Unknown,
},
"nil role": {
tags: map[string]*string{"role": nil},
tags: map[string]*string{cloud.TagRole: nil},
wantRole: role.Unknown,
},
"nil tags": {
@ -280,7 +281,7 @@ func newListContainingNilScaleSetVirtualMachinesStub() *stubVirtualMachineScaleS
ID: to.Ptr("/subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/virtualMachineScaleSets/scale-set-name/virtualMachines/instance-id"),
InstanceID: to.Ptr("instance-id"),
Tags: map[string]*string{
"role": to.Ptr("worker"),
cloud.TagRole: to.Ptr("worker"),
},
Properties: &armcomputev2.VirtualMachineScaleSetVMProperties{
NetworkProfile: &armcomputev2.NetworkProfile{

14
internal/cloud/cloud.go Normal file
View File

@ -0,0 +1,14 @@
/*
Copyright (c) Edgeless Systems GmbH
SPDX-License-Identifier: AGPL-3.0-only
*/
package cloud
const (
// TagRole is the tag/label key used to identify the role of a node.
TagRole = "constellation-role"
// TagUID is the tag/label key used to identify the UID of a cluster.
TagUID = "constellation-uid"
)

1
internal/cloud/gcp/api.go
View File

@ -35,6 +35,7 @@ type forwardingRulesAPI interface {
type metadataAPI interface {
InstanceAttributeValue(attr string) (string, error)
InstanceID() (string, error)
ProjectID() (string, error)
Zone() (string, error)
InstanceName() (string, error)

4
internal/cloud/gcp/ccm.go
View File

@ -26,8 +26,8 @@ type CloudControllerManager struct {
}
// NewCloudControllerManager returns an initialized cloud controller manager configuration struct for GCP.
func NewCloudControllerManager(metadata *Metadata) (*CloudControllerManager, error) {
uid, err := metadata.api.UID()
func NewCloudControllerManager(ctx context.Context, metadata *Metadata) (*CloudControllerManager, error) {
uid, err := metadata.api.UID(ctx)
if err != nil {
return nil, fmt.Errorf("getting uid from metadata: %w", err)
}

44
internal/cloud/gcp/client.go
View File

@ -15,16 +15,17 @@ import (
"strings"
compute "cloud.google.com/go/compute/apiv1"
"github.com/edgelesssys/constellation/v2/internal/cloud"
"github.com/edgelesssys/constellation/v2/internal/cloud/metadata"
"github.com/edgelesssys/constellation/v2/internal/gcpshared"
"github.com/edgelesssys/constellation/v2/internal/role"
"google.golang.org/api/iterator"
computepb "google.golang.org/genproto/googleapis/cloud/compute/v1"
"google.golang.org/protobuf/proto"
)
const (
gcpSSHMetadataKey = "ssh-keys"
constellationUIDMetadataKey = "constellation-uid"
gcpSSHMetadataKey = "ssh-keys"
)
var zoneFromRegionRegex = regexp.MustCompile("([a-z]*-[a-z]*[0-9])")
@ -61,11 +62,12 @@ func NewClient(ctx context.Context) (*Client, error) {
// RetrieveInstances returns list of instances including their ips and metadata.
func (c *Client) RetrieveInstances(ctx context.Context, project, zone string) ([]metadata.InstanceMetadata, error) {
uid, err := c.UID()
uid, err := c.UID(ctx)
if err != nil {
return nil, err
}
req := &computepb.ListInstancesRequest{
Filter: proto.String(fmt.Sprintf("labels.%s:%s", cloud.TagUID, uid)),
Project: project,
Zone: zone,
}
@ -80,11 +82,6 @@ func (c *Client) RetrieveInstances(ctx context.Context, project, zone string) ([
if err != nil {
return nil, fmt.Errorf("retrieving instance list from compute API client: %w", err)
}
metadata := extractInstanceMetadata(resp.Metadata, "", false)
// skip instances not belonging to the current constellation
if instanceUID, ok := metadata[constellationUIDMetadataKey]; !ok || instanceUID != uid {
continue
}
instance, err := convertToCoreInstance(resp, project, zone)
if err != nil {
return nil, err
@ -223,7 +220,7 @@ func (c *Client) RetrieveSubnetworkAliasCIDR(ctx context.Context, project, zone,
// RetrieveLoadBalancerEndpoint returns the endpoint of the load balancer with the constellation-uid tag.
func (c *Client) RetrieveLoadBalancerEndpoint(ctx context.Context, project string) (string, error) {
uid, err := c.UID()
uid, err := c.UID(ctx)
if err != nil {
return "", err
}
@ -240,7 +237,7 @@ func (c *Client) RetrieveLoadBalancerEndpoint(ctx context.Context, project strin
if err != nil {
return "", fmt.Errorf("retrieving load balancer IP failed: %w", err)
}
if resp.Labels["constellation-uid"] == uid {
if resp.Labels[cloud.TagUID] == uid && resp.Labels["constellation-use"] == "kubernetes" {
if resp.PortRange == nil {
return "", errors.New("load balancer with searched UID has no ports")
}
@ -292,13 +289,30 @@ func (c *Client) updateInstanceMetadata(ctx context.Context, project, zone, inst
}
// UID retrieves the current instances uid.
func (c *Client) UID() (string, error) {
func (c *Client) UID(ctx context.Context) (string, error) {
// API endpoint: http://metadata.google.internal/computeMetadata/v1/instance/attributes/constellation-uid
uid, err := c.RetrieveInstanceMetadata(constellationUIDMetadataKey)
instanceID, err := c.InstanceID()
if err != nil {
return "", fmt.Errorf("retrieving constellation uid: %w", err)
return "", fmt.Errorf("retrieving instance ID: %w", err)
}
return uid, nil
project, err := c.ProjectID()
if err != nil {
return "", fmt.Errorf("retrieving project ID: %w", err)
}
zone, err := c.Zone()
if err != nil {
return "", fmt.Errorf("retrieving zone: %w", err)
}
instance, err := c.instanceAPI.Get(ctx, &computepb.GetInstanceRequest{
Project: project,
Zone: zone,
Instance: instanceID,
})
if err != nil {
return "", fmt.Errorf("retrieving instance labels: %w", err)
}
return instance.Labels[cloud.TagUID], nil
}
// extractVPCIP extracts the primary private IP from a list of interfaces.
@ -385,7 +399,7 @@ func convertToCoreInstance(in *computepb.Instance, project string, zone string)
return metadata.InstanceMetadata{
Name: *in.Name,
ProviderID: gcpshared.JoinProviderID(project, zone, *in.Name),
Role: extractRole(mdata),
Role: role.FromString(in.Labels[cloud.TagRole]),
VPCIP: extractVPCIP(in.NetworkInterfaces),
PublicIP: extractPublicIP(in.NetworkInterfaces),
AliasIPRanges: extractAliasIPRanges(in.NetworkInterfaces),

104
internal/cloud/gcp/client_test.go
View File

@ -12,6 +12,7 @@ import (
"testing"
compute "cloud.google.com/go/compute/apiv1"
"github.com/edgelesssys/constellation/v2/internal/cloud"
"github.com/edgelesssys/constellation/v2/internal/cloud/metadata"
"github.com/edgelesssys/constellation/v2/internal/role"
gax "github.com/googleapis/gax-go/v2"
@ -38,6 +39,10 @@ func TestRetrieveInstances(t *testing.T) {
instances: []*computepb.Instance{
{
Name: proto.String("someInstance"),
Labels: map[string]string{
cloud.TagRole: role.ControlPlane.String(),
cloud.TagUID: uid,
},
Metadata: &computepb.Metadata{
Items: []*computepb.Items{
{
@ -48,14 +53,6 @@ func TestRetrieveInstances(t *testing.T) {
Key: proto.String("key-2"),
Value: proto.String("value-2"),
},
{
Key: proto.String(constellationUIDMetadataKey),
Value: proto.String(uid),
},
{
Key: proto.String(roleMetadataKey),
Value: proto.String(role.ControlPlane.String()),
},
},
},
NetworkInterfaces: []*computepb.NetworkInterface{
@ -70,6 +67,13 @@ func TestRetrieveInstances(t *testing.T) {
},
}
}
instance := &computepb.Instance{
Name: proto.String("instance"),
Labels: map[string]string{
cloud.TagRole: role.ControlPlane.String(),
cloud.TagUID: uid,
},
}
testCases := map[string]struct {
client stubInstancesClient
@ -80,7 +84,7 @@ func TestRetrieveInstances(t *testing.T) {
wantErr bool
}{
"retrieve works": {
client: stubInstancesClient{},
client: stubInstancesClient{GetInstance: instance},
metadata: stubMetadataClient{InstanceValue: uid},
instanceIter: newTestIter(),
wantInstances: []metadata.InstanceMetadata{
@ -96,14 +100,14 @@ func TestRetrieveInstances(t *testing.T) {
},
},
"instance name is null": {
client: stubInstancesClient{},
client: stubInstancesClient{GetInstance: instance},
metadata: stubMetadataClient{InstanceValue: uid},
instanceIter: newTestIter(),
instanceIterMutator: func(sii *stubInstanceIterator) { sii.instances[0].Name = nil },
wantErr: true,
},
"no instance with network ip": {
client: stubInstancesClient{},
client: stubInstancesClient{GetInstance: instance},
metadata: stubMetadataClient{InstanceValue: uid},
instanceIter: newTestIter(),
instanceIterMutator: func(sii *stubInstanceIterator) { sii.instances[0].NetworkInterfaces = nil },
@ -120,7 +124,7 @@ func TestRetrieveInstances(t *testing.T) {
},
},
"network ip is nil": {
client: stubInstancesClient{},
client: stubInstancesClient{GetInstance: instance},
metadata: stubMetadataClient{InstanceValue: uid},
instanceIter: newTestIter(),
instanceIterMutator: func(sii *stubInstanceIterator) { sii.instances[0].NetworkInterfaces[0].NetworkIP = nil },
@ -136,24 +140,17 @@ func TestRetrieveInstances(t *testing.T) {
},
},
},
"constellation id is not set": {
client: stubInstancesClient{},
metadata: stubMetadataClient{InstanceValue: uid},
instanceIter: newTestIter(),
instanceIterMutator: func(sii *stubInstanceIterator) { sii.instances[0].Metadata.Items[2].Key = proto.String("") },
wantInstances: []metadata.InstanceMetadata{},
},
"constellation retrieval fails": {
client: stubInstancesClient{},
metadata: stubMetadataClient{InstanceErr: someErr},
client: stubInstancesClient{GetInstance: instance},
metadata: stubMetadataClient{instanceIDErr: someErr},
instanceIter: newTestIter(),
wantErr: true,
},
"role is not set": {
client: stubInstancesClient{},
client: stubInstancesClient{GetInstance: instance},
metadata: stubMetadataClient{InstanceValue: uid},
instanceIter: newTestIter(),
instanceIterMutator: func(sii *stubInstanceIterator) { sii.instances[0].Metadata.Items[3].Key = proto.String("") },
instanceIterMutator: func(sii *stubInstanceIterator) { delete(sii.instances[0].Labels, cloud.TagRole) },
wantInstances: []metadata.InstanceMetadata{
{
Name: "someInstance",
@ -167,7 +164,7 @@ func TestRetrieveInstances(t *testing.T) {
},
},
"instance iterator Next() errors": {
client: stubInstancesClient{},
client: stubInstancesClient{GetInstance: instance},
metadata: stubMetadataClient{InstanceValue: uid},
instanceIter: &stubInstanceIterator{nextErr: someErr},
wantErr: true,
@ -203,7 +200,8 @@ func TestRetrieveInstances(t *testing.T) {
func TestRetrieveInstance(t *testing.T) {
newTestInstance := func() *computepb.Instance {
return &computepb.Instance{
Name: proto.String("someInstance"),
Name: proto.String("someInstance"),
Labels: map[string]string{},
Metadata: &computepb.Metadata{
Items: []*computepb.Items{
{
@ -266,8 +264,7 @@ func TestRetrieveInstance(t *testing.T) {
client: stubInstancesClient{},
clientInstance: newTestInstance(),
clientInstanceMutator: func(i *computepb.Instance) {
i.Metadata.Items[0].Key = proto.String(roleMetadataKey)
i.Metadata.Items[0].Value = proto.String(role.ControlPlane.String())
i.Labels[cloud.TagRole] = role.ControlPlane.String()
},
wantInstance: metadata.InstanceMetadata{
Name: "someInstance",
@ -782,22 +779,31 @@ func TestRetrieveSubnetworkAliasCIDR(t *testing.T) {
func TestRetrieveLoadBalancerEndpoint(t *testing.T) {
loadBalancerIP := "192.0.2.1"
uid := "uid"
use := "kubernetes"
someErr := errors.New("some error")
instance := &computepb.Instance{
Labels: map[string]string{
cloud.TagUID: uid,
},
}
testCases := map[string]struct {
instanceAPI stubInstancesClient
stubForwardingRulesClient stubForwardingRulesClient
stubMetadataClient stubMetadataClient
wantLoadBalancerIP string
wantErr bool
}{
"works": {
stubMetadataClient: stubMetadataClient{InstanceValue: uid},
instanceAPI: stubInstancesClient{GetInstance: instance},
stubMetadataClient: stubMetadataClient{},
stubForwardingRulesClient: stubForwardingRulesClient{
ForwardingRuleIterator: &stubForwardingRuleIterator{
rules: []*computepb.ForwardingRule{
{
IPAddress: proto.String(loadBalancerIP),
PortRange: proto.String("100-100"),
Labels: map[string]string{"constellation-uid": uid},
Labels: map[string]string{cloud.TagUID: uid, "constellation-use": use},
},
},
},
@ -805,7 +811,8 @@ func TestRetrieveLoadBalancerEndpoint(t *testing.T) {
wantLoadBalancerIP: loadBalancerIP,
},
"fails when no matching load balancers exists": {
stubMetadataClient: stubMetadataClient{InstanceValue: uid},
instanceAPI: stubInstancesClient{GetInstance: instance},
stubMetadataClient: stubMetadataClient{},
stubForwardingRulesClient: stubForwardingRulesClient{
ForwardingRuleIterator: &stubForwardingRuleIterator{
rules: []*computepb.ForwardingRule{
@ -819,14 +826,15 @@ func TestRetrieveLoadBalancerEndpoint(t *testing.T) {
wantErr: true,
},
"fails when retrieving uid": {
stubMetadataClient: stubMetadataClient{InstanceErr: someErr},
instanceAPI: stubInstancesClient{GetInstance: instance},
stubMetadataClient: stubMetadataClient{instanceIDErr: someErr},
stubForwardingRulesClient: stubForwardingRulesClient{
ForwardingRuleIterator: &stubForwardingRuleIterator{
rules: []*computepb.ForwardingRule{
{
IPAddress: proto.String(loadBalancerIP),
PortRange: proto.String("100-100"),
Labels: map[string]string{"constellation-uid": uid},
Labels: map[string]string{cloud.TagUID: uid, "constellation-use": use},
},
},
},
@ -834,13 +842,14 @@ func TestRetrieveLoadBalancerEndpoint(t *testing.T) {
wantErr: true,
},
"fails when answer has empty port range": {
stubMetadataClient: stubMetadataClient{InstanceErr: someErr},
instanceAPI: stubInstancesClient{GetInstance: instance},
stubMetadataClient: stubMetadataClient{},
stubForwardingRulesClient: stubForwardingRulesClient{
ForwardingRuleIterator: &stubForwardingRuleIterator{
rules: []*computepb.ForwardingRule{
{
IPAddress: proto.String(loadBalancerIP),
Labels: map[string]string{"constellation-uid": uid},
Labels: map[string]string{cloud.TagUID: uid, "constellation-use": use},
},
},
},
@ -848,6 +857,7 @@ func TestRetrieveLoadBalancerEndpoint(t *testing.T) {
wantErr: true,
},
"fails when retrieving loadbalancer IP": {
instanceAPI: stubInstancesClient{GetInstance: instance},
stubMetadataClient: stubMetadataClient{},
stubForwardingRulesClient: stubForwardingRulesClient{
ForwardingRuleIterator: &stubForwardingRuleIterator{
@ -856,7 +866,23 @@ func TestRetrieveLoadBalancerEndpoint(t *testing.T) {
{
IPAddress: proto.String(loadBalancerIP),
PortRange: proto.String("100-100"),
Labels: map[string]string{"constellation-uid": uid},
Labels: map[string]string{cloud.TagUID: uid, "constellation-use": use},
},
},
},
},
wantErr: true,
},
"fails on incorrect use label": {
instanceAPI: stubInstancesClient{GetInstance: instance},
stubMetadataClient: stubMetadataClient{InstanceValue: uid},
stubForwardingRulesClient: stubForwardingRulesClient{
ForwardingRuleIterator: &stubForwardingRuleIterator{
rules: []*computepb.ForwardingRule{
{
IPAddress: proto.String(loadBalancerIP),
PortRange: proto.String("100-100"),
Labels: map[string]string{cloud.TagUID: uid, "constellation-use": "bootstrapper"},
},
},
},
@ -869,7 +895,7 @@ func TestRetrieveLoadBalancerEndpoint(t *testing.T) {
assert := assert.New(t)
require := require.New(t)
client := Client{forwardingRulesAPI: tc.stubForwardingRulesClient, metadataAPI: tc.stubMetadataClient}
client := Client{instanceAPI: tc.instanceAPI, forwardingRulesAPI: tc.stubForwardingRulesClient, metadataAPI: tc.stubMetadataClient}
aliasCIDR, err := client.RetrieveLoadBalancerEndpoint(context.Background(), "project")
if tc.wantErr {
@ -1049,6 +1075,8 @@ func (s stubForwardingRulesClient) Close() error {
type stubMetadataClient struct {
InstanceValue string
InstanceErr error
instanceIDValue string
instanceIDErr error
ProjectIDValue string
ProjectIDErr error
ZoneValue string
@ -1061,6 +1089,10 @@ func (s stubMetadataClient) InstanceAttributeValue(attr string) (string, error)
return s.InstanceValue, s.InstanceErr
}
func (s stubMetadataClient) InstanceID() (string, error) {
return s.instanceIDValue, s.instanceIDErr
}
func (s stubMetadataClient) ProjectID() (string, error) {
return s.ProjectIDValue, s.ProjectIDErr
}

4
internal/cloud/gcp/metadata.go
View File

@ -17,7 +17,7 @@ import (
// API handles all GCP API requests.
type API interface {
// UID retrieves the current instances uid.
UID() (string, error)
UID(context.Context) (string, error)
// RetrieveInstances retrieves a list of all accessible GCP instances with their metadata.
RetrieveInstances(ctx context.Context, project, zone string) ([]metadata.InstanceMetadata, error)
// RetrieveInstances retrieves a single GCP instances with its metadata.
@ -128,7 +128,7 @@ func (m *Metadata) GetLoadBalancerEndpoint(ctx context.Context) (string, error)
// UID retrieves the UID of the constellation.
func (m *Metadata) UID(ctx context.Context) (string, error) {
return m.api.UID()
return m.api.UID(ctx)
}
// Supported is used to determine if metadata API is implemented for this cloud provider.

9
internal/cloud/gcp/metadata_test.go
View File

@ -11,6 +11,7 @@ import (
"errors"
"testing"
"github.com/edgelesssys/constellation/v2/internal/cloud"
"github.com/edgelesssys/constellation/v2/internal/cloud/metadata"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
@ -41,7 +42,7 @@ func TestList(t *testing.T) {
projectID: "someProjectID",
zone: "someZone",
retrieveInstanceMetadaValues: map[string]string{
"constellation-uid": uid,
cloud.TagUID: uid,
},
},
instancesGenerator: instancesGenerator,
@ -58,7 +59,7 @@ func TestList(t *testing.T) {
projectID: "someProjectID",
zone: "someZone",
retrieveInstanceMetadaValues: map[string]string{
"constellation-uid": uid,
cloud.TagUID: uid,
},
retrieveInstancesErr: err,
},
@ -133,7 +134,7 @@ func TestSelf(t *testing.T) {
projectID: "someProjectID",
zone: "someZone",
retrieveInstanceMetadaValues: map[string]string{
"constellation-uid": uid,
cloud.TagUID: uid,
},
retrieveInstanceErr: err,
},
@ -297,7 +298,7 @@ func (s *stubGCPClient) RetrieveLoadBalancerEndpoint(ctx context.Context, projec
return s.loadBalancerIP, s.retrieveLoadBalancerErr
}
func (s *stubGCPClient) UID() (string, error) {
func (s *stubGCPClient) UID(context.Context) (string, error) {
return s.retrieveUIDValue, s.retrieveUIDErr
}

25
internal/cloud/gcp/role.go
View File

@ -1,25 +0,0 @@
/*
Copyright (c) Edgeless Systems GmbH
SPDX-License-Identifier: AGPL-3.0-only
*/
package gcp
import (
"github.com/edgelesssys/constellation/v2/internal/role"
)
const roleMetadataKey = "constellation-role"
// extractRole extracts role from cloud provider metadata.
func extractRole(metadata map[string]string) role.Role {
switch metadata[roleMetadataKey] {
case role.ControlPlane.String():
return role.ControlPlane
case role.Worker.String():
return role.Worker
default:
return role.Unknown
}
}

53
internal/cloud/gcp/role_test.go
View File

@ -1,53 +0,0 @@
/*
Copyright (c) Edgeless Systems GmbH
SPDX-License-Identifier: AGPL-3.0-only
*/
package gcp
import (
"testing"
"github.com/edgelesssys/constellation/v2/internal/role"
"github.com/stretchr/testify/assert"
)
func TestExtractRole(t *testing.T) {
testCases := map[string]struct {
metadata map[string]string
wantRole role.Role
}{
"bootstrapper role": {
metadata: map[string]string{
roleMetadataKey: role.ControlPlane.String(),
},
wantRole: role.ControlPlane,
},
"node role": {
metadata: map[string]string{
roleMetadataKey: role.Worker.String(),
},
wantRole: role.Worker,
},
"unknown role": {
metadata: map[string]string{
roleMetadataKey: "some-unknown-role",
},
wantRole: role.Unknown,
},
"no role": {
wantRole: role.Unknown,
},
}
for name, tc := range testCases {
t.Run(name, func(t *testing.T) {
assert := assert.New(t)
role := extractRole(tc.metadata)
assert.Equal(tc.wantRole, role)
})
}
}

4
internal/cloud/gcp/wrappers.go
View File

@ -69,6 +69,10 @@ func (c *metadataClient) InstanceAttributeValue(attr string) (string, error) {
return metadata.InstanceAttributeValue(attr)
}
func (c *metadataClient) InstanceID() (string, error) {
return metadata.InstanceID()
}
func (c *metadataClient) ProjectID() (string, error) {
return metadata.ProjectID()
}

2
internal/versions/versions.go
View File

@ -52,7 +52,7 @@ const (
VerificationImage = "ghcr.io/edgelesssys/constellation/verification-service:v2.1.0@sha256:7a1e6bec4cda270924c3495466fa536a2b6cd2d2f9c0be319fc6368710c255e8"
// Check for new versions at https://github.com/GoogleCloudPlatform/guest-agent/releases and update in /.github/workflows/build-gcp-guest-agent.yml.
GcpGuestImage = "ghcr.io/edgelesssys/gcp-guest-agent:20220927.00@sha256:3dea1ae3f162d2353e6584b325f0e325a39cda5f380f41e5a0ee43c6641d3905"
NodeOperatorCatalogImage = "ghcr.io/edgelesssys/constellation/node-operator-catalog:v2.2.0-pre.0.20221012150059-4b2dd1317a77@sha256:f840435fe3a7669afe78aa12b0ba7f36a0087dc6d86d4fe3f3e340395f002e3f"
NodeOperatorCatalogImage = "ghcr.io/edgelesssys/constellation/node-operator-catalog:v2.2.0-pre.0.20221021130530-c6623f2ebd72@sha256:f2750cb3c0f8368808686ffd5835b6d7152ae2845e35ee9891e910738dc26718"
// TODO: switch node maintenance operator catalog back to upstream quay.io/medik8s/node-maintenance-operator-catalog
// once https://github.com/medik8s/node-maintenance-operator/issues/49 is resolved.
NodeMaintenanceOperatorCatalogImage = "ghcr.io/edgelesssys/constellation/node-maintenance-operator-catalog:v0.13.1-alpha1@sha256:d382c3aaf9bc470cde6f6c05c2c6ff5c9dcfd90540d5b11f9cf69c4e1dd1ca9d"

32
operators/constellation-node-operator/internal/azure/client/scaleset.go
View File

@ -1,32 +0,0 @@
/*
Copyright (c) Edgeless Systems GmbH
SPDX-License-Identifier: AGPL-3.0-only
*/
package client
import (
"context"
"fmt"
)
// getScaleSets retrieves the IDs of all scale sets of a resource group.
func (c *Client) getScaleSets(ctx context.Context) ([]string, error) {
pager := c.scaleSetsAPI.NewListPager(c.config.ResourceGroup, nil)
var scaleSets []string
for pager.More() {
page, err := pager.NextPage(ctx)
if err != nil {
return nil, fmt.Errorf("paging scale sets: %w", err)
}
for _, scaleSet := range page.Value {
if scaleSet == nil || scaleSet.ID == nil {
continue
}
scaleSets = append(scaleSets, *scaleSet.ID)
}
}
return scaleSets, nil
}

62
operators/constellation-node-operator/internal/azure/client/scaleset_test.go
View File

@ -1,62 +0,0 @@
/*
Copyright (c) Edgeless Systems GmbH
SPDX-License-Identifier: AGPL-3.0-only
*/
package client
import (
"context"
"errors"
"testing"
"github.com/Azure/azure-sdk-for-go/sdk/azcore/to"
armcomputev2 "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v2"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestGetScaleSets(t *testing.T) {
testCases := map[string]struct {
scaleSet armcomputev2.VirtualMachineScaleSet
fetchPageErr error
wantScaleSets []string
wantErr bool
}{
"fetching scale sets works": {
scaleSet: armcomputev2.VirtualMachineScaleSet{
ID: to.Ptr("/subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/virtualMachineScaleSets/scale-set-name"),
},
wantScaleSets: []string{"/subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/virtualMachineScaleSets/scale-set-name"},
},
"fetching scale sets fails": {
fetchPageErr: errors.New("fetch page error"),
wantErr: true,
},
"scale set is invalid": {},
}
for name, tc := range testCases {
t.Run(name, func(t *testing.T) {
assert := assert.New(t)
require := require.New(t)
client := Client{
scaleSetsAPI: &stubScaleSetsAPI{
pager: &stubVMSSPager{
list: []armcomputev2.VirtualMachineScaleSet{tc.scaleSet},
fetchErr: tc.fetchPageErr,
},
},
}
gotScaleSets, err := client.getScaleSets(context.Background())
if tc.wantErr {
assert.Error(err)
return
}
require.NoError(err)
assert.ElementsMatch(tc.wantScaleSets, gotScaleSets)
})
}
}

43
operators/constellation-node-operator/internal/azure/client/scalinggroup.go
View File

@ -80,34 +80,35 @@ func (c *Client) GetAutoscalingGroupName(scalingGroupID string) (string, error)
// ListScalingGroups retrieves a list of scaling groups for the cluster.
func (c *Client) ListScalingGroups(ctx context.Context, uid string) (controlPlaneGroupIDs []string, workerGroupIDs []string, err error) {
scaleSetIDs, err := c.getScaleSets(ctx)
if err != nil {
return nil, nil, fmt.Errorf("listing scaling groups: %w", err)
}
for _, scaleSetID := range scaleSetIDs {
_, _, scaleSet, err := splitVMSSID(scaleSetID)
pager := c.scaleSetsAPI.NewListPager(c.config.ResourceGroup, nil)
for pager.More() {
page, err := pager.NextPage(ctx)
if err != nil {
return nil, nil, fmt.Errorf("getting scaling group name: %w", err)
return nil, nil, fmt.Errorf("paging scale sets: %w", err)
}
if isControlPlaneInstanceGroup(scaleSet) {
controlPlaneGroupIDs = append(controlPlaneGroupIDs, scaleSetID)
} else if isWorkerInstanceGroup(scaleSet) {
workerGroupIDs = append(workerGroupIDs, scaleSetID)
for _, scaleSet := range page.Value {
if scaleSet == nil || scaleSet.ID == nil {
continue
}
if scaleSet.Tags == nil || scaleSet.Tags["constellation-uid"] == nil || *scaleSet.Tags["constellation-uid"] != uid {
continue
}
if err != nil {
return nil, nil, fmt.Errorf("getting scaling group name: %w", err)
}
switch *scaleSet.Tags["constellation-role"] {
case "control-plane", "controlplane":
controlPlaneGroupIDs = append(controlPlaneGroupIDs, *scaleSet.ID)
case "worker":
workerGroupIDs = append(workerGroupIDs, *scaleSet.ID)
}
}
}
return controlPlaneGroupIDs, workerGroupIDs, nil
}
// isControlPlaneInstanceGroup returns true if the instance group is a control plane instance group.
func isControlPlaneInstanceGroup(instanceGroupName string) bool {
return strings.Contains(instanceGroupName, "control-plane")
}
// isWorkerInstanceGroup returns true if the instance group is a worker instance group.
func isWorkerInstanceGroup(instanceGroupName string) bool {
return strings.Contains(instanceGroupName, "worker")
}
func imageReferenceFromImage(img string) *armcompute.ImageReference {
ref := &armcompute.ImageReference{}

18
operators/constellation-node-operator/internal/azure/client/scalinggroup_test.go
View File

@ -193,15 +193,33 @@ func TestListScalingGroups(t *testing.T) {
"listing control-plane works": {
scaleSet: armcomputev2.VirtualMachineScaleSet{
ID: to.Ptr("/subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/virtualMachineScaleSets/constellation-scale-set-control-planes-uid"),
Tags: map[string]*string{
"constellation-uid": to.Ptr("uid"),
"constellation-role": to.Ptr("control-plane"),
},
},
wantControlPlanes: []string{"/subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/virtualMachineScaleSets/constellation-scale-set-control-planes-uid"},
},
"listing worker works": {
scaleSet: armcomputev2.VirtualMachineScaleSet{
ID: to.Ptr("/subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/virtualMachineScaleSets/constellation-scale-set-workers-uid"),
Tags: map[string]*string{
"constellation-uid": to.Ptr("uid"),
"constellation-role": to.Ptr("worker"),
},
},
wantWorkers: []string{"/subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/virtualMachineScaleSets/constellation-scale-set-workers-uid"},
},
"listing is not dependent on resource name": {
scaleSet: armcomputev2.VirtualMachineScaleSet{
ID: to.Ptr("/subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/virtualMachineScaleSets/some-scale-set"),
Tags: map[string]*string{
"constellation-uid": to.Ptr("uid"),
"constellation-role": to.Ptr("control-plane"),
},
},
wantControlPlanes: []string{"/subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/virtualMachineScaleSets/some-scale-set"},
},
"listing other works": {
scaleSet: armcomputev2.VirtualMachineScaleSet{
ID: to.Ptr("/subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/virtualMachineScaleSets/other"),

11
operators/constellation-node-operator/internal/gcp/client/instancegroupmanagers.go
View File

@ -10,7 +10,6 @@ import (
"context"
"fmt"
"regexp"
"strings"
)
var instanceGroupIDRegex = regexp.MustCompile(`^projects/([^/]+)/zones/([^/]+)/instanceGroupManagers/([^/]+)$`)
@ -36,16 +35,6 @@ func splitInstanceGroupID(instanceGroupID string) (project, zone, instanceGroup
return matches[1], matches[2], matches[3], nil
}
// isControlPlaneInstanceGroup returns true if the instance group is a control plane instance group.
func isControlPlaneInstanceGroup(instanceGroupName string) bool {
return strings.Contains(instanceGroupName, "control-plane")
}
// isWorkerInstanceGroup returns true if the instance group is a worker instance group.
func isWorkerInstanceGroup(instanceGroupName string) bool {
return strings.Contains(instanceGroupName, "worker")
}
// generateInstanceName generates a random instance name.
func generateInstanceName(baseInstanceName string, random prng) string {
letters := []byte("abcdefghijklmnopqrstuvwxyz0123456789")

33
operators/constellation-node-operator/internal/gcp/client/scalinggroup.go
View File

@ -10,10 +10,10 @@ import (
"context"
"errors"
"fmt"
"strings"
"google.golang.org/api/iterator"
computepb "google.golang.org/genproto/googleapis/cloud/compute/v1"
"google.golang.org/protobuf/proto"
)
// GetScalingGroupImage returns the image URI of the scaling group.
@ -108,7 +108,6 @@ func (c *Client) GetAutoscalingGroupName(scalingGroupID string) (string, error)
// ListScalingGroups retrieves a list of scaling groups for the cluster.
func (c *Client) ListScalingGroups(ctx context.Context, uid string) (controlPlaneGroupIDs []string, workerGroupIDs []string, err error) {
iter := c.instanceGroupManagersAPI.AggregatedList(ctx, &computepb.AggregatedListInstanceGroupManagersRequest{
Filter: proto.String(fmt.Sprintf("name eq \".+-%s-.+\"", uid)), // filter by constellation UID
Project: c.projectID,
})
for instanceGroupManagerScopedListPair, err := iter.Next(); ; instanceGroupManagerScopedListPair, err = iter.Next() {
@ -121,18 +120,38 @@ func (c *Client) ListScalingGroups(ctx context.Context, uid string) (controlPlan
if instanceGroupManagerScopedListPair.Value == nil {
continue
}
for _, instanceGroupManager := range instanceGroupManagerScopedListPair.Value.InstanceGroupManagers {
if instanceGroupManager == nil || instanceGroupManager.Name == nil || instanceGroupManager.SelfLink == nil {
for _, grpManager := range instanceGroupManagerScopedListPair.Value.InstanceGroupManagers {
if grpManager == nil || grpManager.Name == nil || grpManager.SelfLink == nil || grpManager.InstanceTemplate == nil {
continue
}
groupID, err := c.canonicalInstanceGroupID(ctx, *instanceGroupManager.SelfLink)
templateURI := strings.Split(*grpManager.InstanceTemplate, "/")
if len(templateURI) < 1 {
continue // invalid template URI
}
template, err := c.instanceTemplateAPI.Get(ctx, &computepb.GetInstanceTemplateRequest{
Project: c.projectID,
InstanceTemplate: templateURI[len(templateURI)-1],
})
if err != nil {
return nil, nil, fmt.Errorf("getting instance template: %w", err)
}
if template.Properties == nil || template.Properties.Labels == nil {
continue
}
if template.Properties.Labels["constellation-uid"] != uid {
continue
}
groupID, err := c.canonicalInstanceGroupID(ctx, *grpManager.SelfLink)
if err != nil {
return nil, nil, fmt.Errorf("normalizing instance group ID: %w", err)
}
if isControlPlaneInstanceGroup(*instanceGroupManager.Name) {
switch strings.ToLower(template.Properties.Labels["constellation-role"]) {
case "control-plane", "controlplane":
controlPlaneGroupIDs = append(controlPlaneGroupIDs, groupID)
} else if isWorkerInstanceGroup(*instanceGroupManager.Name) {
case "worker":
workerGroupIDs = append(workerGroupIDs, groupID)
}
}

61
operators/constellation-node-operator/internal/gcp/client/scalinggroup_test.go
View File

@ -326,7 +326,10 @@ func TestListScalingGroups(t *testing.T) {
testCases := map[string]struct {
name *string
groupID *string
templateRef *string
templateLabels map[string]string
listInstanceGroupManagersErr error
templateGetErr error
wantControlPlanes []string
wantWorkers []string
wantErr bool
@ -335,23 +338,56 @@ func TestListScalingGroups(t *testing.T) {
listInstanceGroupManagersErr: errors.New("list instance group managers error"),
wantErr: true,
},
"get instance template fails": {
name: proto.String("test-control-plane-uid"),
groupID: proto.String("projects/project/zones/zone/instanceGroupManagers/test-control-plane-uid"),
templateRef: proto.String("projects/project/global/instanceTemplates/test-control-plane-uid"),
templateGetErr: errors.New("get instance template error"),
wantErr: true,
},
"list instance group managers for control plane": {
name: proto.String("test-control-plane-uid"),
groupID: proto.String("projects/project/zones/zone/instanceGroupManagers/test-control-plane-uid"),
name: proto.String("test-control-plane-uid"),
groupID: proto.String("projects/project/zones/zone/instanceGroupManagers/test-control-plane-uid"),
templateRef: proto.String("projects/project/global/instanceTemplates/test-control-plane-uid"),
templateLabels: map[string]string{
"constellation-uid": "uid",
"constellation-role": "control-plane",
},
wantControlPlanes: []string{
"projects/project/zones/zone/instanceGroupManagers/test-control-plane-uid",
},
},
"list instance group managers for worker": {
name: proto.String("test-worker-uid"),
groupID: proto.String("projects/project/zones/zone/instanceGroupManagers/test-worker-uid"),
name: proto.String("test-worker-uid"),
groupID: proto.String("projects/project/zones/zone/instanceGroupManagers/test-worker-uid"),
templateRef: proto.String("projects/project/global/instanceTemplates/test-control-plane-uid"),
templateLabels: map[string]string{
"constellation-uid": "uid",
"constellation-role": "worker",
},
wantWorkers: []string{
"projects/project/zones/zone/instanceGroupManagers/test-worker-uid",
},
},
"listing instance group managers is not dependant on resource name": {
name: proto.String("some-instance-group-manager"),
groupID: proto.String("projects/project/zones/zone/instanceGroupManagers/some-instance-group-manager"),
templateRef: proto.String("projects/project/global/instanceTemplates/some-instance-group-template"),
templateLabels: map[string]string{
"constellation-uid": "uid",
"constellation-role": "control-plane",
},
wantControlPlanes: []string{
"projects/project/zones/zone/instanceGroupManagers/some-instance-group-manager",
},
},
"unrelated instance group manager": {
name: proto.String("test-unrelated-uid"),
groupID: proto.String("projects/project/zones/zone/instanceGroupManagers/test-unrelated-uid"),
name: proto.String("test-control-plane-uid"),
groupID: proto.String("projects/project/zones/zone/instanceGroupManagers/test-unrelated-uid"),
templateRef: proto.String("projects/project/global/instanceTemplates/test-control-plane-uid"),
templateLabels: map[string]string{
"label": "value",
},
},
"invalid instance group manager": {},
}
@ -365,10 +401,19 @@ func TestListScalingGroups(t *testing.T) {
instanceGroupManagersAPI: &stubInstanceGroupManagersAPI{
aggregatedListErr: tc.listInstanceGroupManagersErr,
instanceGroupManager: &computepb.InstanceGroupManager{
Name: tc.name,
SelfLink: tc.groupID,
Name: tc.name,
SelfLink: tc.groupID,
InstanceTemplate: tc.templateRef,
},
},
instanceTemplateAPI: &stubInstanceTemplateAPI{
template: &computepb.InstanceTemplate{
Properties: &computepb.InstanceProperties{
Labels: tc.templateLabels,
},
},
getErr: tc.templateGetErr,
},
}
gotControlPlanes, gotWorkers, err := client.ListScalingGroups(context.Background(), "uid")
if tc.wantErr {