diff --git a/bootstrapper/cmd/bootstrapper/main.go b/bootstrapper/cmd/bootstrapper/main.go index 80ccdee29..9a53fcc59 100644 --- a/bootstrapper/cmd/bootstrapper/main.go +++ b/bootstrapper/cmd/bootstrapper/main.go @@ -105,9 +105,8 @@ func main() { log.With(zap.Error(err)).Fatalf("Failed to set up cloud logger") } - cloudControllerManager := &awscloud.CloudControllerManager{} clusterInitJoiner = kubernetes.New( - "aws", k8sapi.NewKubernetesUtil(), &k8sapi.KubdeadmConfiguration{}, kubectl.New(), cloudControllerManager, + "aws", k8sapi.NewKubernetesUtil(), &k8sapi.KubdeadmConfiguration{}, kubectl.New(), metadata, pcrsJSON, helmClient, &kubewaiter.CloudKubeAPIWaiter{}, ) openTPM = vtpm.OpenVTPM @@ -121,30 +120,24 @@ func main() { issuer = initserver.NewIssuerWrapper(gcp.NewIssuer(), vmtype.Unknown, nil) - gcpClient, err := gcpcloud.NewClient(ctx) + metadata, err := gcpcloud.New(ctx) if err != nil { log.With(zap.Error(err)).Fatalf("Failed to create GCP metadata client") } - metadata := gcpcloud.New(gcpClient) - descr, err := metadata.Self(ctx) - if err != nil { - log.With(zap.Error(err)).Fatalf("Failed to get instance metadata") - } - cloudLogger, err = gcpcloud.NewLogger(ctx, descr.ProviderID, "constellation-boot-log") + defer metadata.Close() + + cloudLogger, err = gcpcloud.NewLogger(ctx, "constellation-boot-log") if err != nil { log.With(zap.Error(err)).Fatalf("Failed to set up cloud logger") } + metadataAPI = metadata pcrsJSON, err := json.Marshal(pcrs) if err != nil { log.With(zap.Error(err)).Fatalf("Failed to marshal PCRs") } - cloudControllerManager, err := gcpcloud.NewCloudControllerManager(ctx, metadata) - if err != nil { - log.With(zap.Error(err)).Fatalf("Failed to create cloud controller manager") - } clusterInitJoiner = kubernetes.New( - "gcp", k8sapi.NewKubernetesUtil(), &k8sapi.KubdeadmConfiguration{}, kubectl.New(), cloudControllerManager, + "gcp", k8sapi.NewKubernetesUtil(), &k8sapi.KubdeadmConfiguration{}, kubectl.New(), metadata, pcrsJSON, helmClient, &kubewaiter.CloudKubeAPIWaiter{}, ) openTPM = vtpm.OpenVTPM @@ -178,7 +171,7 @@ func main() { log.With(zap.Error(err)).Fatalf("Failed to marshal PCRs") } clusterInitJoiner = kubernetes.New( - "azure", k8sapi.NewKubernetesUtil(), &k8sapi.KubdeadmConfiguration{}, kubectl.New(), azurecloud.NewCloudControllerManager(metadata), + "azure", k8sapi.NewKubernetesUtil(), &k8sapi.KubdeadmConfiguration{}, kubectl.New(), metadata, pcrsJSON, helmClient, &kubewaiter.CloudKubeAPIWaiter{}, ) @@ -200,7 +193,7 @@ func main() { log.With(zap.Error(err)).Fatalf("Failed to marshal PCRs") } clusterInitJoiner = kubernetes.New( - "qemu", k8sapi.NewKubernetesUtil(), &k8sapi.KubdeadmConfiguration{}, kubectl.New(), &qemucloud.CloudControllerManager{}, + "qemu", k8sapi.NewKubernetesUtil(), &k8sapi.KubdeadmConfiguration{}, kubectl.New(), metadata, pcrsJSON, helmClient, &kubewaiter.CloudKubeAPIWaiter{}, ) metadataAPI = metadata diff --git a/bootstrapper/cmd/bootstrapper/test.go b/bootstrapper/cmd/bootstrapper/test.go index 984fd53a8..449dc0226 100644 --- a/bootstrapper/cmd/bootstrapper/test.go +++ b/bootstrapper/cmd/bootstrapper/test.go @@ -52,18 +52,6 @@ func (f *providerMetadataFake) Self(ctx context.Context) (metadata.InstanceMetad }, nil } -func (f *providerMetadataFake) SignalRole(ctx context.Context, role role.Role) error { - return nil -} - -func (f *providerMetadataFake) SetVPNIP(ctx context.Context, vpnIP string) error { - return nil -} - func (f *providerMetadataFake) GetLoadBalancerEndpoint(ctx context.Context) (string, error) { return "", nil } - -func (f *providerMetadataFake) Supported() bool { - return true -} diff --git a/bootstrapper/internal/kubernetes/cloud_provider.go b/bootstrapper/internal/kubernetes/cloud_provider.go index a81ef8cfb..249f1de07 100644 --- a/bootstrapper/internal/kubernetes/cloud_provider.go +++ b/bootstrapper/internal/kubernetes/cloud_provider.go @@ -10,9 +10,6 @@ import ( "context" "github.com/edgelesssys/constellation/v2/internal/cloud/metadata" - "github.com/edgelesssys/constellation/v2/internal/kubernetes" - "github.com/edgelesssys/constellation/v2/internal/versions" - k8s "k8s.io/api/core/v1" ) // ProviderMetadata implementers read/write cloud provider metadata. @@ -27,63 +24,6 @@ type ProviderMetadata interface { GetLoadBalancerEndpoint(ctx context.Context) (string, error) // GetInstance retrieves an instance using its providerID. GetInstance(ctx context.Context, providerID string) (metadata.InstanceMetadata, error) - // Supported is used to determine if metadata API is implemented for this cloud provider. - Supported() bool -} - -// CloudControllerManager implementers provide configuration for the k8s cloud-controller-manager. -type CloudControllerManager interface { - // Image returns the container image used to provide cloud-controller-manager for the cloud-provider. - Image(k8sVersion versions.ValidK8sVersion) (string, error) - // Path returns the path used by cloud-controller-manager executable within the container image. - Path() string - // Name returns the cloud-provider name as used by k8s cloud-controller-manager (k8s.gcr.io/cloud-controller-manager). - Name() string - // ExtraArgs returns a list of arguments to append to the cloud-controller-manager command. - ExtraArgs() []string - // ConfigMaps returns a list of ConfigMaps to deploy together with the k8s cloud-controller-manager - // Reference: https://kubernetes.io/docs/concepts/configuration/configmap/ . - ConfigMaps() (kubernetes.ConfigMaps, error) - // Secrets returns a list of secrets to deploy together with the k8s cloud-controller-manager. - // Reference: https://kubernetes.io/docs/concepts/configuration/secret/ . - Secrets(ctx context.Context, providerID, cloudServiceAccountURI string) (kubernetes.Secrets, error) - // Volumes returns a list of volumes to deploy together with the k8s cloud-controller-manager. - // Reference: https://kubernetes.io/docs/concepts/storage/volumes/ . - Volumes() []k8s.Volume - // VolumeMounts a list of of volume mounts to deploy together with the k8s cloud-controller-manager. - VolumeMounts() []k8s.VolumeMount - // Env returns a list of k8s environment key-value pairs to deploy together with the k8s cloud-controller-manager. - Env() []k8s.EnvVar - // Supported is used to determine if cloud controller manager is implemented for this cloud provider. - Supported() bool -} - -// CloudNodeManager implementers provide configuration for the k8s cloud-node-manager. -type CloudNodeManager interface { - // Image returns the container image used to provide cloud-node-manager for the cloud-provider. - Image(k8sVersion versions.ValidK8sVersion) (string, error) - // Path returns the path used by cloud-node-manager executable within the container image. - Path() string - // ExtraArgs returns a list of arguments to append to the cloud-node-manager command. - ExtraArgs() []string - // Supported is used to determine if cloud node manager is implemented for this cloud provider. - Supported() bool -} - -// ClusterAutoscaler implementers provide configuration for the k8s cluster-autoscaler. -type ClusterAutoscaler interface { - // Name returns the cloud-provider name as used by k8s cluster-autoscaler. - Name() string - // Secrets returns a list of secrets to deploy together with the k8s cluster-autoscaler. - Secrets(providerID, cloudServiceAccountURI string) (kubernetes.Secrets, error) - // Volumes returns a list of volumes to deploy together with the k8s cluster-autoscaler. - Volumes() []k8s.Volume - // VolumeMounts returns a list of volume mounts to deploy together with the k8s cluster-autoscaler. - VolumeMounts() []k8s.VolumeMount - // Env returns a list of k8s environment key-value pairs to deploy together with the k8s cluster-autoscaler. - Env() []k8s.EnvVar - // Supported is used to determine if cluster autoscaler is implemented for this cloud provider. - Supported() bool } type stubProviderMetadata struct { @@ -99,9 +39,6 @@ type stubProviderMetadata struct { GetInstanceErr error GetInstanceResp metadata.InstanceMetadata - SupportedResp bool - SupportsLoadBalancerResp bool - UIDErr error UIDResp string } @@ -122,90 +59,6 @@ func (m *stubProviderMetadata) GetInstance(ctx context.Context, providerID strin return m.GetInstanceResp, m.GetInstanceErr } -func (m *stubProviderMetadata) Supported() bool { - return m.SupportedResp -} - -func (m *stubProviderMetadata) SupportsLoadBalancer() bool { - return m.SupportsLoadBalancerResp -} - func (m *stubProviderMetadata) UID(ctx context.Context) (string, error) { return m.UIDResp, m.UIDErr } - -type stubCloudControllerManager struct { - SupportedResp bool -} - -func (m *stubCloudControllerManager) Image(k8sVersion versions.ValidK8sVersion) (string, error) { - return "stub-image:latest", nil -} - -func (m *stubCloudControllerManager) Path() string { - return "/stub-controller-manager" -} - -func (m *stubCloudControllerManager) Name() string { - return "stub" -} - -func (m *stubCloudControllerManager) ExtraArgs() []string { - return []string{} -} - -func (m *stubCloudControllerManager) ConfigMaps() (kubernetes.ConfigMaps, error) { - return []*k8s.ConfigMap{}, nil -} - -func (m *stubCloudControllerManager) Secrets(ctx context.Context, instance, cloudServiceAccountURI string) (kubernetes.Secrets, error) { - return []*k8s.Secret{}, nil -} - -func (m *stubCloudControllerManager) Volumes() []k8s.Volume { - return []k8s.Volume{} -} - -func (m *stubCloudControllerManager) VolumeMounts() []k8s.VolumeMount { - return []k8s.VolumeMount{} -} - -func (m *stubCloudControllerManager) Env() []k8s.EnvVar { - return []k8s.EnvVar{} -} - -func (m *stubCloudControllerManager) Supported() bool { - return m.SupportedResp -} - -type stubClusterAutoscaler struct { - SupportedResp bool -} - -func (a *stubClusterAutoscaler) Name() string { - return "stub" -} - -// Secrets returns a list of secrets to deploy together with the k8s cluster-autoscaler. -func (a *stubClusterAutoscaler) Secrets(instance, cloudServiceAccountURI string) (kubernetes.Secrets, error) { - return kubernetes.Secrets{}, nil -} - -// Volumes returns a list of volumes to deploy together with the k8s cluster-autoscaler. -func (a *stubClusterAutoscaler) Volumes() []k8s.Volume { - return []k8s.Volume{} -} - -// VolumeMounts returns a list of volume mounts to deploy together with the k8s cluster-autoscaler. -func (a *stubClusterAutoscaler) VolumeMounts() []k8s.VolumeMount { - return []k8s.VolumeMount{} -} - -// Env returns a list of k8s environment key-value pairs to deploy together with the k8s cluster-autoscaler. -func (a *stubClusterAutoscaler) Env() []k8s.EnvVar { - return []k8s.EnvVar{} -} - -func (a *stubClusterAutoscaler) Supported() bool { - return a.SupportedResp -} diff --git a/bootstrapper/internal/kubernetes/kubernetes.go b/bootstrapper/internal/kubernetes/kubernetes.go index 06cbc19e8..b65185e1f 100644 --- a/bootstrapper/internal/kubernetes/kubernetes.go +++ b/bootstrapper/internal/kubernetes/kubernetes.go @@ -21,12 +21,11 @@ import ( "github.com/edgelesssys/constellation/v2/bootstrapper/internal/kubernetes/k8sapi" "github.com/edgelesssys/constellation/v2/bootstrapper/internal/kubernetes/k8sapi/resources" kubewaiter "github.com/edgelesssys/constellation/v2/bootstrapper/internal/kubernetes/kubeWaiter" - "github.com/edgelesssys/constellation/v2/internal/azureshared" + "github.com/edgelesssys/constellation/v2/internal/cloud/azureshared" "github.com/edgelesssys/constellation/v2/internal/cloud/cloudprovider" - "github.com/edgelesssys/constellation/v2/internal/cloud/metadata" + "github.com/edgelesssys/constellation/v2/internal/cloud/gcpshared" "github.com/edgelesssys/constellation/v2/internal/constants" "github.com/edgelesssys/constellation/v2/internal/deploy/helm" - "github.com/edgelesssys/constellation/v2/internal/gcpshared" "github.com/edgelesssys/constellation/v2/internal/logger" "github.com/edgelesssys/constellation/v2/internal/role" "github.com/edgelesssys/constellation/v2/internal/versions" @@ -61,14 +60,13 @@ type KubeWrapper struct { configProvider configurationProvider client k8sapi.Client kubeconfigReader configReader - cloudControllerManager CloudControllerManager providerMetadata ProviderMetadata initialMeasurementsJSON []byte getIPAddr func() (string, error) } // New creates a new KubeWrapper with real values. -func New(cloudProvider string, clusterUtil clusterUtil, configProvider configurationProvider, client k8sapi.Client, cloudControllerManager CloudControllerManager, +func New(cloudProvider string, clusterUtil clusterUtil, configProvider configurationProvider, client k8sapi.Client, providerMetadata ProviderMetadata, initialMeasurementsJSON []byte, helmClient helmClient, kubeAPIWaiter kubeAPIWaiter, ) *KubeWrapper { return &KubeWrapper{ @@ -79,7 +77,6 @@ func New(cloudProvider string, clusterUtil clusterUtil, configProvider configura configProvider: configProvider, client: client, kubeconfigReader: &KubeconfigReader{fs: afero.Afero{Fs: afero.NewOsFs()}}, - cloudControllerManager: cloudControllerManager, providerMetadata: providerMetadata, initialMeasurementsJSON: initialMeasurementsJSON, getIPAddr: getIPAddr, @@ -101,56 +98,47 @@ func (k *KubeWrapper) InitCluster( return nil, err } - ip, err := k.getIPAddr() - if err != nil { - return nil, err - } - nodeName := ip - var providerID string - var instance metadata.InstanceMetadata var nodePodCIDR string - var subnetworkPodCIDR string - var controlPlaneEndpoint string // this is the endpoint in "kubeadm init --control-plane-endpoint=:" - var nodeIP string var validIPs []net.IP // Step 1: retrieve cloud metadata for Kubernetes configuration - if k.providerMetadata.Supported() { - log.Infof("Retrieving node metadata") - instance, err = k.providerMetadata.Self(ctx) - if err != nil { - return nil, fmt.Errorf("retrieving own instance metadata: %w", err) - } - if instance.VPCIP != "" { - validIPs = append(validIPs, net.ParseIP(instance.VPCIP)) - } - nodeName = k8sCompliantHostname(instance.Name) - providerID = instance.ProviderID - nodeIP = instance.VPCIP - subnetworkPodCIDR = instance.SecondaryIPRange - - if len(instance.AliasIPRanges) > 0 { - nodePodCIDR = instance.AliasIPRanges[0] - } - controlPlaneEndpoint, err = k.providerMetadata.GetLoadBalancerEndpoint(ctx) - if err != nil { - return nil, fmt.Errorf("retrieving load balancer endpoint: %w", err) - } + log.Infof("Retrieving node metadata") + instance, err := k.providerMetadata.Self(ctx) + if err != nil { + return nil, fmt.Errorf("retrieving own instance metadata: %w", err) } + if instance.VPCIP != "" { + validIPs = append(validIPs, net.ParseIP(instance.VPCIP)) + } + nodeName := k8sCompliantHostname(instance.Name) + nodeIP := instance.VPCIP + subnetworkPodCIDR := instance.SecondaryIPRange + if len(instance.AliasIPRanges) > 0 { + nodePodCIDR = instance.AliasIPRanges[0] + } + + // this is the endpoint in "kubeadm init --control-plane-endpoint=:" + controlPlaneEndpoint, err := k.providerMetadata.GetLoadBalancerEndpoint(ctx) + if err != nil { + return nil, fmt.Errorf("retrieving load balancer endpoint: %w", err) + } + log.With( zap.String("nodeName", nodeName), - zap.String("providerID", providerID), + zap.String("providerID", instance.ProviderID), zap.String("nodeIP", nodeIP), zap.String("controlPlaneEndpoint", controlPlaneEndpoint), zap.String("podCIDR", subnetworkPodCIDR), ).Infof("Setting information for node") // Step 2: configure kubeadm init config - initConfig := k.configProvider.InitConfiguration(k.cloudControllerManager.Supported(), k8sVersion) + ccmSupported := cloudprovider.FromString(k.cloudProvider) == cloudprovider.Azure || + cloudprovider.FromString(k.cloudProvider) == cloudprovider.GCP + initConfig := k.configProvider.InitConfiguration(ccmSupported, k8sVersion) initConfig.SetNodeIP(nodeIP) initConfig.SetCertSANs([]string{nodeIP}) initConfig.SetNodeName(nodeName) - initConfig.SetProviderID(providerID) + initConfig.SetProviderID(instance.ProviderID) initConfig.SetControlPlaneEndpoint(controlPlaneEndpoint) initConfigYAML, err := initConfig.Marshal() if err != nil { @@ -262,28 +250,19 @@ func (k *KubeWrapper) JoinCluster(ctx context.Context, args *kubeadm.BootstrapTo } // Step 1: retrieve cloud metadata for Kubernetes configuration - nodeInternalIP, err := k.getIPAddr() + log.Infof("Retrieving node metadata") + instance, err := k.providerMetadata.Self(ctx) if err != nil { - return err + return fmt.Errorf("retrieving own instance metadata: %w", err) } - nodeName := nodeInternalIP - var providerID string - var loadbalancerEndpoint string - if k.providerMetadata.Supported() { - log.Infof("Retrieving node metadata") - instance, err := k.providerMetadata.Self(ctx) - if err != nil { - return fmt.Errorf("retrieving own instance metadata: %w", err) - } - providerID = instance.ProviderID - nodeName = instance.Name - nodeInternalIP = instance.VPCIP - loadbalancerEndpoint, err = k.providerMetadata.GetLoadBalancerEndpoint(ctx) - if err != nil { - return fmt.Errorf("retrieving loadbalancer endpoint: %w", err) - } + providerID := instance.ProviderID + nodeInternalIP := instance.VPCIP + nodeName := k8sCompliantHostname(instance.Name) + + loadbalancerEndpoint, err := k.providerMetadata.GetLoadBalancerEndpoint(ctx) + if err != nil { + return fmt.Errorf("retrieving own instance metadata: %w", err) } - nodeName = k8sCompliantHostname(nodeName) log.With( zap.String("nodeName", nodeName), @@ -292,7 +271,9 @@ func (k *KubeWrapper) JoinCluster(ctx context.Context, args *kubeadm.BootstrapTo ).Infof("Setting information for node") // Step 2: configure kubeadm join config - joinConfig := k.configProvider.JoinConfiguration(k.cloudControllerManager.Supported()) + ccmSupported := cloudprovider.FromString(k.cloudProvider) == cloudprovider.Azure || + cloudprovider.FromString(k.cloudProvider) == cloudprovider.GCP + joinConfig := k.configProvider.JoinConfiguration(ccmSupported) joinConfig.SetAPIServerEndpoint(args.APIServerEndpoint) joinConfig.SetToken(args.Token) joinConfig.AppendDiscoveryTokenCaCertHash(args.CACertHashes[0]) @@ -443,85 +424,85 @@ func (k *KubeWrapper) setupExtraVals(ctx context.Context, initialMeasurementsJSO switch cloudprovider.FromString(k.cloudProvider) { case cloudprovider.GCP: - { - uid, err := k.providerMetadata.UID(ctx) - if err != nil { - return nil, fmt.Errorf("getting uid: %w", err) - } - - projectID, _, _, err := gcpshared.SplitProviderID(instance.ProviderID) - if err != nil { - return nil, fmt.Errorf("splitting providerID: %w", err) - } - - serviceAccountKey, err := gcpshared.ServiceAccountKeyFromURI(cloudServiceAccountURI) - if err != nil { - return nil, fmt.Errorf("getting service account key: %w", err) - } - rawKey, err := json.Marshal(serviceAccountKey) - if err != nil { - return nil, fmt.Errorf("marshaling service account key: %w", err) - } - - ccmVals, ok := extraVals["ccm"].(map[string]any) - if !ok { - return nil, errors.New("invalid ccm values") - } - ccmVals["GCP"] = map[string]any{ - "projectID": projectID, - "uid": uid, - "secretData": string(rawKey), - "subnetworkPodCIDR": subnetworkPodCIDR, - } + uid, err := k.providerMetadata.UID(ctx) + if err != nil { + return nil, fmt.Errorf("getting uid: %w", err) } + + projectID, _, _, err := gcpshared.SplitProviderID(instance.ProviderID) + if err != nil { + return nil, fmt.Errorf("splitting providerID: %w", err) + } + + serviceAccountKey, err := gcpshared.ServiceAccountKeyFromURI(cloudServiceAccountURI) + if err != nil { + return nil, fmt.Errorf("getting service account key: %w", err) + } + rawKey, err := json.Marshal(serviceAccountKey) + if err != nil { + return nil, fmt.Errorf("marshaling service account key: %w", err) + } + + ccmVals, ok := extraVals["ccm"].(map[string]any) + if !ok { + return nil, errors.New("invalid ccm values") + } + ccmVals["GCP"] = map[string]any{ + "projectID": projectID, + "uid": uid, + "secretData": string(rawKey), + "subnetworkPodCIDR": subnetworkPodCIDR, + } + case cloudprovider.Azure: - { - // TODO: After refactoring the ProviderMetadata interface this section should be rewritten. - // Currently, we have to rely on the Secrets(..) method, as GetNetworkSecurityGroupName & GetLoadBalancerName - // rely on Azure specific API endpoints. - ccmSecrets, err := k.cloudControllerManager.Secrets(ctx, instance.ProviderID, cloudServiceAccountURI) - if err != nil { - return nil, fmt.Errorf("creating ccm secret: %w", err) - } - if len(ccmSecrets) < 1 { - return nil, errors.New("missing secret") - } - rawConfig := ccmSecrets[0].Data["azure.json"] - - ccmVals, ok := extraVals["ccm"].(map[string]any) - if !ok { - return nil, errors.New("invalid ccm values") - } - ccmVals["Azure"] = map[string]any{ - "azureConfig": string(rawConfig), - "subnetworkPodCIDR": subnetworkPodCIDR, - } - - joinVals, ok := extraVals["join-service"].(map[string]any) - if !ok { - return nil, errors.New("invalid join-service values") - } - joinVals["idkeydigest"] = hex.EncodeToString(idkeydigest) - - subscriptionID, resourceGroup, err := azureshared.BasicsFromProviderID(instance.ProviderID) - if err != nil { - return nil, err - } - creds, err := azureshared.ApplicationCredentialsFromURI(cloudServiceAccountURI) - if err != nil { - return nil, err - } - - extraVals["autoscaler"] = map[string]any{ - "Azure": map[string]any{ - "clientID": creds.AppClientID, - "clientSecret": creds.ClientSecretValue, - "resourceGroup": resourceGroup, - "subscriptionID": subscriptionID, - "tenantID": creds.TenantID, - }, - } + ccmAzure, ok := k.providerMetadata.(ccmConfigGetter) + if !ok { + return nil, errors.New("invalid cloud provider metadata for Azure") } + + ccmConfig, err := ccmAzure.GetCCMConfig(ctx, instance.ProviderID, cloudServiceAccountURI) + if err != nil { + return nil, fmt.Errorf("creating ccm secret: %w", err) + } + + ccmVals, ok := extraVals["ccm"].(map[string]any) + if !ok { + return nil, errors.New("invalid ccm values") + } + ccmVals["Azure"] = map[string]any{ + "azureConfig": string(ccmConfig), + "subnetworkPodCIDR": subnetworkPodCIDR, + } + + joinVals, ok := extraVals["join-service"].(map[string]any) + if !ok { + return nil, errors.New("invalid join-service values") + } + joinVals["idkeydigest"] = hex.EncodeToString(idkeydigest) + + subscriptionID, resourceGroup, err := azureshared.BasicsFromProviderID(instance.ProviderID) + if err != nil { + return nil, err + } + creds, err := azureshared.ApplicationCredentialsFromURI(cloudServiceAccountURI) + if err != nil { + return nil, err + } + + extraVals["autoscaler"] = map[string]any{ + "Azure": map[string]any{ + "clientID": creds.AppClientID, + "clientSecret": creds.ClientSecretValue, + "resourceGroup": resourceGroup, + "subscriptionID": subscriptionID, + "tenantID": creds.TenantID, + }, + } + } return extraVals, nil } + +type ccmConfigGetter interface { + GetCCMConfig(ctx context.Context, providerID, cloudServiceAccountURI string) ([]byte, error) +} diff --git a/bootstrapper/internal/kubernetes/kubernetes_test.go b/bootstrapper/internal/kubernetes/kubernetes_test.go index 4f5185232..7a70eb591 100644 --- a/bootstrapper/internal/kubernetes/kubernetes_test.go +++ b/bootstrapper/internal/kubernetes/kubernetes_test.go @@ -45,41 +45,16 @@ func TestInitCluster(t *testing.T) { aliasIPRange := "192.0.2.0/24" testCases := map[string]struct { - clusterUtil stubClusterUtil - helmClient stubHelmClient - kubectl stubKubectl - kubeAPIWaiter stubKubeAPIWaiter - providerMetadata ProviderMetadata - CloudControllerManager CloudControllerManager - ClusterAutoscaler ClusterAutoscaler - kubeconfigReader configReader - wantConfig k8sapi.KubeadmInitYAML - wantErr bool - k8sVersion versions.ValidK8sVersion + clusterUtil stubClusterUtil + helmClient stubHelmClient + kubectl stubKubectl + kubeAPIWaiter stubKubeAPIWaiter + providerMetadata ProviderMetadata + kubeconfigReader configReader + wantConfig k8sapi.KubeadmInitYAML + wantErr bool + k8sVersion versions.ValidK8sVersion }{ - "kubeadm init works without metadata": { - clusterUtil: stubClusterUtil{}, - kubeconfigReader: &stubKubeconfigReader{ - Kubeconfig: []byte("someKubeconfig"), - }, - kubeAPIWaiter: stubKubeAPIWaiter{}, - providerMetadata: &stubProviderMetadata{SupportedResp: false}, - CloudControllerManager: &stubCloudControllerManager{}, - ClusterAutoscaler: &stubClusterAutoscaler{}, - wantConfig: k8sapi.KubeadmInitYAML{ - InitConfiguration: kubeadm.InitConfiguration{ - NodeRegistration: kubeadm.NodeRegistrationOptions{ - KubeletExtraArgs: map[string]string{ - "node-ip": "", - "provider-id": "", - }, - Name: privateIP, - }, - }, - ClusterConfiguration: kubeadm.ClusterConfiguration{}, - }, - k8sVersion: versions.Default, - }, "kubeadm init works with metadata and loadbalancer": { clusterUtil: stubClusterUtil{}, kubeconfigReader: &stubKubeconfigReader{ @@ -87,7 +62,6 @@ func TestInitCluster(t *testing.T) { }, kubeAPIWaiter: stubKubeAPIWaiter{}, providerMetadata: &stubProviderMetadata{ - SupportedResp: true, SelfResp: metadata.InstanceMetadata{ Name: nodeName, ProviderID: providerID, @@ -95,10 +69,7 @@ func TestInitCluster(t *testing.T) { AliasIPRanges: []string{aliasIPRange}, }, GetLoadBalancerEndpointResp: loadbalancerIP, - SupportsLoadBalancerResp: true, }, - CloudControllerManager: &stubCloudControllerManager{}, - ClusterAutoscaler: &stubClusterAutoscaler{}, wantConfig: k8sapi.KubeadmInitYAML{ InitConfiguration: kubeadm.InitConfiguration{ NodeRegistration: kubeadm.NodeRegistrationOptions{ @@ -126,13 +97,10 @@ func TestInitCluster(t *testing.T) { }, kubeAPIWaiter: stubKubeAPIWaiter{}, providerMetadata: &stubProviderMetadata{ - SelfErr: someErr, - SupportedResp: true, + SelfErr: someErr, }, - CloudControllerManager: &stubCloudControllerManager{}, - ClusterAutoscaler: &stubClusterAutoscaler{}, - wantErr: true, - k8sVersion: versions.Default, + wantErr: true, + k8sVersion: versions.Default, }, "kubeadm init fails when retrieving metadata loadbalancer ip": { clusterUtil: stubClusterUtil{}, @@ -141,25 +109,19 @@ func TestInitCluster(t *testing.T) { }, providerMetadata: &stubProviderMetadata{ GetLoadBalancerEndpointErr: someErr, - SupportsLoadBalancerResp: true, - SupportedResp: true, }, - CloudControllerManager: &stubCloudControllerManager{}, - ClusterAutoscaler: &stubClusterAutoscaler{}, - wantErr: true, - k8sVersion: versions.Default, + wantErr: true, + k8sVersion: versions.Default, }, "kubeadm init fails when applying the init config": { clusterUtil: stubClusterUtil{initClusterErr: someErr}, kubeconfigReader: &stubKubeconfigReader{ Kubeconfig: []byte("someKubeconfig"), }, - kubeAPIWaiter: stubKubeAPIWaiter{}, - providerMetadata: &stubProviderMetadata{}, - CloudControllerManager: &stubCloudControllerManager{}, - ClusterAutoscaler: &stubClusterAutoscaler{}, - wantErr: true, - k8sVersion: versions.Default, + kubeAPIWaiter: stubKubeAPIWaiter{}, + providerMetadata: &stubProviderMetadata{}, + wantErr: true, + k8sVersion: versions.Default, }, "kubeadm init fails when deploying cilium": { clusterUtil: stubClusterUtil{}, @@ -167,11 +129,9 @@ func TestInitCluster(t *testing.T) { kubeconfigReader: &stubKubeconfigReader{ Kubeconfig: []byte("someKubeconfig"), }, - providerMetadata: &stubProviderMetadata{}, - CloudControllerManager: &stubCloudControllerManager{}, - ClusterAutoscaler: &stubClusterAutoscaler{}, - wantErr: true, - k8sVersion: versions.Default, + providerMetadata: &stubProviderMetadata{}, + wantErr: true, + k8sVersion: versions.Default, }, "kubeadm init fails when setting up constellation-services chart": { clusterUtil: stubClusterUtil{}, @@ -179,12 +139,10 @@ func TestInitCluster(t *testing.T) { kubeconfigReader: &stubKubeconfigReader{ Kubeconfig: []byte("someKubeconfig"), }, - kubeAPIWaiter: stubKubeAPIWaiter{}, - providerMetadata: &stubProviderMetadata{}, - CloudControllerManager: &stubCloudControllerManager{SupportedResp: true}, - ClusterAutoscaler: &stubClusterAutoscaler{}, - wantErr: true, - k8sVersion: versions.Default, + kubeAPIWaiter: stubKubeAPIWaiter{}, + providerMetadata: &stubProviderMetadata{}, + wantErr: true, + k8sVersion: versions.Default, }, "kubeadm init fails when setting the cloud node manager": { clusterUtil: stubClusterUtil{}, @@ -192,12 +150,10 @@ func TestInitCluster(t *testing.T) { kubeconfigReader: &stubKubeconfigReader{ Kubeconfig: []byte("someKubeconfig"), }, - kubeAPIWaiter: stubKubeAPIWaiter{}, - providerMetadata: &stubProviderMetadata{}, - CloudControllerManager: &stubCloudControllerManager{}, - ClusterAutoscaler: &stubClusterAutoscaler{}, - wantErr: true, - k8sVersion: versions.Default, + kubeAPIWaiter: stubKubeAPIWaiter{}, + providerMetadata: &stubProviderMetadata{}, + wantErr: true, + k8sVersion: versions.Default, }, "kubeadm init fails when setting the cluster autoscaler": { clusterUtil: stubClusterUtil{}, @@ -205,72 +161,60 @@ func TestInitCluster(t *testing.T) { kubeconfigReader: &stubKubeconfigReader{ Kubeconfig: []byte("someKubeconfig"), }, - kubeAPIWaiter: stubKubeAPIWaiter{}, - providerMetadata: &stubProviderMetadata{}, - CloudControllerManager: &stubCloudControllerManager{}, - ClusterAutoscaler: &stubClusterAutoscaler{SupportedResp: true}, - wantErr: true, - k8sVersion: versions.Default, + kubeAPIWaiter: stubKubeAPIWaiter{}, + providerMetadata: &stubProviderMetadata{}, + wantErr: true, + k8sVersion: versions.Default, }, "kubeadm init fails when reading kubeconfig": { clusterUtil: stubClusterUtil{}, kubeconfigReader: &stubKubeconfigReader{ ReadErr: someErr, }, - kubeAPIWaiter: stubKubeAPIWaiter{}, - providerMetadata: &stubProviderMetadata{}, - CloudControllerManager: &stubCloudControllerManager{}, - ClusterAutoscaler: &stubClusterAutoscaler{}, - wantErr: true, - k8sVersion: versions.Default, + kubeAPIWaiter: stubKubeAPIWaiter{}, + providerMetadata: &stubProviderMetadata{}, + wantErr: true, + k8sVersion: versions.Default, }, "kubeadm init fails when setting up konnectivity": { clusterUtil: stubClusterUtil{setupKonnectivityError: someErr}, kubeconfigReader: &stubKubeconfigReader{ Kubeconfig: []byte("someKubeconfig"), }, - kubeAPIWaiter: stubKubeAPIWaiter{}, - providerMetadata: &stubProviderMetadata{SupportedResp: false}, - CloudControllerManager: &stubCloudControllerManager{}, - ClusterAutoscaler: &stubClusterAutoscaler{}, - wantErr: true, - k8sVersion: versions.Default, + kubeAPIWaiter: stubKubeAPIWaiter{}, + providerMetadata: &stubProviderMetadata{}, + wantErr: true, + k8sVersion: versions.Default, }, "kubeadm init fails when setting up verification service": { clusterUtil: stubClusterUtil{setupVerificationServiceErr: someErr}, kubeconfigReader: &stubKubeconfigReader{ Kubeconfig: []byte("someKubeconfig"), }, - kubeAPIWaiter: stubKubeAPIWaiter{}, - providerMetadata: &stubProviderMetadata{SupportedResp: false}, - CloudControllerManager: &stubCloudControllerManager{}, - ClusterAutoscaler: &stubClusterAutoscaler{}, - wantErr: true, - k8sVersion: versions.Default, + kubeAPIWaiter: stubKubeAPIWaiter{}, + providerMetadata: &stubProviderMetadata{}, + wantErr: true, + k8sVersion: versions.Default, }, "kubeadm init fails when waiting for kubeAPI server": { clusterUtil: stubClusterUtil{}, kubeconfigReader: &stubKubeconfigReader{ Kubeconfig: []byte("someKubeconfig"), }, - kubeAPIWaiter: stubKubeAPIWaiter{waitErr: someErr}, - providerMetadata: &stubProviderMetadata{SupportedResp: false}, - CloudControllerManager: &stubCloudControllerManager{}, - ClusterAutoscaler: &stubClusterAutoscaler{}, - k8sVersion: versions.Default, - wantErr: true, + kubeAPIWaiter: stubKubeAPIWaiter{waitErr: someErr}, + providerMetadata: &stubProviderMetadata{}, + k8sVersion: versions.Default, + wantErr: true, }, "unsupported k8sVersion fails cluster creation": { clusterUtil: stubClusterUtil{}, kubeconfigReader: &stubKubeconfigReader{ Kubeconfig: []byte("someKubeconfig"), }, - kubeAPIWaiter: stubKubeAPIWaiter{}, - providerMetadata: &stubProviderMetadata{}, - CloudControllerManager: &stubCloudControllerManager{}, - ClusterAutoscaler: &stubClusterAutoscaler{}, - k8sVersion: "1.19", - wantErr: true, + kubeAPIWaiter: stubKubeAPIWaiter{}, + providerMetadata: &stubProviderMetadata{}, + k8sVersion: "1.19", + wantErr: true, }, } @@ -280,15 +224,14 @@ func TestInitCluster(t *testing.T) { require := require.New(t) kube := KubeWrapper{ - clusterUtil: &tc.clusterUtil, - helmClient: &tc.helmClient, - providerMetadata: tc.providerMetadata, - kubeAPIWaiter: &tc.kubeAPIWaiter, - cloudControllerManager: tc.CloudControllerManager, - configProvider: &stubConfigProvider{InitConfig: k8sapi.KubeadmInitYAML{}}, - client: &tc.kubectl, - kubeconfigReader: tc.kubeconfigReader, - getIPAddr: func() (string, error) { return privateIP, nil }, + clusterUtil: &tc.clusterUtil, + helmClient: &tc.helmClient, + providerMetadata: tc.providerMetadata, + kubeAPIWaiter: &tc.kubeAPIWaiter, + configProvider: &stubConfigProvider{InitConfig: k8sapi.KubeadmInitYAML{}}, + client: &tc.kubectl, + kubeconfigReader: tc.kubeconfigReader, + getIPAddr: func() (string, error) { return privateIP, nil }, } _, err := kube.InitCluster( @@ -322,40 +265,22 @@ func TestJoinCluster(t *testing.T) { k8sVersion := versions.Default testCases := map[string]struct { - clusterUtil stubClusterUtil - providerMetadata ProviderMetadata - CloudControllerManager CloudControllerManager - wantConfig kubeadm.JoinConfiguration - role role.Role - wantErr bool + clusterUtil stubClusterUtil + providerMetadata ProviderMetadata + wantConfig kubeadm.JoinConfiguration + role role.Role + wantErr bool }{ - "kubeadm join worker works without metadata": { - clusterUtil: stubClusterUtil{}, - providerMetadata: &stubProviderMetadata{}, - CloudControllerManager: &stubCloudControllerManager{}, - role: role.Worker, - wantConfig: kubeadm.JoinConfiguration{ - Discovery: kubeadm.Discovery{ - BootstrapToken: joinCommand, - }, - NodeRegistration: kubeadm.NodeRegistrationOptions{ - Name: privateIP, - KubeletExtraArgs: map[string]string{"node-ip": privateIP}, - }, - }, - }, "kubeadm join worker works with metadata": { clusterUtil: stubClusterUtil{}, providerMetadata: &stubProviderMetadata{ - SupportedResp: true, SelfResp: metadata.InstanceMetadata{ ProviderID: "provider-id", Name: "metadata-name", VPCIP: "192.0.2.1", }, }, - CloudControllerManager: &stubCloudControllerManager{}, - role: role.Worker, + role: role.Worker, wantConfig: kubeadm.JoinConfiguration{ Discovery: kubeadm.Discovery{ BootstrapToken: joinCommand, @@ -369,16 +294,12 @@ func TestJoinCluster(t *testing.T) { "kubeadm join worker works with metadata and cloud controller manager": { clusterUtil: stubClusterUtil{}, providerMetadata: &stubProviderMetadata{ - SupportedResp: true, SelfResp: metadata.InstanceMetadata{ ProviderID: "provider-id", Name: "metadata-name", VPCIP: "192.0.2.1", }, }, - CloudControllerManager: &stubCloudControllerManager{ - SupportedResp: true, - }, role: role.Worker, wantConfig: kubeadm.JoinConfiguration{ Discovery: kubeadm.Discovery{ @@ -393,15 +314,13 @@ func TestJoinCluster(t *testing.T) { "kubeadm join control-plane node works with metadata": { clusterUtil: stubClusterUtil{}, providerMetadata: &stubProviderMetadata{ - SupportedResp: true, SelfResp: metadata.InstanceMetadata{ ProviderID: "provider-id", Name: "metadata-name", VPCIP: "192.0.2.1", }, }, - CloudControllerManager: &stubCloudControllerManager{}, - role: role.ControlPlane, + role: role.ControlPlane, wantConfig: kubeadm.JoinConfiguration{ Discovery: kubeadm.Discovery{ BootstrapToken: joinCommand, @@ -422,19 +341,16 @@ func TestJoinCluster(t *testing.T) { "kubeadm join worker fails when retrieving self metadata": { clusterUtil: stubClusterUtil{}, providerMetadata: &stubProviderMetadata{ - SupportedResp: true, - SelfErr: someErr, + SelfErr: someErr, }, - CloudControllerManager: &stubCloudControllerManager{}, - role: role.Worker, - wantErr: true, + role: role.Worker, + wantErr: true, }, "kubeadm join worker fails when applying the join config": { - clusterUtil: stubClusterUtil{joinClusterErr: someErr}, - providerMetadata: &stubProviderMetadata{}, - CloudControllerManager: &stubCloudControllerManager{}, - role: role.Worker, - wantErr: true, + clusterUtil: stubClusterUtil{joinClusterErr: someErr}, + providerMetadata: &stubProviderMetadata{}, + role: role.Worker, + wantErr: true, }, } @@ -444,11 +360,10 @@ func TestJoinCluster(t *testing.T) { require := require.New(t) kube := KubeWrapper{ - clusterUtil: &tc.clusterUtil, - providerMetadata: tc.providerMetadata, - cloudControllerManager: tc.CloudControllerManager, - configProvider: &stubConfigProvider{}, - getIPAddr: func() (string, error) { return privateIP, nil }, + clusterUtil: &tc.clusterUtil, + providerMetadata: tc.providerMetadata, + configProvider: &stubConfigProvider{}, + getIPAddr: func() (string, error) { return privateIP, nil }, } err := kube.JoinCluster(context.Background(), joinCommand, tc.role, string(k8sVersion), logger.NewTest(t)) diff --git a/cli/internal/cmd/init.go b/cli/internal/cmd/init.go index 63b366b7a..25605567f 100644 --- a/cli/internal/cmd/init.go +++ b/cli/internal/cmd/init.go @@ -20,14 +20,14 @@ import ( "github.com/edgelesssys/constellation/v2/cli/internal/cloudcmd" "github.com/edgelesssys/constellation/v2/cli/internal/clusterid" "github.com/edgelesssys/constellation/v2/cli/internal/helm" - "github.com/edgelesssys/constellation/v2/internal/azureshared" + "github.com/edgelesssys/constellation/v2/internal/cloud/azureshared" "github.com/edgelesssys/constellation/v2/internal/cloud/cloudprovider" + "github.com/edgelesssys/constellation/v2/internal/cloud/gcpshared" "github.com/edgelesssys/constellation/v2/internal/config" "github.com/edgelesssys/constellation/v2/internal/constants" "github.com/edgelesssys/constellation/v2/internal/crypto" "github.com/edgelesssys/constellation/v2/internal/deploy/ssh" "github.com/edgelesssys/constellation/v2/internal/file" - "github.com/edgelesssys/constellation/v2/internal/gcpshared" "github.com/edgelesssys/constellation/v2/internal/grpc/dialer" grpcRetry "github.com/edgelesssys/constellation/v2/internal/grpc/retry" "github.com/edgelesssys/constellation/v2/internal/license" diff --git a/cli/internal/cmd/init_test.go b/cli/internal/cmd/init_test.go index 118e7a9f5..068e09c1f 100644 --- a/cli/internal/cmd/init_test.go +++ b/cli/internal/cmd/init_test.go @@ -22,10 +22,10 @@ import ( "github.com/edgelesssys/constellation/v2/cli/internal/cloudcmd" "github.com/edgelesssys/constellation/v2/cli/internal/clusterid" "github.com/edgelesssys/constellation/v2/internal/cloud/cloudprovider" + "github.com/edgelesssys/constellation/v2/internal/cloud/gcpshared" "github.com/edgelesssys/constellation/v2/internal/config" "github.com/edgelesssys/constellation/v2/internal/constants" "github.com/edgelesssys/constellation/v2/internal/file" - "github.com/edgelesssys/constellation/v2/internal/gcpshared" "github.com/edgelesssys/constellation/v2/internal/grpc/atlscredentials" "github.com/edgelesssys/constellation/v2/internal/grpc/dialer" "github.com/edgelesssys/constellation/v2/internal/grpc/testdialer" diff --git a/debugd/cmd/debugd/debugd.go b/debugd/cmd/debugd/debugd.go index 8367d1caf..263450a9e 100644 --- a/debugd/cmd/debugd/debugd.go +++ b/debugd/cmd/debugd/debugd.go @@ -20,11 +20,16 @@ import ( "github.com/edgelesssys/constellation/v2/debugd/internal/debugd/metadata/cloudprovider" "github.com/edgelesssys/constellation/v2/debugd/internal/debugd/metadata/fallback" "github.com/edgelesssys/constellation/v2/debugd/internal/debugd/server" + awscloud "github.com/edgelesssys/constellation/v2/internal/cloud/aws" + azurecloud "github.com/edgelesssys/constellation/v2/internal/cloud/azure" platform "github.com/edgelesssys/constellation/v2/internal/cloud/cloudprovider" + gcpcloud "github.com/edgelesssys/constellation/v2/internal/cloud/gcp" + qemucloud "github.com/edgelesssys/constellation/v2/internal/cloud/qemu" "github.com/edgelesssys/constellation/v2/internal/deploy/ssh" "github.com/edgelesssys/constellation/v2/internal/deploy/user" "github.com/edgelesssys/constellation/v2/internal/logger" "github.com/spf13/afero" + "go.uber.org/zap" ) const debugBanner = ` @@ -56,34 +61,39 @@ func main() { csp := os.Getenv("CONSTEL_CSP") switch platform.FromString(csp) { case platform.AWS: - awsFetcher, err := cloudprovider.NewAWS(ctx) + meta, err := awscloud.New(ctx) if err != nil { - log.Fatalf("%s", err) + log.With(zap.Error(err)).Fatalf("Failed to initialize AWS metadata") } - fetcher = awsFetcher + fetcher = cloudprovider.New(meta) + case platform.Azure: - azureFetcher, err := cloudprovider.NewAzure(ctx) + meta, err := azurecloud.NewMetadata(ctx) if err != nil { - log.Fatalf("%s", err) + log.With(zap.Error(err)).Fatalf("Failed to initialize Azure metadata") } - fetcher = azureFetcher + fetcher = cloudprovider.New(meta) + case platform.GCP: - gcpFetcher, err := cloudprovider.NewGCP(ctx) + meta, err := gcpcloud.New(ctx) if err != nil { - log.Fatalf("%s", err) + log.With(zap.Error(err)).Fatalf("Failed to initialize GCP metadata") } - fetcher = gcpFetcher - log.Infof("Added load balancer IP to local routing table") + defer meta.Close() + fetcher = cloudprovider.New(meta) + case platform.QEMU: - fetcher = cloudprovider.NewQEMU() + fetcher = cloudprovider.New(&qemucloud.Metadata{}) + default: log.Errorf("Unknown / unimplemented cloud provider CONSTEL_CSP=%v. Using fallback", csp) fetcher = fallback.Fetcher{} } + sched := metadata.NewScheduler(log.Named("scheduler"), fetcher, ssh, download) serv := server.New(log.Named("server"), ssh, serviceManager, streamer) if err := deploy.DefaultServiceUnit(ctx, serviceManager); err != nil { - log.Fatalf("%s", err) + log.With(zap.Error(err)).Fatalf("Failed to create default service unit") } writeDebugBanner(log) @@ -101,11 +111,11 @@ func main() { func writeDebugBanner(log *logger.Logger) { tty, err := os.OpenFile("/dev/ttyS0", os.O_WRONLY, os.ModeAppend) if err != nil { - log.Infof("Unable to open /dev/ttyS0 for printing banner: %v", err) + log.With(zap.Error(err)).Errorf("Unable to open /dev/ttyS0 for printing banner") return } defer tty.Close() if _, err := fmt.Fprint(tty, debugBanner); err != nil { - log.Infof("Unable to print to /dev/ttyS0: %v", err) + log.With(zap.Error(err)).Errorf("Unable to print to /dev/ttyS0") } } diff --git a/debugd/internal/debugd/metadata/cloudprovider/cloudprovider.go b/debugd/internal/debugd/metadata/cloudprovider/cloudprovider.go index 4aef1dac1..1f18347fa 100644 --- a/debugd/internal/debugd/metadata/cloudprovider/cloudprovider.go +++ b/debugd/internal/debugd/metadata/cloudprovider/cloudprovider.go @@ -11,11 +11,7 @@ import ( "fmt" "net" - awscloud "github.com/edgelesssys/constellation/v2/internal/cloud/aws" - azurecloud "github.com/edgelesssys/constellation/v2/internal/cloud/azure" - gcpcloud "github.com/edgelesssys/constellation/v2/internal/cloud/gcp" "github.com/edgelesssys/constellation/v2/internal/cloud/metadata" - qemucloud "github.com/edgelesssys/constellation/v2/internal/cloud/qemu" "github.com/edgelesssys/constellation/v2/internal/deploy/ssh" "github.com/edgelesssys/constellation/v2/internal/role" ) @@ -34,46 +30,9 @@ type Fetcher struct { metaAPI providerMetadata } -// NewGCP creates a new GCP fetcher. -func NewGCP(ctx context.Context) (*Fetcher, error) { - gcpClient, err := gcpcloud.NewClient(ctx) - if err != nil { - return nil, err - } - metaAPI := gcpcloud.New(gcpClient) - +func New(cloud providerMetadata) *Fetcher { return &Fetcher{ - metaAPI: metaAPI, - }, nil -} - -// NewAzure creates a new Azure fetcher. -func NewAWS(ctx context.Context) (*Fetcher, error) { - metaAPI, err := awscloud.New(ctx) - if err != nil { - return nil, err - } - - return &Fetcher{ - metaAPI: metaAPI, - }, nil -} - -// NewAzure creates a new Azure fetcher. -func NewAzure(ctx context.Context) (*Fetcher, error) { - metaAPI, err := azurecloud.NewMetadata(ctx) - if err != nil { - return nil, err - } - - return &Fetcher{ - metaAPI: metaAPI, - }, nil -} - -func NewQEMU() *Fetcher { - return &Fetcher{ - metaAPI: &qemucloud.Metadata{}, + metaAPI: cloud, } } diff --git a/debugd/internal/debugd/metadata/cloudprovider/cloudprovider_test.go b/debugd/internal/debugd/metadata/cloudprovider/cloudprovider_test.go index a5f791886..b337b3052 100644 --- a/debugd/internal/debugd/metadata/cloudprovider/cloudprovider_test.go +++ b/debugd/internal/debugd/metadata/cloudprovider/cloudprovider_test.go @@ -226,7 +226,6 @@ type stubMetadata struct { getInstanceErr error getLBEndpointRes string getLBEndpointErr error - supportedRes bool } func (m *stubMetadata) List(ctx context.Context) ([]metadata.InstanceMetadata, error) { @@ -244,7 +243,3 @@ func (m *stubMetadata) GetInstance(ctx context.Context, providerID string) (meta func (m *stubMetadata) GetLoadBalancerEndpoint(ctx context.Context) (string, error) { return m.getLBEndpointRes, m.getLBEndpointErr } - -func (m *stubMetadata) Supported() bool { - return m.supportedRes -} diff --git a/disk-mapper/cmd/main.go b/disk-mapper/cmd/main.go index 1919369ed..e0e32be21 100644 --- a/disk-mapper/cmd/main.go +++ b/disk-mapper/cmd/main.go @@ -101,11 +101,12 @@ func main() { log.With(zap.Error(err)).Fatalf("Unable to resolve GCP state disk path") } issuer = gcp.NewIssuer() - gcpClient, err := gcpcloud.NewClient(context.Background()) + gcpMeta, err := gcpcloud.New(context.Background()) if err != nil { log.With(zap.Error).Fatalf("Failed to create GCP client") } - metadataAPI = gcpcloud.New(gcpClient) + defer gcpMeta.Close() + metadataAPI = gcpMeta case cloudprovider.QEMU: diskPath = qemuStateDiskPath diff --git a/internal/azureshared/doc.go b/internal/azureshared/doc.go deleted file mode 100644 index 650602dfc..000000000 --- a/internal/azureshared/doc.go +++ /dev/null @@ -1,16 +0,0 @@ -/* -Copyright (c) Edgeless Systems GmbH - -SPDX-License-Identifier: AGPL-3.0-only -*/ - -package azureshared - -/* -Package azureshared contains code that is related to Microsoft Azure -and is used by multiple microservices. - -This package is intended to have a minimal size and surface. If you -have Azure related code that is not shared by multiple microservices, -please keep the code in the microservice's internal package. -*/ diff --git a/internal/cloud/aws/ccm.go b/internal/cloud/aws/ccm.go deleted file mode 100644 index fbbb1cf23..000000000 --- a/internal/cloud/aws/ccm.go +++ /dev/null @@ -1,81 +0,0 @@ -/* -Copyright (c) Edgeless Systems GmbH - -SPDX-License-Identifier: AGPL-3.0-only -*/ - -package aws - -import ( - "context" - - "github.com/edgelesssys/constellation/v2/internal/cloud/metadata" - "github.com/edgelesssys/constellation/v2/internal/kubernetes" - "github.com/edgelesssys/constellation/v2/internal/versions" - k8s "k8s.io/api/core/v1" -) - -// TODO: Implement for AWS. - -// CloudControllerManager holds the AWS cloud-controller-manager configuration. -type CloudControllerManager struct{} - -// Image returns the container image used to provide cloud-controller-manager for the cloud-provider. -func (c CloudControllerManager) Image(k8sVersion versions.ValidK8sVersion) (string, error) { - return "", nil -} - -// Path returns the path used by cloud-controller-manager executable within the container image. -func (c CloudControllerManager) Path() string { - return "/aws-cloud-controller-manager" -} - -// Name returns the cloud-provider name as used by k8s cloud-controller-manager (k8s.gcr.io/cloud-controller-manager). -func (c CloudControllerManager) Name() string { - return "aws" -} - -// ExtraArgs returns a list of arguments to append to the cloud-controller-manager command. -func (c CloudControllerManager) ExtraArgs() []string { - return []string{} -} - -// ConfigMaps returns a list of ConfigMaps to deploy together with the k8s cloud-controller-manager -// Reference: https://kubernetes.io/docs/concepts/configuration/configmap/ . -func (c CloudControllerManager) ConfigMaps() (kubernetes.ConfigMaps, error) { - return kubernetes.ConfigMaps{}, nil -} - -// Secrets returns a list of secrets to deploy together with the k8s cloud-controller-manager. -// Reference: https://kubernetes.io/docs/concepts/configuration/secret/ . -func (c CloudControllerManager) Secrets(ctx context.Context, providerID, cloudServiceAccountURI string) (kubernetes.Secrets, error) { - return kubernetes.Secrets{}, nil -} - -// Volumes returns a list of volumes to deploy together with the k8s cloud-controller-manager. -// Reference: https://kubernetes.io/docs/concepts/storage/volumes/ . -func (c CloudControllerManager) Volumes() []k8s.Volume { - return []k8s.Volume{} -} - -// VolumeMounts a list of of volume mounts to deploy together with the k8s cloud-controller-manager. -func (c CloudControllerManager) VolumeMounts() []k8s.VolumeMount { - return []k8s.VolumeMount{} -} - -// Env returns a list of k8s environment key-value pairs to deploy together with the k8s cloud-controller-manager. -func (c CloudControllerManager) Env() []k8s.EnvVar { - return []k8s.EnvVar{} -} - -// PrepareInstance is called on every instance before deploying the cloud-controller-manager. -// Allows for cloud-provider specific hooks. -func (c CloudControllerManager) PrepareInstance(instance metadata.InstanceMetadata, vpnIP string) error { - // no specific hook required. - return nil -} - -// Supported is used to determine if cloud controller manager is implemented for this cloud provider. -func (c CloudControllerManager) Supported() bool { - return false -} diff --git a/internal/cloud/aws/cloudnodemanager.go b/internal/cloud/aws/cloudnodemanager.go deleted file mode 100644 index 1a6ff9210..000000000 --- a/internal/cloud/aws/cloudnodemanager.go +++ /dev/null @@ -1,34 +0,0 @@ -/* -Copyright (c) Edgeless Systems GmbH - -SPDX-License-Identifier: AGPL-3.0-only -*/ - -package aws - -import "github.com/edgelesssys/constellation/v2/internal/versions" - -// TODO: Implement for AWS. - -// CloudNodeManager holds the AWS cloud-node-manager configuration. -type CloudNodeManager struct{} - -// Image returns the container image used to provide cloud-node-manager for the cloud-provider. -func (c *CloudNodeManager) Image(k8sVersion versions.ValidK8sVersion) (string, error) { - return "", nil -} - -// Path returns the path used by cloud-node-manager executable within the container image. -func (c *CloudNodeManager) Path() string { - return "" -} - -// ExtraArgs returns a list of arguments to append to the cloud-node-manager command. -func (c *CloudNodeManager) ExtraArgs() []string { - return []string{} -} - -// Supported is used to determine if cloud node manager is implemented for this cloud provider. -func (c *CloudNodeManager) Supported() bool { - return false -} diff --git a/internal/cloud/aws/metadata.go b/internal/cloud/aws/metadata.go index bdb32275b..da815e01f 100644 --- a/internal/cloud/aws/metadata.go +++ b/internal/cloud/aws/metadata.go @@ -70,11 +70,6 @@ func New(ctx context.Context) (*Metadata, error) { }, nil } -// Supported is used to determine if metadata API is implemented for this cloud provider. -func (m *Metadata) Supported() bool { - return true -} - // List retrieves all instances belonging to the current Constellation. func (m *Metadata) List(ctx context.Context) ([]metadata.InstanceMetadata, error) { uid, err := readInstanceTag(ctx, m.imds, cloud.TagUID) @@ -142,11 +137,6 @@ func (m *Metadata) UID(ctx context.Context) (string, error) { return readInstanceTag(ctx, m.imds, cloud.TagUID) } -// SupportsLoadBalancer returns true if the cloud provider supports load balancers. -func (m *Metadata) SupportsLoadBalancer() bool { - return true -} - // GetLoadBalancerEndpoint returns the endpoint of the load balancer. func (m *Metadata) GetLoadBalancerEndpoint(ctx context.Context) (string, error) { uid, err := readInstanceTag(ctx, m.imds, cloud.TagUID) diff --git a/internal/cloud/azure/ccm.go b/internal/cloud/azure/ccm.go deleted file mode 100644 index 4d99f7598..000000000 --- a/internal/cloud/azure/ccm.go +++ /dev/null @@ -1,188 +0,0 @@ -/* -Copyright (c) Edgeless Systems GmbH - -SPDX-License-Identifier: AGPL-3.0-only -*/ - -package azure - -import ( - "context" - "encoding/json" - - "github.com/edgelesssys/constellation/v2/internal/azureshared" - "github.com/edgelesssys/constellation/v2/internal/kubernetes" - "github.com/edgelesssys/constellation/v2/internal/versions" - k8s "k8s.io/api/core/v1" - meta "k8s.io/apimachinery/pkg/apis/meta/v1" -) - -type ccmMetadata interface { - GetNetworkSecurityGroupName(ctx context.Context) (string, error) - GetLoadBalancerName(ctx context.Context) (string, error) -} - -// CloudControllerManager holds the Azure cloud-controller-manager configuration. -type CloudControllerManager struct { - metadata ccmMetadata -} - -func NewCloudControllerManager(metadata ccmMetadata) *CloudControllerManager { - return &CloudControllerManager{ - metadata: metadata, - } -} - -// Image returns the container image used to provide cloud-controller-manager for the cloud-provider. -func (c *CloudControllerManager) Image(k8sVersion versions.ValidK8sVersion) (string, error) { - return versions.VersionConfigs[k8sVersion].CloudControllerManagerImageAzure, nil -} - -// Path returns the path used by cloud-controller-manager executable within the container image. -func (c *CloudControllerManager) Path() string { - return "cloud-controller-manager" -} - -// Name returns the cloud-provider name as used by k8s cloud-controller-manager (k8s.gcr.io/cloud-controller-manager). -func (c *CloudControllerManager) Name() string { - return "azure" -} - -// ExtraArgs returns a list of arguments to append to the cloud-controller-manager command. -func (c *CloudControllerManager) ExtraArgs() []string { - return []string{ - "--controllers=*,-cloud-node", - "--cloud-config=/etc/azure/azure.json", - "--allocate-node-cidrs=false", - "--configure-cloud-routes=true", - } -} - -// ConfigMaps returns a list of ConfigMaps to deploy together with the k8s cloud-controller-manager -// Reference: https://kubernetes.io/docs/concepts/configuration/configmap/ . -func (c *CloudControllerManager) ConfigMaps() (kubernetes.ConfigMaps, error) { - return kubernetes.ConfigMaps{}, nil -} - -// Secrets returns a list of secrets to deploy together with the k8s cloud-controller-manager. -// Reference: https://kubernetes.io/docs/concepts/configuration/secret/ . -func (c *CloudControllerManager) Secrets(ctx context.Context, providerID string, cloudServiceAccountURI string) (kubernetes.Secrets, error) { - // Azure CCM expects cloud provider config to contain cluster configuration and service principal client secrets - // reference: https://kubernetes-sigs.github.io/cloud-provider-azure/install/configs/ - - subscriptionID, resourceGroup, err := azureshared.BasicsFromProviderID(providerID) - if err != nil { - return kubernetes.Secrets{}, err - } - creds, err := azureshared.ApplicationCredentialsFromURI(cloudServiceAccountURI) - if err != nil { - return kubernetes.Secrets{}, err - } - - vmType := "standard" - if _, _, _, _, err := azureshared.ScaleSetInformationFromProviderID(providerID); err == nil { - vmType = "vmss" - } - - securityGroupName, err := c.metadata.GetNetworkSecurityGroupName(ctx) - if err != nil { - return kubernetes.Secrets{}, err - } - - loadBalancerName, err := c.metadata.GetLoadBalancerName(ctx) - if err != nil { - return kubernetes.Secrets{}, err - } - - config := cloudConfig{ - Cloud: "AzurePublicCloud", - TenantID: creds.TenantID, - SubscriptionID: subscriptionID, - ResourceGroup: resourceGroup, - LoadBalancerSku: "standard", - SecurityGroupName: securityGroupName, - LoadBalancerName: loadBalancerName, - UseInstanceMetadata: true, - VMType: vmType, - Location: creds.Location, - AADClientID: creds.AppClientID, - AADClientSecret: creds.ClientSecretValue, - } - - rawConfig, err := json.Marshal(config) - if err != nil { - return kubernetes.Secrets{}, err - } - - return kubernetes.Secrets{ - &k8s.Secret{ - TypeMeta: meta.TypeMeta{ - Kind: "Secret", - APIVersion: "v1", - }, - ObjectMeta: meta.ObjectMeta{ - Name: "azureconfig", - Namespace: "kube-system", - }, - Data: map[string][]byte{ - "azure.json": rawConfig, - }, - }, - }, nil -} - -// Volumes returns a list of volumes to deploy together with the k8s cloud-controller-manager. -// Reference: https://kubernetes.io/docs/concepts/storage/volumes/ . -func (c *CloudControllerManager) Volumes() []k8s.Volume { - return []k8s.Volume{ - { - Name: "azureconfig", - VolumeSource: k8s.VolumeSource{ - Secret: &k8s.SecretVolumeSource{ - SecretName: "azureconfig", - }, - }, - }, - } -} - -// VolumeMounts a list of of volume mounts to deploy together with the k8s cloud-controller-manager. -func (c *CloudControllerManager) VolumeMounts() []k8s.VolumeMount { - return []k8s.VolumeMount{ - { - Name: "azureconfig", - ReadOnly: true, - MountPath: "/etc/azure", - }, - } -} - -// Env returns a list of k8s environment key-value pairs to deploy together with the k8s cloud-controller-manager. -func (c *CloudControllerManager) Env() []k8s.EnvVar { - return []k8s.EnvVar{} -} - -// Supported is used to determine if cloud controller manager is implemented for this cloud provider. -func (c *CloudControllerManager) Supported() bool { - return true -} - -type cloudConfig struct { - Cloud string `json:"cloud,omitempty"` - TenantID string `json:"tenantId,omitempty"` - SubscriptionID string `json:"subscriptionId,omitempty"` - ResourceGroup string `json:"resourceGroup,omitempty"` - Location string `json:"location,omitempty"` - SubnetName string `json:"subnetName,omitempty"` - SecurityGroupName string `json:"securityGroupName,omitempty"` - SecurityGroupResourceGroup string `json:"securityGroupResourceGroup,omitempty"` - LoadBalancerName string `json:"loadBalancerName,omitempty"` - LoadBalancerSku string `json:"loadBalancerSku,omitempty"` - VNetName string `json:"vnetName,omitempty"` - VNetResourceGroup string `json:"vnetResourceGroup,omitempty"` - CloudProviderBackoff bool `json:"cloudProviderBackoff,omitempty"` - UseInstanceMetadata bool `json:"useInstanceMetadata,omitempty"` - VMType string `json:"vmType,omitempty"` - AADClientID string `json:"aadClientId,omitempty"` - AADClientSecret string `json:"aadClientSecret,omitempty"` -} diff --git a/internal/cloud/azure/ccm_test.go b/internal/cloud/azure/ccm_test.go deleted file mode 100644 index 0e27122df..000000000 --- a/internal/cloud/azure/ccm_test.go +++ /dev/null @@ -1,122 +0,0 @@ -/* -Copyright (c) Edgeless Systems GmbH - -SPDX-License-Identifier: AGPL-3.0-only -*/ - -package azure - -import ( - "context" - "errors" - "testing" - - "github.com/edgelesssys/constellation/v2/internal/kubernetes" - "github.com/edgelesssys/constellation/v2/internal/versions" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - k8s "k8s.io/api/core/v1" - meta "k8s.io/apimachinery/pkg/apis/meta/v1" -) - -func TestSecrets(t *testing.T) { - someErr := errors.New("some error") - testCases := map[string]struct { - providerID string - metadata ccmMetadata - cloudServiceAccountURI string - wantSecrets kubernetes.Secrets - wantErr bool - }{ - "Secrets works for scale sets": { - providerID: "azure:///subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/virtualMachineScaleSets/scale-set-name/virtualMachines/instance-id", - cloudServiceAccountURI: "serviceaccount://azure?tenant_id=tenant-id&client_id=client-id&client_secret=client-secret&location=location", - metadata: &ccmMetadataStub{loadBalancerName: "load-balancer-name", networkSecurityGroupName: "network-security-group-name"}, - wantSecrets: kubernetes.Secrets{ - &k8s.Secret{ - TypeMeta: meta.TypeMeta{ - Kind: "Secret", - APIVersion: "v1", - }, - ObjectMeta: meta.ObjectMeta{ - Name: "azureconfig", - Namespace: "kube-system", - }, - Data: map[string][]byte{ - "azure.json": []byte(`{"cloud":"AzurePublicCloud","tenantId":"tenant-id","subscriptionId":"subscription-id","resourceGroup":"resource-group","location":"location","securityGroupName":"network-security-group-name","loadBalancerName":"load-balancer-name","loadBalancerSku":"standard","useInstanceMetadata":true,"vmType":"vmss","aadClientId":"client-id","aadClientSecret":"client-secret"}`), - }, - }, - }, - }, - "cannot get load balancer Name": { - providerID: "azure:///subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/virtualMachineScaleSets/scale-set-name/virtualMachines/instance-id", - cloudServiceAccountURI: "serviceaccount://azure?tenant_id=tenant-id&client_id=client-id&client_secret=client-secret&location=location", - metadata: &ccmMetadataStub{getLoadBalancerNameErr: someErr}, - wantErr: true, - }, - "cannot get network security group name": { - providerID: "azure:///subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/virtualMachineScaleSets/scale-set-name/virtualMachines/instance-id", - cloudServiceAccountURI: "serviceaccount://azure?tenant_id=tenant-id&client_id=client-id&client_secret=client-secret&location=location", - metadata: &ccmMetadataStub{getNetworkSecurityGroupNameErr: someErr}, - wantErr: true, - }, - "invalid providerID fails": { - providerID: "invalid", - metadata: &ccmMetadataStub{}, - wantErr: true, - }, - "invalid cloudServiceAccountURI fails": { - providerID: "azure:///subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/virtualMachines/instance-name", - metadata: &ccmMetadataStub{}, - cloudServiceAccountURI: "invalid", - wantErr: true, - }, - } - - for name, tc := range testCases { - t.Run(name, func(t *testing.T) { - assert := assert.New(t) - require := require.New(t) - - cloud := NewCloudControllerManager(tc.metadata) - secrets, err := cloud.Secrets(context.Background(), tc.providerID, tc.cloudServiceAccountURI) - if tc.wantErr { - assert.Error(err) - return - } - require.NoError(err) - assert.Equal(tc.wantSecrets, secrets) - }) - } -} - -func TestTrivialCCMFunctions(t *testing.T) { - assert := assert.New(t) - cloud := CloudControllerManager{} - - assert.NotEmpty(cloud.Image(versions.Default)) - assert.NotEmpty(cloud.Path()) - assert.NotEmpty(cloud.Name()) - assert.NotEmpty(cloud.ExtraArgs()) - assert.Empty(cloud.ConfigMaps()) - assert.NotEmpty(cloud.Volumes()) - assert.NotEmpty(cloud.VolumeMounts()) - assert.Empty(cloud.Env()) - assert.True(cloud.Supported()) -} - -type ccmMetadataStub struct { - networkSecurityGroupName string - loadBalancerName string - - getNetworkSecurityGroupNameErr error - getLoadBalancerNameErr error -} - -func (c *ccmMetadataStub) GetNetworkSecurityGroupName(ctx context.Context) (string, error) { - return c.networkSecurityGroupName, c.getNetworkSecurityGroupNameErr -} - -func (c *ccmMetadataStub) GetLoadBalancerName(ctx context.Context) (string, error) { - return c.loadBalancerName, c.getLoadBalancerNameErr -} diff --git a/internal/cloud/azure/cloudnodemanager.go b/internal/cloud/azure/cloudnodemanager.go deleted file mode 100644 index 1da3e7e63..000000000 --- a/internal/cloud/azure/cloudnodemanager.go +++ /dev/null @@ -1,37 +0,0 @@ -/* -Copyright (c) Edgeless Systems GmbH - -SPDX-License-Identifier: AGPL-3.0-only -*/ - -package azure - -import ( - "github.com/edgelesssys/constellation/v2/internal/versions" -) - -// CloudNodeManager holds the Azure cloud-node-manager configuration. -// reference: https://raw.githubusercontent.com/kubernetes-sigs/cloud-provider-azure/master/examples/out-of-tree/cloud-node-manager.yaml . -type CloudNodeManager struct{} - -// Image returns the container image used to provide cloud-node-manager for the cloud-provider. -func (c *CloudNodeManager) Image(k8sVersion versions.ValidK8sVersion) (string, error) { - return versions.VersionConfigs[k8sVersion].CloudNodeManagerImageAzure, nil -} - -// Path returns the path used by cloud-node-manager executable within the container image. -func (c *CloudNodeManager) Path() string { - return "cloud-node-manager" -} - -// ExtraArgs returns a list of arguments to append to the cloud-node-manager command. -func (c *CloudNodeManager) ExtraArgs() []string { - return []string{ - "--wait-routes=true", - } -} - -// Supported is used to determine if cloud node manager is implemented for this cloud provider. -func (c *CloudNodeManager) Supported() bool { - return true -} diff --git a/internal/cloud/azure/cloudnodemanager_test.go b/internal/cloud/azure/cloudnodemanager_test.go deleted file mode 100644 index 5866d16b1..000000000 --- a/internal/cloud/azure/cloudnodemanager_test.go +++ /dev/null @@ -1,24 +0,0 @@ -/* -Copyright (c) Edgeless Systems GmbH - -SPDX-License-Identifier: AGPL-3.0-only -*/ - -package azure - -import ( - "testing" - - "github.com/edgelesssys/constellation/v2/internal/versions" - "github.com/stretchr/testify/assert" -) - -func TestTrivialCNMFunctions(t *testing.T) { - assert := assert.New(t) - cloud := CloudNodeManager{} - - assert.NotEmpty(cloud.Image(versions.Default)) - assert.NotEmpty(cloud.Path()) - assert.NotEmpty(cloud.ExtraArgs()) - assert.True(cloud.Supported()) -} diff --git a/internal/cloud/azure/metadata.go b/internal/cloud/azure/metadata.go index e7c383a7f..348491b49 100644 --- a/internal/cloud/azure/metadata.go +++ b/internal/cloud/azure/metadata.go @@ -8,6 +8,7 @@ package azure import ( "context" + "encoding/json" "fmt" "net/http" "regexp" @@ -18,6 +19,7 @@ import ( "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork" "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources" "github.com/edgelesssys/constellation/v2/internal/cloud" + "github.com/edgelesssys/constellation/v2/internal/cloud/azureshared" "github.com/edgelesssys/constellation/v2/internal/cloud/metadata" ) @@ -201,11 +203,6 @@ func (m *Metadata) getLoadBalancer(ctx context.Context) (*armnetwork.LoadBalance return nil, fmt.Errorf("could not get any load balancer") } -// SupportsLoadBalancer returns true if the cloud provider supports load balancers. -func (m *Metadata) SupportsLoadBalancer() bool { - return true -} - // GetLoadBalancerName returns the load balancer name of the resource group. func (m *Metadata) GetLoadBalancerName(ctx context.Context) (string, error) { lb, err := m.getLoadBalancer(ctx) @@ -265,9 +262,48 @@ func (m *Metadata) GetLoadBalancerEndpoint(ctx context.Context) (string, error) return *resp.Properties.IPAddress, nil } -// Supported is used to determine if metadata API is implemented for this cloud provider. -func (m *Metadata) Supported() bool { - return true +// GetCCMConfig returns the configuration needed for the CCM on Azure. +func (m *Metadata) GetCCMConfig(ctx context.Context, providerID string, cloudServiceAccountURI string) ([]byte, error) { + subscriptionID, resourceGroup, err := azureshared.BasicsFromProviderID(providerID) + if err != nil { + return nil, err + } + creds, err := azureshared.ApplicationCredentialsFromURI(cloudServiceAccountURI) + if err != nil { + return nil, err + } + + vmType := "standard" + if _, _, _, _, err := azureshared.ScaleSetInformationFromProviderID(providerID); err == nil { + vmType = "vmss" + } + + securityGroupName, err := m.GetNetworkSecurityGroupName(ctx) + if err != nil { + return nil, err + } + + loadBalancerName, err := m.GetLoadBalancerName(ctx) + if err != nil { + return nil, err + } + + config := cloudConfig{ + Cloud: "AzurePublicCloud", + TenantID: creds.TenantID, + SubscriptionID: subscriptionID, + ResourceGroup: resourceGroup, + LoadBalancerSku: "standard", + SecurityGroupName: securityGroupName, + LoadBalancerName: loadBalancerName, + UseInstanceMetadata: true, + VMType: vmType, + Location: creds.Location, + AADClientID: creds.AppClientID, + AADClientSecret: creds.ClientSecretValue, + } + + return json.Marshal(config) } // providerID retrieves the current instances providerID. @@ -343,3 +379,23 @@ func extractSSHKeys(sshConfig armcomputev2.SSHConfiguration) map[string][]string } return sshKeys } + +type cloudConfig struct { + Cloud string `json:"cloud,omitempty"` + TenantID string `json:"tenantId,omitempty"` + SubscriptionID string `json:"subscriptionId,omitempty"` + ResourceGroup string `json:"resourceGroup,omitempty"` + Location string `json:"location,omitempty"` + SubnetName string `json:"subnetName,omitempty"` + SecurityGroupName string `json:"securityGroupName,omitempty"` + SecurityGroupResourceGroup string `json:"securityGroupResourceGroup,omitempty"` + LoadBalancerName string `json:"loadBalancerName,omitempty"` + LoadBalancerSku string `json:"loadBalancerSku,omitempty"` + VNetName string `json:"vnetName,omitempty"` + VNetResourceGroup string `json:"vnetResourceGroup,omitempty"` + CloudProviderBackoff bool `json:"cloudProviderBackoff,omitempty"` + UseInstanceMetadata bool `json:"useInstanceMetadata,omitempty"` + VMType string `json:"vmType,omitempty"` + AADClientID string `json:"aadClientId,omitempty"` + AADClientSecret string `json:"aadClientSecret,omitempty"` +} diff --git a/internal/cloud/azure/metadata_test.go b/internal/cloud/azure/metadata_test.go index 0a8b81738..523c17519 100644 --- a/internal/cloud/azure/metadata_test.go +++ b/internal/cloud/azure/metadata_test.go @@ -463,12 +463,6 @@ func TestGetLoadBalancerEndpoint(t *testing.T) { } } -func TestMetadataSupported(t *testing.T) { - assert := assert.New(t) - metadata := Metadata{} - assert.True(metadata.Supported()) -} - func TestProviderID(t *testing.T) { testCases := map[string]struct { imdsAPI imdsAPI diff --git a/internal/cloud/azure/scaleset.go b/internal/cloud/azure/scaleset.go index b9c2a874d..02f1f9786 100644 --- a/internal/cloud/azure/scaleset.go +++ b/internal/cloud/azure/scaleset.go @@ -15,8 +15,8 @@ import ( "github.com/Azure/azure-sdk-for-go/sdk/azcore/to" armcomputev2 "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v2" "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork" - "github.com/edgelesssys/constellation/v2/internal/azureshared" "github.com/edgelesssys/constellation/v2/internal/cloud" + "github.com/edgelesssys/constellation/v2/internal/cloud/azureshared" "github.com/edgelesssys/constellation/v2/internal/cloud/metadata" "github.com/edgelesssys/constellation/v2/internal/role" ) @@ -35,12 +35,8 @@ func (m *Metadata) getScaleSetVM(ctx context.Context, providerID string) (metada if err != nil { return metadata.InstanceMetadata{}, err } - publicIPAddress, err := m.getScaleSetVMPublicIPAddress(ctx, resourceGroup, scaleSet, instanceID, networkInterfaces) - if err != nil { - return metadata.InstanceMetadata{}, err - } - return convertScaleSetVMToCoreInstance(vmResp.VirtualMachineScaleSetVM, networkInterfaces, publicIPAddress) + return convertScaleSetVMToCoreInstance(vmResp.VirtualMachineScaleSetVM, networkInterfaces) } // listScaleSetVMs lists all scale set VMs in the current resource group. @@ -70,7 +66,7 @@ func (m *Metadata) listScaleSetVMs(ctx context.Context, resourceGroup string) ([ if err != nil { return nil, err } - instance, err := convertScaleSetVMToCoreInstance(*vm, interfaces, "") + instance, err := convertScaleSetVMToCoreInstance(*vm, interfaces) if err != nil { return nil, err } @@ -84,7 +80,6 @@ func (m *Metadata) listScaleSetVMs(ctx context.Context, resourceGroup string) ([ // convertScaleSetVMToCoreInstance converts an azure scale set virtual machine with interface configurations into a core.Instance. func convertScaleSetVMToCoreInstance(vm armcomputev2.VirtualMachineScaleSetVM, networkInterfaces []armnetwork.Interface, - publicIPAddress string, ) (metadata.InstanceMetadata, error) { if vm.ID == nil { return metadata.InstanceMetadata{}, errors.New("retrieving instance from armcompute API client returned no instance ID") @@ -108,7 +103,6 @@ func convertScaleSetVMToCoreInstance(vm armcomputev2.VirtualMachineScaleSetVM, n ProviderID: "azure://" + *vm.ID, Role: extractScaleSetVMRole(vm.Tags), VPCIP: extractVPCIP(networkInterfaces), - PublicIP: publicIPAddress, SSHKeys: sshKeys, }, nil } diff --git a/internal/cloud/azure/scaleset_test.go b/internal/cloud/azure/scaleset_test.go index 853d020ee..f6134846e 100644 --- a/internal/cloud/azure/scaleset_test.go +++ b/internal/cloud/azure/scaleset_test.go @@ -155,7 +155,6 @@ func TestConvertScaleSetVMToCoreInstance(t *testing.T) { testCases := map[string]struct { inVM armcomputev2.VirtualMachineScaleSetVM inInterface []armnetwork.Interface - inPublicIP string wantErr bool wantInstance metadata.InstanceMetadata }{ @@ -186,12 +185,10 @@ func TestConvertScaleSetVMToCoreInstance(t *testing.T) { }, }, }, - inPublicIP: "192.0.2.100", wantInstance: metadata.InstanceMetadata{ Name: "scale-set-name-instance-id", ProviderID: "azure:///subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/virtualMachineScaleSets/scale-set-name/virtualMachines/instance-id", VPCIP: "192.0.2.0", - PublicIP: "192.0.2.100", SSHKeys: map[string][]string{}, }, }, @@ -206,7 +203,7 @@ func TestConvertScaleSetVMToCoreInstance(t *testing.T) { assert := assert.New(t) require := require.New(t) - instance, err := convertScaleSetVMToCoreInstance(tc.inVM, tc.inInterface, tc.inPublicIP) + instance, err := convertScaleSetVMToCoreInstance(tc.inVM, tc.inInterface) if tc.wantErr { assert.Error(err) diff --git a/internal/azureshared/appcredentials.go b/internal/cloud/azureshared/appcredentials.go similarity index 100% rename from internal/azureshared/appcredentials.go rename to internal/cloud/azureshared/appcredentials.go diff --git a/internal/azureshared/appcredentials_test.go b/internal/cloud/azureshared/appcredentials_test.go similarity index 100% rename from internal/azureshared/appcredentials_test.go rename to internal/cloud/azureshared/appcredentials_test.go diff --git a/internal/cloud/azureshared/doc.go b/internal/cloud/azureshared/doc.go new file mode 100644 index 000000000..38cf82b27 --- /dev/null +++ b/internal/cloud/azureshared/doc.go @@ -0,0 +1,16 @@ +/* +Copyright (c) Edgeless Systems GmbH + +SPDX-License-Identifier: AGPL-3.0-only +*/ + +/* +Package gcpshared contains code to parse and define data types +relevant for Microsoft Azure. + +This package is intended to have a minimal size and surface. If you +have Azure related code that is not shared by multiple applications, +or if the code interacts with the GCP API, please keep the code in +the application's internal package or add it to the Azure cloud package. +*/ +package azureshared diff --git a/internal/azureshared/metadata.go b/internal/cloud/azureshared/metadata.go similarity index 100% rename from internal/azureshared/metadata.go rename to internal/cloud/azureshared/metadata.go diff --git a/internal/azureshared/metadata_test.go b/internal/cloud/azureshared/metadata_test.go similarity index 100% rename from internal/azureshared/metadata_test.go rename to internal/cloud/azureshared/metadata_test.go diff --git a/internal/cloud/gcp/ccm.go b/internal/cloud/gcp/ccm.go deleted file mode 100644 index a57313edc..000000000 --- a/internal/cloud/gcp/ccm.go +++ /dev/null @@ -1,183 +0,0 @@ -/* -Copyright (c) Edgeless Systems GmbH - -SPDX-License-Identifier: AGPL-3.0-only -*/ - -package gcp - -import ( - "context" - "encoding/json" - "fmt" - "strings" - - "github.com/edgelesssys/constellation/v2/internal/gcpshared" - "github.com/edgelesssys/constellation/v2/internal/kubernetes" - "github.com/edgelesssys/constellation/v2/internal/versions" - k8s "k8s.io/api/core/v1" - v1 "k8s.io/apimachinery/pkg/apis/meta/v1" -) - -// CloudControllerManager holds the gcp cloud-controller-manager configuration. -type CloudControllerManager struct { - uid string - projectID string -} - -// NewCloudControllerManager returns an initialized cloud controller manager configuration struct for GCP. -func NewCloudControllerManager(ctx context.Context, metadata *Metadata) (*CloudControllerManager, error) { - uid, err := metadata.api.UID(ctx) - if err != nil { - return nil, fmt.Errorf("getting uid from metadata: %w", err) - } - projectID, err := metadata.api.RetrieveProjectID() - if err != nil { - return nil, fmt.Errorf("getting project id from metadata: %w", err) - } - - return &CloudControllerManager{ - uid: uid, - projectID: projectID, - }, nil -} - -// Image returns the container image used to provide cloud-controller-manager for the cloud-provider. -func (c *CloudControllerManager) Image(k8sVersion versions.ValidK8sVersion) (string, error) { - return versions.VersionConfigs[k8sVersion].CloudControllerManagerImageGCP, nil -} - -// Path returns the path used by cloud-controller-manager executable within the container image. -func (c *CloudControllerManager) Path() string { - return "/cloud-controller-manager" -} - -// Name returns the cloud-provider name as used by k8s cloud-controller-manager (k8s.gcr.io/cloud-controller-manager). -func (c *CloudControllerManager) Name() string { - return "gce" -} - -// ExtraArgs returns a list of arguments to append to the cloud-controller-manager command. -func (c *CloudControllerManager) ExtraArgs() []string { - return []string{ - "--use-service-account-credentials", - "--controllers=cloud-node,cloud-node-lifecycle,nodeipam,service,route", - "--cloud-config=/etc/gce/gce.conf", - "--cidr-allocator-type=CloudAllocator", - "--allocate-node-cidrs=true", - "--configure-cloud-routes=false", - } -} - -// ConfigMaps returns a list of ConfigMaps to deploy together with the k8s cloud-controller-manager -// Reference: https://kubernetes.io/docs/concepts/configuration/configmap/ . -func (c *CloudControllerManager) ConfigMaps() (kubernetes.ConfigMaps, error) { - // GCP CCM expects cloud config to contain the GCP project-id and other configuration. - // reference: https://github.com/kubernetes/cloud-provider-gcp/blob/master/cluster/gce/gci/configure-helper.sh#L791-L892 - var config strings.Builder - config.WriteString("[global]\n") - config.WriteString(fmt.Sprintf("project-id = %s\n", c.projectID)) - config.WriteString("use-metadata-server = true\n") - config.WriteString(fmt.Sprintf("node-tags = constellation-%s\n", c.uid)) - - return kubernetes.ConfigMaps{ - &k8s.ConfigMap{ - TypeMeta: v1.TypeMeta{ - Kind: "ConfigMap", - APIVersion: "v1", - }, - ObjectMeta: v1.ObjectMeta{ - Name: "gceconf", - Namespace: "kube-system", - }, - Data: map[string]string{ - "gce.conf": config.String(), - }, - }, - }, nil -} - -// Secrets returns a list of secrets to deploy together with the k8s cloud-controller-manager. -// Reference: https://kubernetes.io/docs/concepts/configuration/secret/ . -func (c *CloudControllerManager) Secrets(_ context.Context, _ string, cloudServiceAccountURI string) (kubernetes.Secrets, error) { - serviceAccountKey, err := gcpshared.ServiceAccountKeyFromURI(cloudServiceAccountURI) - if err != nil { - return kubernetes.Secrets{}, err - } - rawKey, err := json.Marshal(serviceAccountKey) - if err != nil { - return kubernetes.Secrets{}, err - } - - return kubernetes.Secrets{ - &k8s.Secret{ - TypeMeta: v1.TypeMeta{ - Kind: "Secret", - APIVersion: "v1", - }, - ObjectMeta: v1.ObjectMeta{ - Name: "gcekey", - Namespace: "kube-system", - }, - Data: map[string][]byte{ - "key.json": rawKey, - }, - }, - }, nil -} - -// Volumes returns a list of volumes to deploy together with the k8s cloud-controller-manager. -// Reference: https://kubernetes.io/docs/concepts/storage/volumes/ . -func (c *CloudControllerManager) Volumes() []k8s.Volume { - return []k8s.Volume{ - { - Name: "gceconf", - VolumeSource: k8s.VolumeSource{ - ConfigMap: &k8s.ConfigMapVolumeSource{ - LocalObjectReference: k8s.LocalObjectReference{ - Name: "gceconf", - }, - }, - }, - }, - { - Name: "gcekey", - VolumeSource: k8s.VolumeSource{ - Secret: &k8s.SecretVolumeSource{ - SecretName: "gcekey", - }, - }, - }, - } -} - -// VolumeMounts returns a list of volume mounts to deploy together with the k8s cloud-controller-manager. -func (c *CloudControllerManager) VolumeMounts() []k8s.VolumeMount { - return []k8s.VolumeMount{ - { - Name: "gceconf", - ReadOnly: true, - MountPath: "/etc/gce", - }, - { - Name: "gcekey", - ReadOnly: true, - MountPath: "/var/secrets/google", - }, - } -} - -// Env returns a list of k8s environment key-value pairs to deploy together with the k8s cloud-controller-manager. -func (c *CloudControllerManager) Env() []k8s.EnvVar { - return []k8s.EnvVar{ - { - Name: "GOOGLE_APPLICATION_CREDENTIALS", - Value: "/var/secrets/google/key.json", - }, - } -} - -// Supported is used to determine if cloud controller manager is implemented for this cloud provider. -func (c *CloudControllerManager) Supported() bool { - return true -} diff --git a/internal/cloud/gcp/ccm_test.go b/internal/cloud/gcp/ccm_test.go deleted file mode 100644 index 7fbb1a1a5..000000000 --- a/internal/cloud/gcp/ccm_test.go +++ /dev/null @@ -1,148 +0,0 @@ -/* -Copyright (c) Edgeless Systems GmbH - -SPDX-License-Identifier: AGPL-3.0-only -*/ - -package gcp - -import ( - "context" - "encoding/json" - "testing" - - "github.com/edgelesssys/constellation/v2/internal/cloud/metadata" - "github.com/edgelesssys/constellation/v2/internal/gcpshared" - "github.com/edgelesssys/constellation/v2/internal/kubernetes" - "github.com/edgelesssys/constellation/v2/internal/versions" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - k8s "k8s.io/api/core/v1" - v1 "k8s.io/apimachinery/pkg/apis/meta/v1" -) - -func TestConfigMaps(t *testing.T) { - testCases := map[string]struct { - instance metadata.InstanceMetadata - wantConfigMaps kubernetes.ConfigMaps - wantErr bool - }{ - "ConfigMaps works": { - wantConfigMaps: kubernetes.ConfigMaps{ - &k8s.ConfigMap{ - TypeMeta: v1.TypeMeta{ - Kind: "ConfigMap", - APIVersion: "v1", - }, - ObjectMeta: v1.ObjectMeta{ - Name: "gceconf", - Namespace: "kube-system", - }, - Data: map[string]string{ - "gce.conf": `[global] -project-id = project-id -use-metadata-server = true -node-tags = constellation-UID -`, - }, - }, - }, - }, - } - - for name, tc := range testCases { - t.Run(name, func(t *testing.T) { - assert := assert.New(t) - require := require.New(t) - - cloud := CloudControllerManager{ - projectID: "project-id", - uid: "UID", - } - configMaps, err := cloud.ConfigMaps() - - if tc.wantErr { - assert.Error(err) - return - } - require.NoError(err) - assert.Equal(tc.wantConfigMaps, configMaps) - }) - } -} - -func TestSecrets(t *testing.T) { - serviceAccountKey := gcpshared.ServiceAccountKey{ - Type: "type", - ProjectID: "project-id", - PrivateKeyID: "private-key-id", - PrivateKey: "private-key", - ClientEmail: "client-email", - ClientID: "client-id", - AuthURI: "auth-uri", - TokenURI: "token-uri", - AuthProviderX509CertURL: "auth-provider-x509-cert-url", - ClientX509CertURL: "client-x509-cert-url", - } - rawKey, err := json.Marshal(serviceAccountKey) - require.NoError(t, err) - testCases := map[string]struct { - instance metadata.InstanceMetadata - cloudServiceAccountURI string - wantSecrets kubernetes.Secrets - wantErr bool - }{ - "Secrets works": { - cloudServiceAccountURI: "serviceaccount://gcp?type=type&project_id=project-id&private_key_id=private-key-id&private_key=private-key&client_email=client-email&client_id=client-id&auth_uri=auth-uri&token_uri=token-uri&auth_provider_x509_cert_url=auth-provider-x509-cert-url&client_x509_cert_url=client-x509-cert-url", - wantSecrets: kubernetes.Secrets{ - &k8s.Secret{ - TypeMeta: v1.TypeMeta{ - Kind: "Secret", - APIVersion: "v1", - }, - ObjectMeta: v1.ObjectMeta{ - Name: "gcekey", - Namespace: "kube-system", - }, - Data: map[string][]byte{ - "key.json": rawKey, - }, - }, - }, - }, - "invalid serviceAccountKey fails": { - cloudServiceAccountURI: "invalid", - wantErr: true, - }, - } - - for name, tc := range testCases { - t.Run(name, func(t *testing.T) { - assert := assert.New(t) - require := require.New(t) - - cloud := CloudControllerManager{} - secrets, err := cloud.Secrets(context.Background(), tc.instance.ProviderID, tc.cloudServiceAccountURI) - if tc.wantErr { - assert.Error(err) - return - } - require.NoError(err) - assert.Equal(tc.wantSecrets, secrets) - }) - } -} - -func TestTrivialCCMFunctions(t *testing.T) { - assert := assert.New(t) - cloud := CloudControllerManager{} - - assert.NotEmpty(cloud.Image(versions.Default)) - assert.NotEmpty(cloud.Path()) - assert.NotEmpty(cloud.Name()) - assert.NotEmpty(cloud.ExtraArgs()) - assert.NotEmpty(cloud.Volumes()) - assert.NotEmpty(cloud.VolumeMounts()) - assert.NotEmpty(cloud.Env()) - assert.True(cloud.Supported()) -} diff --git a/internal/cloud/gcp/client.go b/internal/cloud/gcp/client.go deleted file mode 100644 index 2c25114a5..000000000 --- a/internal/cloud/gcp/client.go +++ /dev/null @@ -1,439 +0,0 @@ -/* -Copyright (c) Edgeless Systems GmbH - -SPDX-License-Identifier: AGPL-3.0-only -*/ - -package gcp - -import ( - "context" - "errors" - "fmt" - "net" - "regexp" - "strings" - - compute "cloud.google.com/go/compute/apiv1" - "github.com/edgelesssys/constellation/v2/internal/cloud" - "github.com/edgelesssys/constellation/v2/internal/cloud/metadata" - "github.com/edgelesssys/constellation/v2/internal/gcpshared" - "github.com/edgelesssys/constellation/v2/internal/role" - "google.golang.org/api/iterator" - computepb "google.golang.org/genproto/googleapis/cloud/compute/v1" - "google.golang.org/protobuf/proto" -) - -const ( - gcpSSHMetadataKey = "ssh-keys" -) - -var zoneFromRegionRegex = regexp.MustCompile("([a-z]*-[a-z]*[0-9])") - -// Client implements the gcp.API interface. -type Client struct { - instanceAPI - subnetworkAPI - metadataAPI - forwardingRulesAPI -} - -// NewClient creates a new Client. -func NewClient(ctx context.Context) (*Client, error) { - insAPI, err := compute.NewInstancesRESTClient(ctx) - if err != nil { - return nil, err - } - subnetAPI, err := compute.NewSubnetworksRESTClient(ctx) - if err != nil { - return nil, err - } - forwardingRulesAPI, err := compute.NewGlobalForwardingRulesRESTClient(ctx) - if err != nil { - return nil, err - } - return &Client{ - instanceAPI: &instanceClient{insAPI}, - subnetworkAPI: &subnetworkClient{subnetAPI}, - forwardingRulesAPI: &forwardingRulesClient{forwardingRulesAPI}, - metadataAPI: &metadataClient{}, - }, nil -} - -// RetrieveInstances returns list of instances including their ips and metadata. -func (c *Client) RetrieveInstances(ctx context.Context, project, zone string) ([]metadata.InstanceMetadata, error) { - uid, err := c.UID(ctx) - if err != nil { - return nil, err - } - req := &computepb.ListInstancesRequest{ - Filter: proto.String(fmt.Sprintf("labels.%s:%s", cloud.TagUID, uid)), - Project: project, - Zone: zone, - } - instanceIterator := c.instanceAPI.List(ctx, req) - - instances := []metadata.InstanceMetadata{} - for { - resp, err := instanceIterator.Next() - if err == iterator.Done { - break - } - if err != nil { - return nil, fmt.Errorf("retrieving instance list from compute API client: %w", err) - } - instance, err := convertToCoreInstance(resp, project, zone) - if err != nil { - return nil, err - } - - instances = append(instances, instance) - } - return instances, nil -} - -// RetrieveInstance returns a an instance including ips and metadata. -func (c *Client) RetrieveInstance(ctx context.Context, project, zone, instanceName string) (metadata.InstanceMetadata, error) { - instance, err := c.getComputeInstance(ctx, project, zone, instanceName) - if err != nil { - return metadata.InstanceMetadata{}, err - } - - return convertToCoreInstance(instance, project, zone) -} - -// RetrieveProjectID retrieves the GCP projectID containing the current instance. -func (c *Client) RetrieveProjectID() (string, error) { - value, err := c.metadataAPI.ProjectID() - if err != nil { - return "", fmt.Errorf("requesting GCP projectID failed %w", err) - } - return value, nil -} - -// RetrieveZone retrieves the GCP zone containing the current instance. -func (c *Client) RetrieveZone() (string, error) { - value, err := c.metadataAPI.Zone() - if err != nil { - return "", fmt.Errorf("requesting GCP zone failed %w", err) - } - return value, nil -} - -func (c *Client) RetrieveInstanceName() (string, error) { - value, err := c.metadataAPI.InstanceName() - if err != nil { - return "", fmt.Errorf("requesting GCP instanceName failed %w", err) - } - return value, nil -} - -func (c *Client) RetrieveInstanceMetadata(attr string) (string, error) { - value, err := c.metadataAPI.InstanceAttributeValue(attr) - if err != nil { - return "", fmt.Errorf("requesting GCP instance metadata: %w", err) - } - return value, nil -} - -// SetInstanceMetadata modifies a key value pair of metadata for the instance specified by project, zone and instanceName. -func (c *Client) SetInstanceMetadata(ctx context.Context, project, zone, instanceName, key, value string) error { - instance, err := c.getComputeInstance(ctx, project, zone, instanceName) - if err != nil { - return fmt.Errorf("retrieving instance metadata: %w", err) - } - if instance == nil || instance.Metadata == nil { - return fmt.Errorf("retrieving instance metadata returned invalid results") - } - - // convert instance metadata to map to handle duplicate keys correctly - metadataMap := extractInstanceMetadata(instance.Metadata, key, false) - metadataMap[key] = value - // convert instance metadata back to flat list - metadata := flattenInstanceMetadata(metadataMap, instance.Metadata.Fingerprint, instance.Metadata.Kind) - - if err := c.updateInstanceMetadata(ctx, project, zone, instanceName, metadata); err != nil { - return fmt.Errorf("setting instance metadata %v: %v: %w", key, value, err) - } - return nil -} - -// UnsetInstanceMetadata modifies a key value pair of metadata for the instance specified by project, zone and instanceName. -func (c *Client) UnsetInstanceMetadata(ctx context.Context, project, zone, instanceName, key string) error { - instance, err := c.getComputeInstance(ctx, project, zone, instanceName) - if err != nil { - return fmt.Errorf("retrieving instance metadata: %w", err) - } - if instance == nil || instance.Metadata == nil { - return fmt.Errorf("retrieving instance metadata returned invalid results") - } - - // convert instance metadata to map to handle duplicate keys correctly - // and skip the key to be removed - metadataMap := extractInstanceMetadata(instance.Metadata, key, true) - - // convert instance metadata back to flat list - metadata := flattenInstanceMetadata(metadataMap, instance.Metadata.Fingerprint, instance.Metadata.Kind) - - if err := c.updateInstanceMetadata(ctx, project, zone, instanceName, metadata); err != nil { - return fmt.Errorf("unsetting instance metadata key %v: %w", key, err) - } - return nil -} - -// RetrieveSubnetworkAliasCIDR returns the alias CIDR of the subnetwork specified by project, zone and subnetworkName. -func (c *Client) RetrieveSubnetworkAliasCIDR(ctx context.Context, project, zone, instanceName string) (string, error) { - instance, err := c.getComputeInstance(ctx, project, zone, instanceName) - if err != nil { - return "", err - } - if instance == nil || instance.NetworkInterfaces == nil || len(instance.NetworkInterfaces) == 0 || instance.NetworkInterfaces[0].Subnetwork == nil { - return "", fmt.Errorf("retrieving instance network interfaces failed") - } - subnetworkURL := *instance.NetworkInterfaces[0].Subnetwork - subnetworkURLFragments := strings.Split(subnetworkURL, "/") - subnetworkName := subnetworkURLFragments[len(subnetworkURLFragments)-1] - - // convert: - // zone --> region - // europe-west3-b --> europe-west3 - region := zoneFromRegionRegex.FindString(zone) - if region == "" { - return "", fmt.Errorf("invalid zone %s", zone) - } - - req := &computepb.GetSubnetworkRequest{ - Project: project, - Region: region, - Subnetwork: subnetworkName, - } - subnetwork, err := c.subnetworkAPI.Get(ctx, req) - if err != nil { - return "", fmt.Errorf("retrieving subnetwork alias CIDR failed: %w", err) - } - if subnetwork == nil || len(subnetwork.SecondaryIpRanges) == 0 || (subnetwork.SecondaryIpRanges[0]).IpCidrRange == nil { - return "", fmt.Errorf("retrieving subnetwork alias CIDR returned invalid results") - } - - return *(subnetwork.SecondaryIpRanges[0]).IpCidrRange, nil -} - -// RetrieveLoadBalancerEndpoint returns the endpoint of the load balancer with the constellation-uid tag. -func (c *Client) RetrieveLoadBalancerEndpoint(ctx context.Context, project string) (string, error) { - uid, err := c.UID(ctx) - if err != nil { - return "", err - } - - req := &computepb.ListGlobalForwardingRulesRequest{ - Project: project, - } - iter := c.forwardingRulesAPI.List(ctx, req) - for { - resp, err := iter.Next() - if err == iterator.Done { - break - } - if err != nil { - return "", fmt.Errorf("retrieving load balancer IP failed: %w", err) - } - if resp.Labels[cloud.TagUID] == uid && resp.Labels["constellation-use"] == "kubernetes" { - if resp.PortRange == nil { - return "", errors.New("load balancer with searched UID has no ports") - } - portRange := strings.Split(*resp.PortRange, "-") - return net.JoinHostPort(*resp.IPAddress, portRange[0]), nil - } - } - - return "", fmt.Errorf("retrieving load balancer IP failed: load balancer not found") -} - -// Close closes the instanceAPI client. -func (c *Client) Close() error { - if err := c.subnetworkAPI.Close(); err != nil { - return err - } - if err := c.forwardingRulesAPI.Close(); err != nil { - return err - } - return c.instanceAPI.Close() -} - -func (c *Client) getComputeInstance(ctx context.Context, project, zone, instanceName string) (*computepb.Instance, error) { - instanceGetReq := &computepb.GetInstanceRequest{ - Project: project, - Zone: zone, - Instance: instanceName, - } - instance, err := c.instanceAPI.Get(ctx, instanceGetReq) - if err != nil { - return nil, fmt.Errorf("retrieving compute instance: %w", err) - } - return instance, nil -} - -// updateInstanceMetadata updates all instance metadata key-value pairs. -func (c *Client) updateInstanceMetadata(ctx context.Context, project, zone, instanceName string, metadata *computepb.Metadata) error { - setMetadataReq := &computepb.SetMetadataInstanceRequest{ - Project: project, - Zone: zone, - Instance: instanceName, - MetadataResource: metadata, - } - - if _, err := c.instanceAPI.SetMetadata(ctx, setMetadataReq); err != nil { - return fmt.Errorf("updating instance metadata: %w", err) - } - return nil -} - -// UID retrieves the current instances uid. -func (c *Client) UID(ctx context.Context) (string, error) { - // API endpoint: http://metadata.google.internal/computeMetadata/v1/instance/attributes/constellation-uid - instanceID, err := c.InstanceID() - if err != nil { - return "", fmt.Errorf("retrieving instance ID: %w", err) - } - project, err := c.ProjectID() - if err != nil { - return "", fmt.Errorf("retrieving project ID: %w", err) - } - zone, err := c.Zone() - if err != nil { - return "", fmt.Errorf("retrieving zone: %w", err) - } - - instance, err := c.instanceAPI.Get(ctx, &computepb.GetInstanceRequest{ - Project: project, - Zone: zone, - Instance: instanceID, - }) - if err != nil { - return "", fmt.Errorf("retrieving instance labels: %w", err) - } - return instance.Labels[cloud.TagUID], nil -} - -// extractVPCIP extracts the primary private IP from a list of interfaces. -func extractVPCIP(interfaces []*computepb.NetworkInterface) string { - for _, interf := range interfaces { - if interf == nil || interf.NetworkIP == nil || interf.Name == nil || *interf.Name != "nic0" { - continue - } - // return private IP from the default interface - return *interf.NetworkIP - } - return "" -} - -// extractPublicIP extracts a public IP from a list of interfaces. -func extractPublicIP(interfaces []*computepb.NetworkInterface) string { - for _, interf := range interfaces { - if interf == nil || interf.AccessConfigs == nil || interf.Name == nil || *interf.Name != "nic0" { - continue - } - - // return public IP from the default interface - // GCP only supports one type of access config, so returning the first IP should result in a valid public IP - for _, accessConfig := range interf.AccessConfigs { - if accessConfig == nil || accessConfig.NatIP == nil { - continue - } - return *accessConfig.NatIP - } - } - return "" -} - -// extractAliasIPRanges extracts alias interface IPs from a list of interfaces. -func extractAliasIPRanges(interfaces []*computepb.NetworkInterface) []string { - ips := []string{} - for _, interf := range interfaces { - if interf == nil || interf.AliasIpRanges == nil { - continue - } - for _, aliasIP := range interf.AliasIpRanges { - if aliasIP == nil || aliasIP.IpCidrRange == nil { - continue - } - ips = append(ips, *aliasIP.IpCidrRange) - } - } - return ips -} - -// extractSSHKeys extracts SSH keys from GCP instance metadata. -// reference: https://cloud.google.com/compute/docs/connect/add-ssh-keys . -func extractSSHKeys(metadata map[string]string) map[string][]string { - sshKeysRaw, ok := metadata[gcpSSHMetadataKey] - if !ok { - // ignore missing metadata entry - return map[string][]string{} - } - - sshKeyLines := strings.Split(sshKeysRaw, "\n") - keys := map[string][]string{} - for _, sshKeyRaw := range sshKeyLines { - keyParts := strings.SplitN(sshKeyRaw, ":", 2) - if len(keyParts) != 2 { - continue - } - username := keyParts[0] - keyParts = strings.SplitN(keyParts[1], " ", 3) - if len(keyParts) < 2 { - continue - } - keyValue := fmt.Sprintf("%s %s", keyParts[0], keyParts[1]) - keys[username] = append(keys[username], keyValue) - } - return keys -} - -// convertToCoreInstance converts a *computepb.Instance to a core.Instance. -func convertToCoreInstance(in *computepb.Instance, project string, zone string) (metadata.InstanceMetadata, error) { - if in.Name == nil { - return metadata.InstanceMetadata{}, fmt.Errorf("retrieving instance from compute API client returned invalid instance Name: %v", in.Name) - } - mdata := extractInstanceMetadata(in.Metadata, "", false) - return metadata.InstanceMetadata{ - Name: *in.Name, - ProviderID: gcpshared.JoinProviderID(project, zone, *in.Name), - Role: role.FromString(in.Labels[cloud.TagRole]), - VPCIP: extractVPCIP(in.NetworkInterfaces), - PublicIP: extractPublicIP(in.NetworkInterfaces), - AliasIPRanges: extractAliasIPRanges(in.NetworkInterfaces), - SSHKeys: extractSSHKeys(mdata), - }, nil -} - -// extractInstanceMetadata will extract the list of instance metadata key-value pairs into a map. -// If "skipKey" is true, "key" will be skipped. -func extractInstanceMetadata(in *computepb.Metadata, key string, skipKey bool) map[string]string { - metadataMap := map[string]string{} - for _, item := range in.Items { - if item == nil || item.Key == nil || item.Value == nil { - continue - } - if skipKey && *item.Key == key { - continue - } - metadataMap[*item.Key] = *item.Value - } - return metadataMap -} - -// flattenInstanceMetadata takes a map of metadata key-value pairs and returns a flat list of computepb.Items inside computepb.Metadata. -func flattenInstanceMetadata(metadataMap map[string]string, fingerprint, kind *string) *computepb.Metadata { - metadata := &computepb.Metadata{ - Fingerprint: fingerprint, - Kind: kind, - Items: make([]*computepb.Items, len(metadataMap)), - } - i := 0 - for mapKey, mapValue := range metadataMap { - metadata.Items[i] = &computepb.Items{Key: proto.String(mapKey), Value: proto.String(mapValue)} - i++ - } - return metadata -} diff --git a/internal/cloud/gcp/client_test.go b/internal/cloud/gcp/client_test.go deleted file mode 100644 index cddfeb1ec..000000000 --- a/internal/cloud/gcp/client_test.go +++ /dev/null @@ -1,1106 +0,0 @@ -/* -Copyright (c) Edgeless Systems GmbH - -SPDX-License-Identifier: AGPL-3.0-only -*/ - -package gcp - -import ( - "context" - "errors" - "testing" - - compute "cloud.google.com/go/compute/apiv1" - "github.com/edgelesssys/constellation/v2/internal/cloud" - "github.com/edgelesssys/constellation/v2/internal/cloud/metadata" - "github.com/edgelesssys/constellation/v2/internal/role" - gax "github.com/googleapis/gax-go/v2" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "go.uber.org/goleak" - "google.golang.org/api/iterator" - computepb "google.golang.org/genproto/googleapis/cloud/compute/v1" - "google.golang.org/protobuf/proto" -) - -func TestMain(m *testing.M) { - goleak.VerifyTestMain(m, - // https://github.com/census-instrumentation/opencensus-go/issues/1262 - goleak.IgnoreTopFunction("go.opencensus.io/stats/view.(*worker).start"), - ) -} - -func TestRetrieveInstances(t *testing.T) { - uid := "1234" - someErr := errors.New("failed") - newTestIter := func() *stubInstanceIterator { - return &stubInstanceIterator{ - instances: []*computepb.Instance{ - { - Name: proto.String("someInstance"), - Labels: map[string]string{ - cloud.TagRole: role.ControlPlane.String(), - cloud.TagUID: uid, - }, - Metadata: &computepb.Metadata{ - Items: []*computepb.Items{ - { - Key: proto.String("ssh-keys"), - Value: proto.String("bob:ssh-rsa bobskey"), - }, - { - Key: proto.String("key-2"), - Value: proto.String("value-2"), - }, - }, - }, - NetworkInterfaces: []*computepb.NetworkInterface{ - { - Name: proto.String("nic0"), - NetworkIP: proto.String("192.0.2.0"), - AliasIpRanges: []*computepb.AliasIpRange{{IpCidrRange: proto.String("192.0.2.0/16")}}, - AccessConfigs: []*computepb.AccessConfig{{NatIP: proto.String("192.0.2.1")}}, - }, - }, - }, - }, - } - } - instance := &computepb.Instance{ - Name: proto.String("instance"), - Labels: map[string]string{ - cloud.TagRole: role.ControlPlane.String(), - cloud.TagUID: uid, - }, - } - - testCases := map[string]struct { - client stubInstancesClient - metadata stubMetadataClient - instanceIter *stubInstanceIterator - instanceIterMutator func(*stubInstanceIterator) - wantInstances []metadata.InstanceMetadata - wantErr bool - }{ - "retrieve works": { - client: stubInstancesClient{GetInstance: instance}, - metadata: stubMetadataClient{InstanceValue: uid}, - instanceIter: newTestIter(), - wantInstances: []metadata.InstanceMetadata{ - { - Name: "someInstance", - ProviderID: "gce://someProject/someZone/someInstance", - Role: role.ControlPlane, - AliasIPRanges: []string{"192.0.2.0/16"}, - PublicIP: "192.0.2.1", - VPCIP: "192.0.2.0", - SSHKeys: map[string][]string{"bob": {"ssh-rsa bobskey"}}, - }, - }, - }, - "instance name is null": { - client: stubInstancesClient{GetInstance: instance}, - metadata: stubMetadataClient{InstanceValue: uid}, - instanceIter: newTestIter(), - instanceIterMutator: func(sii *stubInstanceIterator) { sii.instances[0].Name = nil }, - wantErr: true, - }, - "no instance with network ip": { - client: stubInstancesClient{GetInstance: instance}, - metadata: stubMetadataClient{InstanceValue: uid}, - instanceIter: newTestIter(), - instanceIterMutator: func(sii *stubInstanceIterator) { sii.instances[0].NetworkInterfaces = nil }, - wantInstances: []metadata.InstanceMetadata{ - { - Name: "someInstance", - ProviderID: "gce://someProject/someZone/someInstance", - Role: role.ControlPlane, - AliasIPRanges: []string{}, - PublicIP: "", - VPCIP: "", - SSHKeys: map[string][]string{"bob": {"ssh-rsa bobskey"}}, - }, - }, - }, - "network ip is nil": { - client: stubInstancesClient{GetInstance: instance}, - metadata: stubMetadataClient{InstanceValue: uid}, - instanceIter: newTestIter(), - instanceIterMutator: func(sii *stubInstanceIterator) { sii.instances[0].NetworkInterfaces[0].NetworkIP = nil }, - wantInstances: []metadata.InstanceMetadata{ - { - Name: "someInstance", - ProviderID: "gce://someProject/someZone/someInstance", - Role: role.ControlPlane, - AliasIPRanges: []string{"192.0.2.0/16"}, - PublicIP: "192.0.2.1", - VPCIP: "", - SSHKeys: map[string][]string{"bob": {"ssh-rsa bobskey"}}, - }, - }, - }, - "constellation retrieval fails": { - client: stubInstancesClient{GetInstance: instance}, - metadata: stubMetadataClient{instanceIDErr: someErr}, - instanceIter: newTestIter(), - wantErr: true, - }, - "role is not set": { - client: stubInstancesClient{GetInstance: instance}, - metadata: stubMetadataClient{InstanceValue: uid}, - instanceIter: newTestIter(), - instanceIterMutator: func(sii *stubInstanceIterator) { delete(sii.instances[0].Labels, cloud.TagRole) }, - wantInstances: []metadata.InstanceMetadata{ - { - Name: "someInstance", - ProviderID: "gce://someProject/someZone/someInstance", - Role: role.Unknown, - AliasIPRanges: []string{"192.0.2.0/16"}, - PublicIP: "192.0.2.1", - VPCIP: "192.0.2.0", - SSHKeys: map[string][]string{"bob": {"ssh-rsa bobskey"}}, - }, - }, - }, - "instance iterator Next() errors": { - client: stubInstancesClient{GetInstance: instance}, - metadata: stubMetadataClient{InstanceValue: uid}, - instanceIter: &stubInstanceIterator{nextErr: someErr}, - wantErr: true, - }, - } - - for name, tc := range testCases { - t.Run(name, func(t *testing.T) { - assert := assert.New(t) - require := require.New(t) - - if tc.instanceIterMutator != nil { - tc.instanceIterMutator(tc.instanceIter) - } - tc.client.ListInstanceIterator = tc.instanceIter - client := Client{ - instanceAPI: tc.client, - metadataAPI: tc.metadata, - } - - instances, err := client.RetrieveInstances(context.Background(), "someProject", "someZone") - - if tc.wantErr { - assert.Error(err) - return - } - require.NoError(err) - assert.Equal(tc.wantInstances, instances) - }) - } -} - -func TestRetrieveInstance(t *testing.T) { - newTestInstance := func() *computepb.Instance { - return &computepb.Instance{ - Name: proto.String("someInstance"), - Labels: map[string]string{}, - Metadata: &computepb.Metadata{ - Items: []*computepb.Items{ - { - Key: proto.String("key-1"), - Value: proto.String("value-1"), - }, - { - Key: proto.String("key-2"), - Value: proto.String("value-2"), - }, - }, - }, - NetworkInterfaces: []*computepb.NetworkInterface{ - { - Name: proto.String("nic0"), - NetworkIP: proto.String("192.0.2.0"), - AliasIpRanges: []*computepb.AliasIpRange{{IpCidrRange: proto.String("192.0.2.0/16")}}, - AccessConfigs: []*computepb.AccessConfig{{NatIP: proto.String("192.0.2.1")}}, - }, - }, - } - } - - testCases := map[string]struct { - client stubInstancesClient - clientInstance *computepb.Instance - clientInstanceMutator func(*computepb.Instance) - wantInstance metadata.InstanceMetadata - wantErr bool - }{ - "retrieve works": { - client: stubInstancesClient{}, - clientInstance: newTestInstance(), - wantInstance: metadata.InstanceMetadata{ - Name: "someInstance", - ProviderID: "gce://someProject/someZone/someInstance", - AliasIPRanges: []string{"192.0.2.0/16"}, - PublicIP: "192.0.2.1", - VPCIP: "192.0.2.0", - SSHKeys: map[string][]string{}, - }, - }, - "retrieve with SSH key works": { - client: stubInstancesClient{}, - clientInstance: newTestInstance(), - clientInstanceMutator: func(i *computepb.Instance) { - i.Metadata.Items[0].Key = proto.String("ssh-keys") - i.Metadata.Items[0].Value = proto.String("bob:ssh-rsa bobskey") - }, - wantInstance: metadata.InstanceMetadata{ - Name: "someInstance", - ProviderID: "gce://someProject/someZone/someInstance", - AliasIPRanges: []string{"192.0.2.0/16"}, - PublicIP: "192.0.2.1", - VPCIP: "192.0.2.0", - SSHKeys: map[string][]string{"bob": {"ssh-rsa bobskey"}}, - }, - }, - "retrieve with Role works": { - client: stubInstancesClient{}, - clientInstance: newTestInstance(), - clientInstanceMutator: func(i *computepb.Instance) { - i.Labels[cloud.TagRole] = role.ControlPlane.String() - }, - wantInstance: metadata.InstanceMetadata{ - Name: "someInstance", - ProviderID: "gce://someProject/someZone/someInstance", - AliasIPRanges: []string{"192.0.2.0/16"}, - PublicIP: "192.0.2.1", - Role: role.ControlPlane, - VPCIP: "192.0.2.0", - SSHKeys: map[string][]string{}, - }, - }, - "retrieve fails": { - client: stubInstancesClient{ - GetErr: errors.New("retrieve error"), - }, - clientInstance: nil, - wantErr: true, - }, - "metadata item is null": { - client: stubInstancesClient{}, - clientInstance: newTestInstance(), - clientInstanceMutator: func(i *computepb.Instance) { i.Metadata.Items[0] = nil }, - wantInstance: metadata.InstanceMetadata{ - Name: "someInstance", - ProviderID: "gce://someProject/someZone/someInstance", - AliasIPRanges: []string{"192.0.2.0/16"}, - PublicIP: "192.0.2.1", - VPCIP: "192.0.2.0", - SSHKeys: map[string][]string{}, - }, - }, - "metadata key is null": { - client: stubInstancesClient{}, - clientInstance: newTestInstance(), - clientInstanceMutator: func(i *computepb.Instance) { i.Metadata.Items[0].Key = nil }, - wantInstance: metadata.InstanceMetadata{ - Name: "someInstance", - ProviderID: "gce://someProject/someZone/someInstance", - AliasIPRanges: []string{"192.0.2.0/16"}, - PublicIP: "192.0.2.1", - VPCIP: "192.0.2.0", - SSHKeys: map[string][]string{}, - }, - }, - "metadata value is null": { - client: stubInstancesClient{}, - clientInstance: newTestInstance(), - clientInstanceMutator: func(i *computepb.Instance) { i.Metadata.Items[0].Value = nil }, - wantInstance: metadata.InstanceMetadata{ - Name: "someInstance", - ProviderID: "gce://someProject/someZone/someInstance", - AliasIPRanges: []string{"192.0.2.0/16"}, - PublicIP: "192.0.2.1", - VPCIP: "192.0.2.0", - SSHKeys: map[string][]string{}, - }, - }, - "instance without network ip": { - client: stubInstancesClient{}, - clientInstance: newTestInstance(), - clientInstanceMutator: func(i *computepb.Instance) { i.NetworkInterfaces[0] = nil }, - wantInstance: metadata.InstanceMetadata{ - Name: "someInstance", - ProviderID: "gce://someProject/someZone/someInstance", - AliasIPRanges: []string{}, - PublicIP: "", - VPCIP: "", - SSHKeys: map[string][]string{}, - }, - }, - "network ip is nil": { - client: stubInstancesClient{}, - clientInstance: newTestInstance(), - clientInstanceMutator: func(i *computepb.Instance) { i.NetworkInterfaces[0].NetworkIP = nil }, - wantInstance: metadata.InstanceMetadata{ - Name: "someInstance", - ProviderID: "gce://someProject/someZone/someInstance", - AliasIPRanges: []string{"192.0.2.0/16"}, - PublicIP: "192.0.2.1", - VPCIP: "", - SSHKeys: map[string][]string{}, - }, - }, - "network alias cidr is nil": { - client: stubInstancesClient{}, - clientInstance: newTestInstance(), - clientInstanceMutator: func(i *computepb.Instance) { i.NetworkInterfaces[0].AliasIpRanges[0].IpCidrRange = nil }, - wantInstance: metadata.InstanceMetadata{ - Name: "someInstance", - ProviderID: "gce://someProject/someZone/someInstance", - AliasIPRanges: []string{}, - PublicIP: "192.0.2.1", - VPCIP: "192.0.2.0", - SSHKeys: map[string][]string{}, - }, - }, - "network public ip is nil": { - client: stubInstancesClient{}, - clientInstance: newTestInstance(), - clientInstanceMutator: func(i *computepb.Instance) { i.NetworkInterfaces[0].AccessConfigs[0].NatIP = nil }, - wantInstance: metadata.InstanceMetadata{ - Name: "someInstance", - ProviderID: "gce://someProject/someZone/someInstance", - AliasIPRanges: []string{"192.0.2.0/16"}, - PublicIP: "", - VPCIP: "192.0.2.0", - SSHKeys: map[string][]string{}, - }, - }, - } - - for name, tc := range testCases { - t.Run(name, func(t *testing.T) { - assert := assert.New(t) - require := require.New(t) - - if tc.clientInstanceMutator != nil { - tc.clientInstanceMutator(tc.clientInstance) - } - tc.client.GetInstance = tc.clientInstance - client := Client{instanceAPI: tc.client} - - instance, err := client.RetrieveInstance(context.Background(), "someProject", "someZone", "someInstance") - - if tc.wantErr { - assert.Error(err) - return - } - require.NoError(err) - assert.Equal(tc.wantInstance, instance) - }) - } -} - -func TestRetrieveProjectID(t *testing.T) { - someErr := errors.New("failed") - - testCases := map[string]struct { - client stubMetadataClient - wantValue string - wantErr bool - }{ - "retrieve works": { - client: stubMetadataClient{ProjectIDValue: "someProjectID"}, - wantValue: "someProjectID", - }, - "retrieve fails": { - client: stubMetadataClient{ProjectIDErr: someErr}, - wantErr: true, - }, - } - - for name, tc := range testCases { - t.Run(name, func(t *testing.T) { - assert := assert.New(t) - require := require.New(t) - - client := Client{metadataAPI: tc.client} - value, err := client.RetrieveProjectID() - - if tc.wantErr { - assert.Error(err) - return - } - require.NoError(err) - assert.Equal(tc.wantValue, value) - }) - } -} - -func TestRetrieveZone(t *testing.T) { - someErr := errors.New("failed") - - testCases := map[string]struct { - client stubMetadataClient - wantValue string - wantErr bool - }{ - "retrieve works": { - client: stubMetadataClient{ZoneValue: "someZone"}, - wantValue: "someZone", - }, - "retrieve fails": { - client: stubMetadataClient{ZoneErr: someErr}, - wantErr: true, - }, - } - - for name, tc := range testCases { - t.Run(name, func(t *testing.T) { - assert := assert.New(t) - require := require.New(t) - - client := Client{metadataAPI: tc.client} - value, err := client.RetrieveZone() - - if tc.wantErr { - assert.Error(err) - return - } - require.NoError(err) - assert.Equal(tc.wantValue, value) - }) - } -} - -func TestRetrieveInstanceName(t *testing.T) { - someErr := errors.New("failed") - - testCases := map[string]struct { - client stubMetadataClient - wantValue string - wantErr bool - }{ - "retrieve works": { - client: stubMetadataClient{InstanceNameValue: "someInstanceName"}, - wantValue: "someInstanceName", - }, - "retrieve fails": { - client: stubMetadataClient{InstanceNameErr: someErr}, - wantErr: true, - }, - } - - for name, tc := range testCases { - t.Run(name, func(t *testing.T) { - assert := assert.New(t) - require := require.New(t) - - client := Client{metadataAPI: tc.client} - value, err := client.RetrieveInstanceName() - - if tc.wantErr { - assert.Error(err) - return - } - require.NoError(err) - assert.Equal(tc.wantValue, value) - }) - } -} - -func TestRetrieveInstanceMetadata(t *testing.T) { - someErr := errors.New("failed") - attr := "someAttribute" - - testCases := map[string]struct { - client stubMetadataClient - attr string - wantValue string - wantErr bool - }{ - "retrieve works": { - client: stubMetadataClient{ - InstanceValue: "someValue", - InstanceErr: nil, - }, - wantValue: "someValue", - }, - "retrieve fails": { - client: stubMetadataClient{ - InstanceValue: "", - InstanceErr: someErr, - }, - wantErr: true, - }, - } - - for name, tc := range testCases { - t.Run(name, func(t *testing.T) { - assert := assert.New(t) - require := require.New(t) - - client := Client{metadataAPI: tc.client} - value, err := client.RetrieveInstanceMetadata(attr) - - if tc.wantErr { - assert.Error(err) - return - } - require.NoError(err) - assert.Equal(tc.wantValue, value) - }) - } -} - -func TestSetInstanceMetadata(t *testing.T) { - someErr := errors.New("failed") - - testCases := map[string]struct { - client stubInstancesClient - wantErr bool - }{ - "set works": { - client: stubInstancesClient{ - GetInstance: &computepb.Instance{ - Metadata: &computepb.Metadata{ - Fingerprint: proto.String("someFingerprint"), - Kind: proto.String("compute#metadata"), - Items: []*computepb.Items{}, - }, - }, - }, - }, - "retrieve fails": { - client: stubInstancesClient{ - GetErr: someErr, - }, - wantErr: true, - }, - "retrieve returns nil": { - wantErr: true, - }, - "setting fails": { - client: stubInstancesClient{ - GetInstance: &computepb.Instance{ - Metadata: &computepb.Metadata{ - Fingerprint: proto.String("someFingerprint"), - Kind: proto.String("compute#metadata"), - Items: []*computepb.Items{}, - }, - }, - SetMetadataErr: someErr, - }, - wantErr: true, - }, - } - - for name, tc := range testCases { - t.Run(name, func(t *testing.T) { - assert := assert.New(t) - require := require.New(t) - - client := Client{instanceAPI: tc.client} - err := client.SetInstanceMetadata(context.Background(), "project", "zone", "instanceName", "key", "value") - - if tc.wantErr { - assert.Error(err) - return - } - require.NoError(err) - }) - } -} - -func TestUnsetInstanceMetadata(t *testing.T) { - someErr := errors.New("failed") - - testCases := map[string]struct { - client stubInstancesClient - wantErr bool - }{ - "unset works": { - client: stubInstancesClient{ - GetInstance: &computepb.Instance{ - Metadata: &computepb.Metadata{ - Fingerprint: proto.String("someFingerprint"), - Kind: proto.String("compute#metadata"), - Items: []*computepb.Items{}, - }, - }, - }, - }, - "unset with existing key works": { - client: stubInstancesClient{ - GetInstance: &computepb.Instance{ - Metadata: &computepb.Metadata{ - Fingerprint: proto.String("someFingerprint"), - Kind: proto.String("compute#metadata"), - Items: []*computepb.Items{ - { - Key: proto.String("key"), - Value: proto.String("value"), - }, - }, - }, - }, - }, - }, - "retrieve fails": { - client: stubInstancesClient{GetErr: someErr}, - wantErr: true, - }, - "retrieve returns nil": { - wantErr: true, - }, - "setting fails": { - client: stubInstancesClient{ - GetInstance: &computepb.Instance{ - Metadata: &computepb.Metadata{ - Fingerprint: proto.String("someFingerprint"), - Kind: proto.String("compute#metadata"), - Items: []*computepb.Items{}, - }, - }, - SetMetadataErr: someErr, - }, - wantErr: true, - }, - } - - for name, tc := range testCases { - t.Run(name, func(t *testing.T) { - assert := assert.New(t) - require := require.New(t) - - client := Client{instanceAPI: tc.client} - err := client.UnsetInstanceMetadata(context.Background(), "project", "zone", "instanceName", "key") - - if tc.wantErr { - assert.Error(err) - return - } - require.NoError(err) - }) - } -} - -func TestRetrieveSubnetworkAliasCIDR(t *testing.T) { - aliasCIDR := "192.0.2.1/24" - someErr := errors.New("some error") - testCases := map[string]struct { - stubInstancesClient stubInstancesClient - stubSubnetworksClient stubSubnetworksClient - wantAliasCIDR string - wantErr bool - }{ - "RetrieveSubnetworkAliasCIDR works": { - stubInstancesClient: stubInstancesClient{ - GetInstance: &computepb.Instance{ - NetworkInterfaces: []*computepb.NetworkInterface{ - { - Subnetwork: proto.String("projects/project/regions/region/subnetworks/subnetwork"), - }, - }, - }, - }, - stubSubnetworksClient: stubSubnetworksClient{ - GetSubnetwork: &computepb.Subnetwork{ - SecondaryIpRanges: []*computepb.SubnetworkSecondaryRange{ - { - IpCidrRange: proto.String(aliasCIDR), - }, - }, - }, - }, - wantAliasCIDR: aliasCIDR, - }, - "instance has no network interface": { - stubInstancesClient: stubInstancesClient{ - GetInstance: &computepb.Instance{ - NetworkInterfaces: []*computepb.NetworkInterface{}, - }, - }, - wantErr: true, - }, - "cannot get instance": { - stubInstancesClient: stubInstancesClient{ - GetErr: someErr, - }, - wantErr: true, - }, - "cannot get subnetwork": { - stubInstancesClient: stubInstancesClient{ - GetInstance: &computepb.Instance{ - NetworkInterfaces: []*computepb.NetworkInterface{ - { - Subnetwork: proto.String("projects/project/regions/region/subnetworks/subnetwork"), - }, - }, - }, - }, - stubSubnetworksClient: stubSubnetworksClient{ - GetErr: someErr, - }, - wantErr: true, - }, - "subnetwork has no cidr range": { - stubInstancesClient: stubInstancesClient{ - GetInstance: &computepb.Instance{ - NetworkInterfaces: []*computepb.NetworkInterface{ - { - Subnetwork: proto.String("projects/project/regions/region/subnetworks/subnetwork"), - }, - }, - }, - }, - stubSubnetworksClient: stubSubnetworksClient{ - GetSubnetwork: &computepb.Subnetwork{}, - }, - wantErr: true, - }, - } - for name, tc := range testCases { - t.Run(name, func(t *testing.T) { - assert := assert.New(t) - require := require.New(t) - - client := Client{instanceAPI: tc.stubInstancesClient, subnetworkAPI: tc.stubSubnetworksClient} - aliasCIDR, err := client.RetrieveSubnetworkAliasCIDR(context.Background(), "project", "us-central1-a", "subnetwork") - - if tc.wantErr { - assert.Error(err) - return - } - require.NoError(err) - assert.Equal(tc.wantAliasCIDR, aliasCIDR) - }) - } -} - -func TestRetrieveLoadBalancerEndpoint(t *testing.T) { - loadBalancerIP := "192.0.2.1" - uid := "uid" - use := "kubernetes" - someErr := errors.New("some error") - instance := &computepb.Instance{ - Labels: map[string]string{ - cloud.TagUID: uid, - }, - } - - testCases := map[string]struct { - instanceAPI stubInstancesClient - stubForwardingRulesClient stubForwardingRulesClient - stubMetadataClient stubMetadataClient - wantLoadBalancerIP string - wantErr bool - }{ - "works": { - instanceAPI: stubInstancesClient{GetInstance: instance}, - stubMetadataClient: stubMetadataClient{}, - stubForwardingRulesClient: stubForwardingRulesClient{ - ForwardingRuleIterator: &stubForwardingRuleIterator{ - rules: []*computepb.ForwardingRule{ - { - IPAddress: proto.String(loadBalancerIP), - PortRange: proto.String("100-100"), - Labels: map[string]string{cloud.TagUID: uid, "constellation-use": use}, - }, - }, - }, - }, - wantLoadBalancerIP: loadBalancerIP, - }, - "fails when no matching load balancers exists": { - instanceAPI: stubInstancesClient{GetInstance: instance}, - stubMetadataClient: stubMetadataClient{}, - stubForwardingRulesClient: stubForwardingRulesClient{ - ForwardingRuleIterator: &stubForwardingRuleIterator{ - rules: []*computepb.ForwardingRule{ - { - IPAddress: proto.String(loadBalancerIP), - PortRange: proto.String("100-100"), - }, - }, - }, - }, - wantErr: true, - }, - "fails when retrieving uid": { - instanceAPI: stubInstancesClient{GetInstance: instance}, - stubMetadataClient: stubMetadataClient{instanceIDErr: someErr}, - stubForwardingRulesClient: stubForwardingRulesClient{ - ForwardingRuleIterator: &stubForwardingRuleIterator{ - rules: []*computepb.ForwardingRule{ - { - IPAddress: proto.String(loadBalancerIP), - PortRange: proto.String("100-100"), - Labels: map[string]string{cloud.TagUID: uid, "constellation-use": use}, - }, - }, - }, - }, - wantErr: true, - }, - "fails when answer has empty port range": { - instanceAPI: stubInstancesClient{GetInstance: instance}, - stubMetadataClient: stubMetadataClient{}, - stubForwardingRulesClient: stubForwardingRulesClient{ - ForwardingRuleIterator: &stubForwardingRuleIterator{ - rules: []*computepb.ForwardingRule{ - { - IPAddress: proto.String(loadBalancerIP), - Labels: map[string]string{cloud.TagUID: uid, "constellation-use": use}, - }, - }, - }, - }, - wantErr: true, - }, - "fails when retrieving loadbalancer IP": { - instanceAPI: stubInstancesClient{GetInstance: instance}, - stubMetadataClient: stubMetadataClient{}, - stubForwardingRulesClient: stubForwardingRulesClient{ - ForwardingRuleIterator: &stubForwardingRuleIterator{ - nextErr: someErr, - rules: []*computepb.ForwardingRule{ - { - IPAddress: proto.String(loadBalancerIP), - PortRange: proto.String("100-100"), - Labels: map[string]string{cloud.TagUID: uid, "constellation-use": use}, - }, - }, - }, - }, - wantErr: true, - }, - "fails on incorrect use label": { - instanceAPI: stubInstancesClient{GetInstance: instance}, - stubMetadataClient: stubMetadataClient{InstanceValue: uid}, - stubForwardingRulesClient: stubForwardingRulesClient{ - ForwardingRuleIterator: &stubForwardingRuleIterator{ - rules: []*computepb.ForwardingRule{ - { - IPAddress: proto.String(loadBalancerIP), - PortRange: proto.String("100-100"), - Labels: map[string]string{cloud.TagUID: uid, "constellation-use": "bootstrapper"}, - }, - }, - }, - }, - wantErr: true, - }, - } - for name, tc := range testCases { - t.Run(name, func(t *testing.T) { - assert := assert.New(t) - require := require.New(t) - - client := Client{instanceAPI: tc.instanceAPI, forwardingRulesAPI: tc.stubForwardingRulesClient, metadataAPI: tc.stubMetadataClient} - aliasCIDR, err := client.RetrieveLoadBalancerEndpoint(context.Background(), "project") - - if tc.wantErr { - assert.Error(err) - return - } - require.NoError(err) - assert.Equal(tc.wantLoadBalancerIP+":100", aliasCIDR) - }) - } -} - -func TestClose(t *testing.T) { - someErr := errors.New("failed") - - assert := assert.New(t) - - client := Client{instanceAPI: stubInstancesClient{}, subnetworkAPI: stubSubnetworksClient{}, forwardingRulesAPI: stubForwardingRulesClient{}} - assert.NoError(client.Close()) - - client = Client{instanceAPI: stubInstancesClient{CloseErr: someErr}, subnetworkAPI: stubSubnetworksClient{}, forwardingRulesAPI: stubForwardingRulesClient{}} - assert.Error(client.Close()) - - client = Client{instanceAPI: stubInstancesClient{}, subnetworkAPI: stubSubnetworksClient{CloseErr: someErr}, forwardingRulesAPI: stubForwardingRulesClient{}} - assert.Error(client.Close()) - - client = Client{instanceAPI: stubInstancesClient{}, subnetworkAPI: stubSubnetworksClient{}, forwardingRulesAPI: stubForwardingRulesClient{CloseErr: someErr}} - assert.Error(client.Close()) -} - -func TestFetchSSHKeys(t *testing.T) { - testCases := map[string]struct { - metadata map[string]string - wantKeys map[string][]string - }{ - "fetch works": { - metadata: map[string]string{"ssh-keys": "bob:ssh-rsa bobskey"}, - wantKeys: map[string][]string{"bob": {"ssh-rsa bobskey"}}, - }, - "google ssh key metadata is ignored": { - metadata: map[string]string{"ssh-keys": "bob:ssh-rsa bobskey google-ssh {\"userName\":\"bob\",\"expireOn\":\"2021-06-14T16:59:03+0000\"}"}, - wantKeys: map[string][]string{"bob": {"ssh-rsa bobskey"}}, - }, - "ssh key format error is ignored": { - metadata: map[string]string{"ssh-keys": "incorrect-format"}, - wantKeys: map[string][]string{}, - }, - "ssh key format space error is ignored": { - metadata: map[string]string{"ssh-keys": "user:incorrect-key-format"}, - wantKeys: map[string][]string{}, - }, - "metadata field empty": { - metadata: map[string]string{"ssh-keys": ""}, - wantKeys: map[string][]string{}, - }, - "metadata field missing": { - metadata: map[string]string{}, - wantKeys: map[string][]string{}, - }, - "multiple keys": { - metadata: map[string]string{"ssh-keys": "bob:ssh-rsa bobskey\nalice:ssh-rsa alicekey"}, - wantKeys: map[string][]string{ - "bob": {"ssh-rsa bobskey"}, - "alice": {"ssh-rsa alicekey"}, - }, - }, - } - - for name, tc := range testCases { - t.Run(name, func(t *testing.T) { - assert := assert.New(t) - - keys := extractSSHKeys(tc.metadata) - assert.Equal(tc.wantKeys, keys) - }) - } -} - -type stubInstanceIterator struct { - instances []*computepb.Instance - nextErr error - - internalCounter int -} - -func (i *stubInstanceIterator) Next() (*computepb.Instance, error) { - if i.nextErr != nil { - return nil, i.nextErr - } - if i.internalCounter >= len(i.instances) { - i.internalCounter = 0 - return nil, iterator.Done - } - resp := i.instances[i.internalCounter] - i.internalCounter++ - return resp, nil -} - -type stubInstancesClient struct { - GetInstance *computepb.Instance - GetErr error - ListInstanceIterator InstanceIterator - SetMetadataOperation *compute.Operation - SetMetadataErr error - CloseErr error -} - -func (s stubInstancesClient) Get(ctx context.Context, req *computepb.GetInstanceRequest, opts ...gax.CallOption) (*computepb.Instance, error) { - return s.GetInstance, s.GetErr -} - -func (s stubInstancesClient) List(ctx context.Context, req *computepb.ListInstancesRequest, opts ...gax.CallOption) InstanceIterator { - return s.ListInstanceIterator -} - -func (s stubInstancesClient) SetMetadata(ctx context.Context, req *computepb.SetMetadataInstanceRequest, opts ...gax.CallOption) (*compute.Operation, error) { - return s.SetMetadataOperation, s.SetMetadataErr -} - -func (s stubInstancesClient) Close() error { - return s.CloseErr -} - -type stubSubnetworksClient struct { - GetSubnetwork *computepb.Subnetwork - GetErr error - SubnetworkIterator SubnetworkIterator - CloseErr error -} - -func (s stubSubnetworksClient) Get(ctx context.Context, req *computepb.GetSubnetworkRequest, opts ...gax.CallOption) (*computepb.Subnetwork, error) { - return s.GetSubnetwork, s.GetErr -} - -func (s stubSubnetworksClient) List(ctx context.Context, req *computepb.ListSubnetworksRequest, opts ...gax.CallOption) SubnetworkIterator { - return s.SubnetworkIterator -} - -func (s stubSubnetworksClient) Close() error { - return s.CloseErr -} - -type stubForwardingRuleIterator struct { - rules []*computepb.ForwardingRule - nextErr error - - internalCounter int -} - -func (i *stubForwardingRuleIterator) Next() (*computepb.ForwardingRule, error) { - if i.nextErr != nil { - return nil, i.nextErr - } - if i.internalCounter >= len(i.rules) { - i.internalCounter = 0 - return nil, iterator.Done - } - resp := i.rules[i.internalCounter] - i.internalCounter++ - return resp, nil -} - -type stubForwardingRulesClient struct { - ForwardingRuleIterator ForwardingRuleIterator - GetErr error - CloseErr error -} - -func (s stubForwardingRulesClient) List(ctx context.Context, req *computepb.ListGlobalForwardingRulesRequest, opts ...gax.CallOption) ForwardingRuleIterator { - return s.ForwardingRuleIterator -} - -func (s stubForwardingRulesClient) Close() error { - return s.CloseErr -} - -type stubMetadataClient struct { - InstanceValue string - InstanceErr error - instanceIDValue string - instanceIDErr error - ProjectIDValue string - ProjectIDErr error - ZoneValue string - ZoneErr error - InstanceNameValue string - InstanceNameErr error -} - -func (s stubMetadataClient) InstanceAttributeValue(attr string) (string, error) { - return s.InstanceValue, s.InstanceErr -} - -func (s stubMetadataClient) InstanceID() (string, error) { - return s.instanceIDValue, s.instanceIDErr -} - -func (s stubMetadataClient) ProjectID() (string, error) { - return s.ProjectIDValue, s.ProjectIDErr -} - -func (s stubMetadataClient) Zone() (string, error) { - return s.ZoneValue, s.ZoneErr -} - -func (s stubMetadataClient) InstanceName() (string, error) { - return s.InstanceNameValue, s.InstanceNameErr -} diff --git a/internal/cloud/gcp/cloud.go b/internal/cloud/gcp/cloud.go new file mode 100644 index 000000000..b4f987940 --- /dev/null +++ b/internal/cloud/gcp/cloud.go @@ -0,0 +1,319 @@ +/* +Copyright (c) Edgeless Systems GmbH + +SPDX-License-Identifier: AGPL-3.0-only +*/ + +package gcp + +import ( + "context" + "errors" + "fmt" + "net" + "path" + "regexp" + "strings" + + compute "cloud.google.com/go/compute/apiv1" + imds "cloud.google.com/go/compute/metadata" + "github.com/edgelesssys/constellation/v2/internal/cloud" + "github.com/edgelesssys/constellation/v2/internal/cloud/gcpshared" + "github.com/edgelesssys/constellation/v2/internal/cloud/metadata" + "github.com/edgelesssys/constellation/v2/internal/role" + "google.golang.org/api/iterator" + computepb "google.golang.org/genproto/googleapis/cloud/compute/v1" + "google.golang.org/protobuf/proto" +) + +const ( + // tagUsage is a label key used to indicate the use of the resource. + tagUsage = "constellation-use" +) + +var zoneFromRegionRegex = regexp.MustCompile("([a-z]*-[a-z]*[0-9])") + +// Cloud provides GCP cloud metadata information and API access. +type Cloud struct { + forwardingRulesAPI forwardingRulesAPI + imds imdsAPI + instanceAPI instanceAPI + subnetAPI subnetAPI + + closers []func() error +} + +// New creates and initializes Cloud. +// The Close method should be called when Cloud is no longer needed. +func New(ctx context.Context) (cloud *Cloud, err error) { + var closers []func() error + + insAPI, err := compute.NewInstancesRESTClient(ctx) + if err != nil { + return nil, err + } + closers = append(closers, insAPI.Close) + forwardingRulesAPI, err := compute.NewGlobalForwardingRulesRESTClient(ctx) + if err != nil { + return nil, err + } + closers = append(closers, forwardingRulesAPI.Close) + subnetAPI, err := compute.NewSubnetworksRESTClient(ctx) + if err != nil { + return nil, err + } + closers = append(closers, subnetAPI.Close) + + return &Cloud{ + imds: imds.NewClient(nil), + instanceAPI: &instanceClient{insAPI}, + forwardingRulesAPI: &forwardingRulesClient{forwardingRulesAPI}, + subnetAPI: subnetAPI, + closers: closers, + }, nil +} + +// Close closes all connections to the GCP API server. +func (c *Cloud) Close() { + for _, close := range c.closers { + _ = close() + } +} + +// GetInstance retrieves an instance using its providerID. +func (c *Cloud) GetInstance(ctx context.Context, providerID string) (metadata.InstanceMetadata, error) { + project, zone, instanceName, err := gcpshared.SplitProviderID(providerID) + if err != nil { + return metadata.InstanceMetadata{}, fmt.Errorf("invalid providerID: %w", err) + } + + return c.getInstance(ctx, project, zone, instanceName) +} + +// GetLoadBalancerEndpoint returns the endpoint of the load balancer. +func (c *Cloud) GetLoadBalancerEndpoint(ctx context.Context) (string, error) { + project, zone, instanceName, err := c.retrieveInstanceInfo() + if err != nil { + return "", err + } + uid, err := c.uid(ctx, project, zone, instanceName) + if err != nil { + return "", err + } + + var resp *computepb.ForwardingRule + iter := c.forwardingRulesAPI.List(ctx, &computepb.ListGlobalForwardingRulesRequest{ + Project: project, + Filter: proto.String(fmt.Sprintf("(labels.%s:%s) AND (labels.%s:kubernetes)", cloud.TagUID, uid, tagUsage)), + }) + for resp, err = iter.Next(); err == nil; resp, err = iter.Next() { + if resp.PortRange == nil { + continue + } + if resp.IPAddress == nil { + continue + } + portRange := strings.Split(*resp.PortRange, "-") + return net.JoinHostPort(*resp.IPAddress, portRange[0]), nil + } + + return "", fmt.Errorf("kubernetes load balancer with UID %s not found: %w", uid, err) +} + +// List retrieves all instances belonging to the current constellation. +func (c *Cloud) List(ctx context.Context) ([]metadata.InstanceMetadata, error) { + project, zone, instanceName, err := c.retrieveInstanceInfo() + if err != nil { + return nil, err + } + uid, err := c.uid(ctx, project, zone, instanceName) + if err != nil { + return nil, err + } + + var instances []metadata.InstanceMetadata + var resp *computepb.Instance + iter := c.instanceAPI.List(ctx, &computepb.ListInstancesRequest{ + Filter: proto.String(fmt.Sprintf("labels.%s:%s", cloud.TagUID, uid)), + Project: project, + Zone: zone, + }) + for resp, err = iter.Next(); err == nil; resp, err = iter.Next() { + instance, err := convertToInstanceMetadata(resp, project, zone) + if err != nil { + return nil, fmt.Errorf("retrieving instance list from GCP: failed to convert instance: %w", err) + } + + // convertToInstanceMetadata already checks for nil resp.NetworkInterfaces + if len(resp.NetworkInterfaces) == 0 || resp.NetworkInterfaces[0] == nil || + resp.NetworkInterfaces[0].Subnetwork == nil { + return nil, errors.New("retrieving compute instance: received invalid instance") + } + + subnetCIDR, err := c.retrieveSubnetworkAliasCIDR(ctx, project, zone, *resp.NetworkInterfaces[0].Subnetwork) + if err != nil { + return nil, fmt.Errorf("retrieving compute instance: failed to retrieve subnet CIDR: %w", err) + } + instance.SecondaryIPRange = subnetCIDR + + instances = append(instances, instance) + } + if errors.Is(err, iterator.Done) { + return instances, nil + } + return nil, fmt.Errorf("retrieving instance list from GCP: %w", err) +} + +// ProviderID returns the providerID of the current instance. +func (c *Cloud) ProviderID(ctx context.Context) (string, error) { + project, zone, instanceName, err := c.retrieveInstanceInfo() + if err != nil { + return "", err + } + return gcpshared.JoinProviderID(project, zone, instanceName), nil +} + +// Self retrieves the current instance. +func (c *Cloud) Self(ctx context.Context) (metadata.InstanceMetadata, error) { + project, zone, instanceName, err := c.retrieveInstanceInfo() + if err != nil { + return metadata.InstanceMetadata{}, err + } + return c.getInstance(ctx, project, zone, instanceName) +} + +// UID retrieves the UID of the constellation. +func (c *Cloud) UID(ctx context.Context) (string, error) { + project, zone, instanceName, err := c.retrieveInstanceInfo() + if err != nil { + return "", err + } + return c.uid(ctx, project, zone, instanceName) +} + +// getInstance retrieves an instance using its project, zone and name, and parses it to metadata.InstanceMetadata. +func (c *Cloud) getInstance(ctx context.Context, project, zone, instanceName string) (metadata.InstanceMetadata, error) { + gcpInstance, err := c.instanceAPI.Get(ctx, &computepb.GetInstanceRequest{ + Project: project, + Zone: zone, + Instance: instanceName, + }) + if err != nil { + return metadata.InstanceMetadata{}, fmt.Errorf("retrieving compute instance: %w", err) + } + + if gcpInstance == nil || gcpInstance.NetworkInterfaces == nil || len(gcpInstance.NetworkInterfaces) == 0 || + gcpInstance.NetworkInterfaces[0] == nil || gcpInstance.NetworkInterfaces[0].Subnetwork == nil { + return metadata.InstanceMetadata{}, errors.New("retrieving compute instance: received invalid instance") + } + subnetCIDR, err := c.retrieveSubnetworkAliasCIDR(ctx, project, zone, *gcpInstance.NetworkInterfaces[0].Subnetwork) + if err != nil { + return metadata.InstanceMetadata{}, err + } + + instance, err := convertToInstanceMetadata(gcpInstance, project, zone) + if err != nil { + return metadata.InstanceMetadata{}, fmt.Errorf("converting instance: %w", err) + } + instance.SecondaryIPRange = subnetCIDR + return instance, nil +} + +// retrieveInstanceInfo retrieves the project, zone and instance name of the current instance using the imds API. +func (c *Cloud) retrieveInstanceInfo() (project, zone, instanceName string, err error) { + project, err = c.imds.ProjectID() + if err != nil { + return "", "", "", fmt.Errorf("retrieving project ID from imds: %w", err) + } + zone, err = c.imds.Zone() + if err != nil { + return "", "", "", fmt.Errorf("retrieving zone from imds: %w", err) + } + instanceName, err = c.imds.InstanceName() + if err != nil { + return "", "", "", fmt.Errorf("retrieving instance name from imds: %w", err) + } + return project, zone, instanceName, nil +} + +// retrieveSubnetworkAliasCIDR retrieves the secondary IP range CIDR of the subnetwork, +// identified by project, zone and subnetworkURI. +func (c *Cloud) retrieveSubnetworkAliasCIDR(ctx context.Context, project, zone, subnetworkURI string) (string, error) { + // convert: + // zone --> region + // europe-west3-b --> europe-west3 + region := zoneFromRegionRegex.FindString(zone) + if region == "" { + return "", fmt.Errorf("invalid zone %s", zone) + } + + req := &computepb.GetSubnetworkRequest{ + Project: project, + Region: region, + Subnetwork: path.Base(subnetworkURI), + } + subnetwork, err := c.subnetAPI.Get(ctx, req) + if err != nil { + return "", fmt.Errorf("retrieving subnetwork alias CIDR failed: %w", err) + } + if subnetwork == nil || len(subnetwork.SecondaryIpRanges) == 0 || + subnetwork.SecondaryIpRanges[0] == nil || subnetwork.SecondaryIpRanges[0].IpCidrRange == nil { + return "", fmt.Errorf("retrieving subnetwork alias CIDR failed: received invalid subnetwork") + } + + return *subnetwork.SecondaryIpRanges[0].IpCidrRange, nil +} + +// uid retrieves the UID of the instance identified by project, zone and instanceName. +// The UID is retrieved from the instance's labels. +func (c *Cloud) uid(ctx context.Context, project, zone, instanceName string) (string, error) { + instance, err := c.instanceAPI.Get(ctx, &computepb.GetInstanceRequest{ + Project: project, + Zone: zone, + Instance: instanceName, + }) + if err != nil { + return "", fmt.Errorf("retrieving compute instance: %w", err) + } + if instance == nil || instance.Labels == nil { + return "", errors.New("retrieving compute instance: received instance with invalid labels") + } + return instance.Labels[cloud.TagUID], nil +} + +// convertToInstanceMetadata converts a *computepb.Instance to a metadata.InstanceMetadata. +func convertToInstanceMetadata(in *computepb.Instance, project string, zone string) (metadata.InstanceMetadata, error) { + if in.Name == nil { + return metadata.InstanceMetadata{}, fmt.Errorf("missing instance name") + } + + var vpcIP string + var ips []string + for _, interf := range in.NetworkInterfaces { + if interf == nil { + continue + } + + // use private IP from the default interface + if interf.NetworkIP != nil && interf.Name != nil && *interf.Name == "nic0" { + vpcIP = *interf.NetworkIP + } + + if interf.AliasIpRanges == nil { + continue + } + for _, aliasIP := range interf.AliasIpRanges { + if aliasIP != nil && aliasIP.IpCidrRange != nil { + ips = append(ips, *aliasIP.IpCidrRange) + } + } + } + + return metadata.InstanceMetadata{ + Name: *in.Name, + ProviderID: gcpshared.JoinProviderID(project, zone, *in.Name), + Role: role.FromString(in.Labels[cloud.TagRole]), + VPCIP: vpcIP, + AliasIPRanges: ips, + }, nil +} diff --git a/internal/cloud/gcp/cloud_test.go b/internal/cloud/gcp/cloud_test.go new file mode 100644 index 000000000..edf06ad7e --- /dev/null +++ b/internal/cloud/gcp/cloud_test.go @@ -0,0 +1,898 @@ +/* +Copyright (c) Edgeless Systems GmbH + +SPDX-License-Identifier: AGPL-3.0-only +*/ + +package gcp + +import ( + "context" + "errors" + "testing" + + "github.com/edgelesssys/constellation/v2/internal/cloud" + "github.com/edgelesssys/constellation/v2/internal/cloud/metadata" + "github.com/edgelesssys/constellation/v2/internal/role" + gax "github.com/googleapis/gax-go/v2" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.uber.org/goleak" + "google.golang.org/api/iterator" + computepb "google.golang.org/genproto/googleapis/cloud/compute/v1" + "google.golang.org/protobuf/proto" +) + +func TestMain(m *testing.M) { + goleak.VerifyTestMain(m, + // https://github.com/census-instrumentation/opencensus-go/issues/1262 + goleak.IgnoreTopFunction("go.opencensus.io/stats/view.(*worker).start"), + ) +} + +func TestGetInstance(t *testing.T) { + someErr := errors.New("failed") + goodInstance := &computepb.Instance{ + Name: proto.String("someInstance"), + Zone: proto.String("someZone-west3-b"), + Labels: map[string]string{ + cloud.TagUID: "1234", + cloud.TagRole: role.ControlPlane.String(), + }, + NetworkInterfaces: []*computepb.NetworkInterface{ + { + Name: proto.String("nic0"), + NetworkIP: proto.String("192.0.2.0"), + AliasIpRanges: []*computepb.AliasIpRange{ + { + IpCidrRange: proto.String("192.0.3.0/8"), + }, + }, + Subnetwork: proto.String("projects/someProject/regions/someRegion/subnetworks/someSubnetwork"), + }, + }, + } + + testCases := map[string]struct { + projectID, instanceName, zone string + instanceAPI stubInstanceAPI + subnetAPI stubSubnetAPI + wantErr bool + wantInstance metadata.InstanceMetadata + }{ + "success": { + instanceName: "someInstance", + projectID: "someProject", + zone: "someZone-west3-b", + instanceAPI: stubInstanceAPI{ + instance: goodInstance, + }, + subnetAPI: stubSubnetAPI{ + subnet: &computepb.Subnetwork{ + SecondaryIpRanges: []*computepb.SubnetworkSecondaryRange{ + { + IpCidrRange: proto.String("198.51.100.0/24"), + }, + }, + }, + }, + wantInstance: metadata.InstanceMetadata{ + Name: "someInstance", + Role: role.ControlPlane, + ProviderID: "gce://someProject/someZone-west3-b/someInstance", + VPCIP: "192.0.2.0", + AliasIPRanges: []string{"192.0.3.0/8"}, + SecondaryIPRange: "198.51.100.0/24", + }, + }, + "get instance error": { + instanceName: "someInstance", + projectID: "someProject", + zone: "someZone-west3-b", + instanceAPI: stubInstanceAPI{ + instanceErr: someErr, + }, + subnetAPI: stubSubnetAPI{ + subnet: &computepb.Subnetwork{ + SecondaryIpRanges: []*computepb.SubnetworkSecondaryRange{ + { + IpCidrRange: proto.String("198.51.100.0/24"), + }, + }, + }, + }, + wantErr: true, + }, + "get subnet error": { + instanceName: "someInstance", + projectID: "someProject", + zone: "someZone-west3-b", + instanceAPI: stubInstanceAPI{ + instance: goodInstance, + }, + subnetAPI: stubSubnetAPI{ + subnetErr: someErr, + }, + wantErr: true, + }, + "invalid instance": { + instanceName: "someInstance", + projectID: "someProject", + zone: "someZone-west3-b", + instanceAPI: stubInstanceAPI{ + instance: nil, + }, + subnetAPI: stubSubnetAPI{ + subnet: &computepb.Subnetwork{ + SecondaryIpRanges: []*computepb.SubnetworkSecondaryRange{ + { + IpCidrRange: proto.String("198.51.100.0/24"), + }, + }, + }, + }, + wantErr: true, + }, + "invalid zone": { + instanceName: "someInstance", + projectID: "someProject", + zone: "invalidZone", + instanceAPI: stubInstanceAPI{ + instance: goodInstance, + }, + subnetAPI: stubSubnetAPI{ + subnet: &computepb.Subnetwork{ + SecondaryIpRanges: []*computepb.SubnetworkSecondaryRange{ + { + IpCidrRange: proto.String("198.51.100.0/24"), + }, + }, + }, + }, + wantErr: true, + }, + } + + for name, tc := range testCases { + t.Run(name, func(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + + cloud := &Cloud{ + instanceAPI: &tc.instanceAPI, + subnetAPI: &tc.subnetAPI, + } + instance, err := cloud.getInstance(context.Background(), tc.projectID, tc.zone, tc.instanceName) + + if tc.wantErr { + assert.Error(err) + return + } + require.NoError(err) + assert.Equal(tc.wantInstance, instance) + }) + } +} + +func TestGetLoadbalancerEndpoint(t *testing.T) { + someErr := errors.New("failed") + goodInstance := &computepb.Instance{ + Name: proto.String("someInstance"), + Zone: proto.String("someZone-west3-b"), + Labels: map[string]string{ + cloud.TagUID: "1234", + cloud.TagRole: role.ControlPlane.String(), + }, + NetworkInterfaces: []*computepb.NetworkInterface{ + { + Name: proto.String("nic0"), + NetworkIP: proto.String("192.0.2.0"), + AliasIpRanges: []*computepb.AliasIpRange{ + { + IpCidrRange: proto.String("192.0.3.0/8"), + }, + }, + Subnetwork: proto.String("projects/someProject/regions/someRegion/subnetworks/someSubnetwork"), + }, + }, + } + + testCases := map[string]struct { + imds stubIMDS + instanceAPI stubInstanceAPI + forwardingRulesAPI stubForwardingRulesAPI + wantEndpoint string + wantErr bool + }{ + "success": { + imds: stubIMDS{ + projectID: "someProject", + zone: "someZone-west3-b", + instanceName: "someInstance", + }, + instanceAPI: stubInstanceAPI{ + instance: goodInstance, + }, + forwardingRulesAPI: stubForwardingRulesAPI{ + iterator: &stubForwardingRulesIterator{ + forwardingRules: []*computepb.ForwardingRule{ + { + PortRange: proto.String("6443"), + IPAddress: proto.String("192.0.2.255"), + }, + }, + }, + }, + wantEndpoint: "192.0.2.255:6443", + }, + "imds error": { + imds: stubIMDS{ + projectIDErr: someErr, + zone: "someZone-west3-b", + instanceName: "someInstance", + }, + instanceAPI: stubInstanceAPI{ + instance: goodInstance, + }, + forwardingRulesAPI: stubForwardingRulesAPI{ + iterator: &stubForwardingRulesIterator{ + forwardingRules: []*computepb.ForwardingRule{ + { + PortRange: proto.String("6443"), + IPAddress: proto.String("192.0.2.255"), + }, + }, + }, + }, + wantErr: true, + }, + "iterator error": { + imds: stubIMDS{ + projectID: "someProject", + zone: "someZone-west3-b", + instanceName: "someInstance", + }, + instanceAPI: stubInstanceAPI{ + instance: goodInstance, + }, + forwardingRulesAPI: stubForwardingRulesAPI{ + iterator: &stubForwardingRulesIterator{ + err: someErr, + }, + }, + wantErr: true, + }, + "no forwarding rules": { + imds: stubIMDS{ + projectID: "someProject", + zone: "someZone-west3-b", + instanceName: "someInstance", + }, + instanceAPI: stubInstanceAPI{ + instance: goodInstance, + }, + forwardingRulesAPI: stubForwardingRulesAPI{ + iterator: &stubForwardingRulesIterator{}, + }, + wantErr: true, + }, + "missing port range": { + imds: stubIMDS{ + projectID: "someProject", + zone: "someZone-west3-b", + instanceName: "someInstance", + }, + instanceAPI: stubInstanceAPI{ + instance: goodInstance, + }, + forwardingRulesAPI: stubForwardingRulesAPI{ + iterator: &stubForwardingRulesIterator{ + forwardingRules: []*computepb.ForwardingRule{ + { + IPAddress: proto.String("192.0.2.255"), + }, + }, + }, + }, + wantErr: true, + }, + "missing IP address": { + imds: stubIMDS{ + projectID: "someProject", + zone: "someZone-west3-b", + instanceName: "someInstance", + }, + instanceAPI: stubInstanceAPI{ + instance: goodInstance, + }, + forwardingRulesAPI: stubForwardingRulesAPI{ + iterator: &stubForwardingRulesIterator{ + forwardingRules: []*computepb.ForwardingRule{ + { + PortRange: proto.String("6443"), + }, + }, + }, + }, + wantErr: true, + }, + "get instance error": { + imds: stubIMDS{ + projectID: "someProject", + zone: "someZone-west3-b", + instanceName: "someInstance", + }, + instanceAPI: stubInstanceAPI{ + instanceErr: someErr, + }, + forwardingRulesAPI: stubForwardingRulesAPI{ + iterator: &stubForwardingRulesIterator{ + forwardingRules: []*computepb.ForwardingRule{ + { + PortRange: proto.String("6443"), + IPAddress: proto.String("192.0.2.255"), + }, + }, + }, + }, + wantErr: true, + }, + "invalid instance": { + imds: stubIMDS{ + projectID: "someProject", + zone: "someZone-west3-b", + instanceName: "someInstance", + }, + instanceAPI: stubInstanceAPI{ + instance: nil, + }, + forwardingRulesAPI: stubForwardingRulesAPI{ + iterator: &stubForwardingRulesIterator{ + forwardingRules: []*computepb.ForwardingRule{ + { + PortRange: proto.String("6443"), + IPAddress: proto.String("192.0.2.255"), + }, + }, + }, + }, + wantErr: true, + }, + } + + for name, tc := range testCases { + t.Run(name, func(t *testing.T) { + assert := assert.New(t) + + cloud := &Cloud{ + imds: &tc.imds, + instanceAPI: &tc.instanceAPI, + forwardingRulesAPI: &tc.forwardingRulesAPI, + } + + endpoint, err := cloud.GetLoadBalancerEndpoint(context.Background()) + if tc.wantErr { + assert.Error(err) + return + } + assert.NoError(err) + assert.Equal(tc.wantEndpoint, endpoint) + }) + } +} + +func TestList(t *testing.T) { + someErr := errors.New("failed") + goodInstance := &computepb.Instance{ + Name: proto.String("someInstance"), + Zone: proto.String("someZone-west3-b"), + Labels: map[string]string{ + cloud.TagUID: "1234", + cloud.TagRole: role.ControlPlane.String(), + }, + NetworkInterfaces: []*computepb.NetworkInterface{ + { + Name: proto.String("nic0"), + NetworkIP: proto.String("192.0.2.0"), + AliasIpRanges: []*computepb.AliasIpRange{ + { + IpCidrRange: proto.String("198.51.100.0/24"), + }, + }, + Subnetwork: proto.String("projects/someProject/regions/someRegion/subnetworks/someSubnetwork"), + }, + }, + } + goodSubnet := &computepb.Subnetwork{ + SecondaryIpRanges: []*computepb.SubnetworkSecondaryRange{ + { + IpCidrRange: proto.String("198.51.100.0/24"), + }, + }, + } + + testCases := map[string]struct { + imds stubIMDS + instanceAPI stubInstanceAPI + subnetAPI stubSubnetAPI + wantErr bool + wantInstances []metadata.InstanceMetadata + }{ + "success": { + imds: stubIMDS{ + projectID: "someProject", + zone: "someZone-west3-b", + instanceName: "someInstance", + }, + instanceAPI: stubInstanceAPI{ + instance: goodInstance, + iterator: &stubInstanceIterator{ + instances: []*computepb.Instance{ + goodInstance, + }, + }, + }, + subnetAPI: stubSubnetAPI{ + subnet: goodSubnet, + }, + wantInstances: []metadata.InstanceMetadata{ + { + Name: "someInstance", + Role: role.ControlPlane, + ProviderID: "gce://someProject/someZone-west3-b/someInstance", + VPCIP: "192.0.2.0", + AliasIPRanges: []string{"198.51.100.0/24"}, + SecondaryIPRange: "198.51.100.0/24", + }, + }, + }, + "list multiple instances": { + imds: stubIMDS{ + projectID: "someProject", + zone: "someZone-west3-b", + instanceName: "someInstance", + }, + instanceAPI: stubInstanceAPI{ + instance: goodInstance, + iterator: &stubInstanceIterator{ + instances: []*computepb.Instance{ + goodInstance, + { + Name: proto.String("anotherInstance"), + Zone: proto.String("someZone-west3-b"), + Labels: map[string]string{ + cloud.TagUID: "1234", + cloud.TagRole: role.Worker.String(), + }, + NetworkInterfaces: []*computepb.NetworkInterface{ + { + Name: proto.String("nic0"), + NetworkIP: proto.String("192.0.2.1"), + AliasIpRanges: []*computepb.AliasIpRange{ + { + IpCidrRange: proto.String("198.51.100.0/24"), + }, + }, + Subnetwork: proto.String("projects/someProject/regions/someRegion/subnetworks/someSubnetwork"), + }, + }, + }, + }, + }, + }, + subnetAPI: stubSubnetAPI{ + subnet: goodSubnet, + }, + wantInstances: []metadata.InstanceMetadata{ + { + Name: "someInstance", + Role: role.ControlPlane, + ProviderID: "gce://someProject/someZone-west3-b/someInstance", + VPCIP: "192.0.2.0", + AliasIPRanges: []string{"198.51.100.0/24"}, + SecondaryIPRange: "198.51.100.0/24", + }, + { + Name: "anotherInstance", + Role: role.Worker, + ProviderID: "gce://someProject/someZone-west3-b/anotherInstance", + VPCIP: "192.0.2.1", + AliasIPRanges: []string{"198.51.100.0/24"}, + SecondaryIPRange: "198.51.100.0/24", + }, + }, + }, + "imds error": { + imds: stubIMDS{ + projectIDErr: someErr, + }, + instanceAPI: stubInstanceAPI{ + instance: goodInstance, + iterator: &stubInstanceIterator{ + instances: []*computepb.Instance{ + goodInstance, + }, + }, + }, + subnetAPI: stubSubnetAPI{ + subnet: goodSubnet, + }, + wantErr: true, + }, + "iterator error": { + imds: stubIMDS{ + projectID: "someProject", + zone: "someZone-west3-b", + instanceName: "someInstance", + }, + instanceAPI: stubInstanceAPI{ + instance: goodInstance, + iterator: &stubInstanceIterator{ + err: someErr, + }, + }, + subnetAPI: stubSubnetAPI{ + subnet: goodSubnet, + }, + wantErr: true, + }, + "get instance error": { + imds: stubIMDS{ + projectID: "someProject", + zone: "someZone-west3-b", + instanceName: "someInstance", + }, + instanceAPI: stubInstanceAPI{ + instanceErr: someErr, + iterator: &stubInstanceIterator{ + instances: []*computepb.Instance{ + goodInstance, + }, + }, + }, + subnetAPI: stubSubnetAPI{ + subnet: goodSubnet, + }, + wantErr: true, + }, + "get subnet error": { + imds: stubIMDS{ + projectID: "someProject", + zone: "someZone-west3-b", + instanceName: "someInstance", + }, + instanceAPI: stubInstanceAPI{ + instance: goodInstance, + iterator: &stubInstanceIterator{ + instances: []*computepb.Instance{ + goodInstance, + }, + }, + }, + subnetAPI: stubSubnetAPI{ + subnetErr: someErr, + }, + wantErr: true, + }, + } + + for name, tc := range testCases { + t.Run(name, func(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + + cloud := &Cloud{ + imds: &tc.imds, + instanceAPI: &tc.instanceAPI, + subnetAPI: &tc.subnetAPI, + } + + instances, err := cloud.List(context.Background()) + if tc.wantErr { + assert.Error(err) + return + } + require.NoError(err) + assert.ElementsMatch(tc.wantInstances, instances) + }) + } +} + +func TestRetrieveInstanceInfo(t *testing.T) { + someErr := errors.New("failed") + + testCases := map[string]struct { + imds stubIMDS + wantErr bool + }{ + "success": { + imds: stubIMDS{ + projectID: "someProject", + zone: "someZone-west3-b", + instanceName: "someInstance", + }, + }, + "get project id error": { + imds: stubIMDS{ + zone: "someZone-west3-b", + instanceName: "someInstance", + projectIDErr: someErr, + }, + wantErr: true, + }, + "get zone error": { + imds: stubIMDS{ + projectID: "someProject", + instanceName: "someInstance", + zoneErr: someErr, + }, + wantErr: true, + }, + "get instance name error": { + imds: stubIMDS{ + projectID: "someProject", + zone: "someZone-west3-b", + instanceNameErr: someErr, + }, + wantErr: true, + }, + } + + for name, tc := range testCases { + t.Run(name, func(t *testing.T) { + assert := assert.New(t) + + cloud := &Cloud{ + imds: &tc.imds, + } + + project, zone, instance, err := cloud.retrieveInstanceInfo() + if tc.wantErr { + assert.Error(err) + return + } + assert.NoError(err) + assert.Equal(tc.imds.projectID, project) + assert.Equal(tc.imds.zone, zone) + assert.Equal(tc.imds.instanceName, instance) + }) + } +} + +func TestUID(t *testing.T) { + someErr := errors.New("failed") + + testCases := map[string]struct { + imds stubIMDS + instanceAPI stubInstanceAPI + wantUID string + wantErr bool + }{ + "success": { + imds: stubIMDS{ + projectID: "someProject", + zone: "someZone-west3-b", + instanceName: "someInstance", + }, + instanceAPI: stubInstanceAPI{ + instance: &computepb.Instance{ + Name: proto.String("someInstance"), + Zone: proto.String("someZone-west3-b"), + Labels: map[string]string{ + cloud.TagUID: "1234", + cloud.TagRole: role.ControlPlane.String(), + }, + }, + }, + wantUID: "1234", + }, + "imds error": { + imds: stubIMDS{ + projectIDErr: someErr, + zone: "someZone-west3-b", + instanceName: "someInstance", + }, + instanceAPI: stubInstanceAPI{ + instance: &computepb.Instance{ + Name: proto.String("someInstance"), + Zone: proto.String("someZone-west3-b"), + Labels: map[string]string{ + cloud.TagUID: "1234", + cloud.TagRole: role.ControlPlane.String(), + }, + }, + }, + wantErr: true, + }, + "instance error": { + imds: stubIMDS{ + projectID: "someProject", + zone: "someZone-west3-b", + instanceName: "someInstance", + }, + instanceAPI: stubInstanceAPI{ + instanceErr: someErr, + }, + wantErr: true, + }, + "invalid instance": { + imds: stubIMDS{ + projectID: "someProject", + zone: "someZone-west3-b", + instanceName: "someInstance", + }, + instanceAPI: stubInstanceAPI{ + instance: nil, + }, + wantErr: true, + }, + } + + for name, tc := range testCases { + t.Run(name, func(t *testing.T) { + assert := assert.New(t) + + cloud := &Cloud{ + imds: &tc.imds, + instanceAPI: &tc.instanceAPI, + } + + uid, err := cloud.UID(context.Background()) + if tc.wantErr { + assert.Error(err) + return + } + assert.NoError(err) + assert.Equal(tc.wantUID, uid) + }) + } +} + +func TestSelfGetInstance(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + + cloud := &Cloud{ + imds: &stubIMDS{ + projectID: "someProject", + zone: "someZone-west3-b", + instanceName: "someInstance", + }, + instanceAPI: &stubInstanceAPI{ + instance: &computepb.Instance{ + Name: proto.String("someInstance"), + Zone: proto.String("someZone-west3-b"), + Labels: map[string]string{ + cloud.TagUID: "1234", + cloud.TagRole: role.ControlPlane.String(), + }, + NetworkInterfaces: []*computepb.NetworkInterface{ + { + Name: proto.String("nic0"), + NetworkIP: proto.String("192.0.2.0"), + AliasIpRanges: []*computepb.AliasIpRange{ + { + IpCidrRange: proto.String("192.0.3.0/8"), + }, + }, + Subnetwork: proto.String("projects/someProject/regions/someRegion/subnetworks/someSubnetwork"), + }, + }, + }, + }, + subnetAPI: &stubSubnetAPI{ + subnet: &computepb.Subnetwork{ + SecondaryIpRanges: []*computepb.SubnetworkSecondaryRange{ + { + IpCidrRange: proto.String("198.51.100.0/24"), + }, + }, + }, + }, + } + + self, err := cloud.Self(context.Background()) + require.NoError(err) + + instance, err := cloud.GetInstance(context.Background(), self.ProviderID) + require.NoError(err) + + assert.Equal(self, instance) +} + +type stubForwardingRulesAPI struct { + iterator forwardingRuleIterator +} + +func (s *stubForwardingRulesAPI) List( + ctx context.Context, req *computepb.ListGlobalForwardingRulesRequest, opts ...gax.CallOption, +) forwardingRuleIterator { + return s.iterator +} + +func (s *stubForwardingRulesAPI) Close() error { return nil } + +type stubForwardingRulesIterator struct { + ctr int + forwardingRules []*computepb.ForwardingRule + err error +} + +func (s *stubForwardingRulesIterator) Next() (*computepb.ForwardingRule, error) { + if s.err != nil { + return nil, s.err + } + if s.ctr >= len(s.forwardingRules) { + return nil, iterator.Done + } + s.ctr++ + return s.forwardingRules[s.ctr-1], nil +} + +type stubIMDS struct { + instanceID string + projectID string + zone string + instanceName string + instanceIDErr error + projectIDErr error + zoneErr error + instanceNameErr error +} + +func (s *stubIMDS) InstanceID() (string, error) { return s.instanceID, s.instanceIDErr } + +func (s *stubIMDS) ProjectID() (string, error) { return s.projectID, s.projectIDErr } + +func (s *stubIMDS) Zone() (string, error) { return s.zone, s.zoneErr } + +func (s *stubIMDS) InstanceName() (string, error) { return s.instanceName, s.instanceNameErr } + +type stubInstanceAPI struct { + instance *computepb.Instance + instanceErr error + iterator *stubInstanceIterator +} + +func (s *stubInstanceAPI) Get( + ctx context.Context, req *computepb.GetInstanceRequest, opts ...gax.CallOption, +) (*computepb.Instance, error) { + return s.instance, s.instanceErr +} + +func (s *stubInstanceAPI) List( + ctx context.Context, req *computepb.ListInstancesRequest, opts ...gax.CallOption, +) instanceIterator { + return s.iterator +} + +func (s *stubInstanceAPI) Close() error { return nil } + +type stubInstanceIterator struct { + ctr int + instances []*computepb.Instance + err error +} + +func (s *stubInstanceIterator) Next() (*computepb.Instance, error) { + if s.err != nil { + return nil, s.err + } + if s.ctr >= len(s.instances) { + return nil, iterator.Done + } + s.ctr++ + return s.instances[s.ctr-1], nil +} + +type stubSubnetAPI struct { + subnet *computepb.Subnetwork + subnetErr error +} + +func (s *stubSubnetAPI) Get( + ctx context.Context, req *computepb.GetSubnetworkRequest, opts ...gax.CallOption, +) (*computepb.Subnetwork, error) { + return s.subnet, s.subnetErr +} +func (s *stubSubnetAPI) Close() error { return nil } diff --git a/internal/cloud/gcp/cloudnodemanager.go b/internal/cloud/gcp/cloudnodemanager.go deleted file mode 100644 index bb4b55f8f..000000000 --- a/internal/cloud/gcp/cloudnodemanager.go +++ /dev/null @@ -1,35 +0,0 @@ -/* -Copyright (c) Edgeless Systems GmbH - -SPDX-License-Identifier: AGPL-3.0-only -*/ - -package gcp - -import "github.com/edgelesssys/constellation/v2/internal/versions" - -// CloudNodeManager holds the GCP cloud-node-manager configuration. -type CloudNodeManager struct{} - -// Image returns the container image used to provide cloud-node-manager for the cloud-provider. -// Not used on GCP. -func (c *CloudNodeManager) Image(k8sVersion versions.ValidK8sVersion) (string, error) { - return "", nil -} - -// Path returns the path used by cloud-node-manager executable within the container image. -// Not used on GCP. -func (c *CloudNodeManager) Path() string { - return "" -} - -// ExtraArgs returns a list of arguments to append to the cloud-node-manager command. -// Not used on GCP. -func (c *CloudNodeManager) ExtraArgs() []string { - return []string{} -} - -// Supported is used to determine if cloud node manager is implemented for this cloud provider. -func (c *CloudNodeManager) Supported() bool { - return false -} diff --git a/internal/cloud/gcp/cloudnodemanager_test.go b/internal/cloud/gcp/cloudnodemanager_test.go deleted file mode 100644 index cda3a8aec..000000000 --- a/internal/cloud/gcp/cloudnodemanager_test.go +++ /dev/null @@ -1,23 +0,0 @@ -/* -Copyright (c) Edgeless Systems GmbH - -SPDX-License-Identifier: AGPL-3.0-only -*/ - -package gcp - -import ( - "testing" - - "github.com/stretchr/testify/assert" -) - -func TestTrivialCNMFunctions(t *testing.T) { - assert := assert.New(t) - cloud := CloudNodeManager{} - - assert.Empty(cloud.Image("")) - assert.Empty(cloud.Path()) - assert.Empty(cloud.ExtraArgs()) - assert.False(cloud.Supported()) -} diff --git a/internal/cloud/gcp/api.go b/internal/cloud/gcp/interface.go similarity index 50% rename from internal/cloud/gcp/api.go rename to internal/cloud/gcp/interface.go index aa729bfb3..d9cfe8adb 100644 --- a/internal/cloud/gcp/api.go +++ b/internal/cloud/gcp/interface.go @@ -9,50 +9,29 @@ package gcp import ( "context" - compute "cloud.google.com/go/compute/apiv1" - "github.com/googleapis/gax-go/v2" computepb "google.golang.org/genproto/googleapis/cloud/compute/v1" ) -type instanceAPI interface { - Get(ctx context.Context, req *computepb.GetInstanceRequest, opts ...gax.CallOption) (*computepb.Instance, error) - List(ctx context.Context, req *computepb.ListInstancesRequest, opts ...gax.CallOption) InstanceIterator - SetMetadata(ctx context.Context, req *computepb.SetMetadataInstanceRequest, opts ...gax.CallOption) (*compute.Operation, error) - Close() error -} - -type subnetworkAPI interface { - List(ctx context.Context, req *computepb.ListSubnetworksRequest, opts ...gax.CallOption) SubnetworkIterator - Get(ctx context.Context, req *computepb.GetSubnetworkRequest, opts ...gax.CallOption) (*computepb.Subnetwork, error) - Close() error -} - type forwardingRulesAPI interface { - List(ctx context.Context, req *computepb.ListGlobalForwardingRulesRequest, opts ...gax.CallOption) ForwardingRuleIterator + List(ctx context.Context, req *computepb.ListGlobalForwardingRulesRequest, opts ...gax.CallOption) forwardingRuleIterator Close() error } -type metadataAPI interface { - InstanceAttributeValue(attr string) (string, error) +type imdsAPI interface { InstanceID() (string, error) ProjectID() (string, error) Zone() (string, error) InstanceName() (string, error) } -type Operation interface { - Proto() *computepb.Operation +type instanceAPI interface { + Get(ctx context.Context, req *computepb.GetInstanceRequest, opts ...gax.CallOption) (*computepb.Instance, error) + List(ctx context.Context, req *computepb.ListInstancesRequest, opts ...gax.CallOption) instanceIterator + Close() error } -type InstanceIterator interface { - Next() (*computepb.Instance, error) -} - -type SubnetworkIterator interface { - Next() (*computepb.Subnetwork, error) -} - -type ForwardingRuleIterator interface { - Next() (*computepb.ForwardingRule, error) +type subnetAPI interface { + Get(ctx context.Context, req *computepb.GetSubnetworkRequest, opts ...gax.CallOption) (*computepb.Subnetwork, error) + Close() error } diff --git a/internal/cloud/gcp/logger.go b/internal/cloud/gcp/logger.go index 90c27bd3b..50e66cf50 100644 --- a/internal/cloud/gcp/logger.go +++ b/internal/cloud/gcp/logger.go @@ -8,10 +8,11 @@ package gcp import ( "context" + "fmt" "log" + "cloud.google.com/go/compute/metadata" "cloud.google.com/go/logging" - "github.com/edgelesssys/constellation/v2/internal/gcpshared" ) type Logger struct { @@ -21,11 +22,12 @@ type Logger struct { // NewLogger creates a new Cloud Logger for GCP. // https://cloud.google.com/logging/docs/setup/go -func NewLogger(ctx context.Context, providerID string, logName string) (*Logger, error) { - projectID, _, _, err := gcpshared.SplitProviderID(providerID) +func NewLogger(ctx context.Context, logName string) (*Logger, error) { + projectID, err := metadata.NewClient(nil).ProjectID() if err != nil { - return nil, err + return nil, fmt.Errorf("retrieving project ID from imds: %w", err) } + client, err := logging.NewClient(ctx, projectID) if err != nil { return nil, err diff --git a/internal/cloud/gcp/metadata.go b/internal/cloud/gcp/metadata.go deleted file mode 100644 index bb726b9f1..000000000 --- a/internal/cloud/gcp/metadata.go +++ /dev/null @@ -1,131 +0,0 @@ -/* -Copyright (c) Edgeless Systems GmbH - -SPDX-License-Identifier: AGPL-3.0-only -*/ - -package gcp - -import ( - "context" - "fmt" - - "github.com/edgelesssys/constellation/v2/internal/cloud/metadata" - "github.com/edgelesssys/constellation/v2/internal/gcpshared" -) - -// API handles all GCP API requests. -type API interface { - // UID retrieves the current instances uid. - UID(context.Context) (string, error) - // RetrieveInstances retrieves a list of all accessible GCP instances with their metadata. - RetrieveInstances(ctx context.Context, project, zone string) ([]metadata.InstanceMetadata, error) - // RetrieveInstances retrieves a single GCP instances with its metadata. - RetrieveInstance(ctx context.Context, project, zone, instanceName string) (metadata.InstanceMetadata, error) - // RetrieveInstanceMetadata retrieves the GCP instance metadata of the current instance. - RetrieveInstanceMetadata(attr string) (string, error) - // RetrieveProjectID retrieves the GCP projectID containing the current instance. - RetrieveProjectID() (string, error) - // RetrieveZone retrieves the GCP zone containing the current instance. - RetrieveZone() (string, error) - // RetrieveInstanceName retrieves the instance name of the current instance. - RetrieveInstanceName() (string, error) - // RetrieveSubnetworkAliasCIDR retrieves the subnetwork CIDR of the current instance. - RetrieveSubnetworkAliasCIDR(ctx context.Context, project, zone, instanceName string) (string, error) - // RetrieveLoadBalancerEndpoint retrieves the load balancer endpoint of the current instance. - RetrieveLoadBalancerEndpoint(ctx context.Context, project string) (string, error) - // SetInstanceMetadata sets metadata key: value of the instance specified by project, zone and instanceName. - SetInstanceMetadata(ctx context.Context, project, zone, instanceName, key, value string) error - // UnsetInstanceMetadata removes a metadata key-value pair of the instance specified by project, zone and instanceName. - UnsetInstanceMetadata(ctx context.Context, project, zone, instanceName, key string) error -} - -// Metadata implements core.ProviderMetadata interface. -type Metadata struct { - api API -} - -// New creates a new Provider with real API and FS. -func New(api API) *Metadata { - return &Metadata{ - api: api, - } -} - -// List retrieves all instances belonging to the current constellation. -func (m *Metadata) List(ctx context.Context) ([]metadata.InstanceMetadata, error) { - project, err := m.api.RetrieveProjectID() - if err != nil { - return nil, err - } - zone, err := m.api.RetrieveZone() - if err != nil { - return nil, err - } - instances, err := m.api.RetrieveInstances(ctx, project, zone) - if err != nil { - return nil, fmt.Errorf("retrieving instances list from GCP api: %w", err) - } - return instances, nil -} - -// Self retrieves the current instance. -func (m *Metadata) Self(ctx context.Context) (metadata.InstanceMetadata, error) { - project, err := m.api.RetrieveProjectID() - if err != nil { - return metadata.InstanceMetadata{}, err - } - zone, err := m.api.RetrieveZone() - if err != nil { - return metadata.InstanceMetadata{}, err - } - instanceName, err := m.api.RetrieveInstanceName() - if err != nil { - return metadata.InstanceMetadata{}, err - } - subnetCIDR, err := m.api.RetrieveSubnetworkAliasCIDR(ctx, project, zone, instanceName) - if err != nil { - return metadata.InstanceMetadata{}, err - } - - instance, err := m.api.RetrieveInstance(ctx, project, zone, instanceName) - if err != nil { - return metadata.InstanceMetadata{}, err - } - instance.SecondaryIPRange = subnetCIDR - - return instance, nil -} - -// GetInstance retrieves an instance using its providerID. -func (m *Metadata) GetInstance(ctx context.Context, providerID string) (metadata.InstanceMetadata, error) { - project, zone, instanceName, err := gcpshared.SplitProviderID(providerID) - if err != nil { - return metadata.InstanceMetadata{}, fmt.Errorf("invalid providerID: %w", err) - } - return m.api.RetrieveInstance(ctx, project, zone, instanceName) -} - -// SupportsLoadBalancer returns true if the cloud provider supports load balancers. -func (m *Metadata) SupportsLoadBalancer() bool { - return true -} - -// GetLoadBalancerEndpoint returns the endpoint of the load balancer. -func (m *Metadata) GetLoadBalancerEndpoint(ctx context.Context) (string, error) { - project, err := m.api.RetrieveProjectID() - if err != nil { - return "", err - } - return m.api.RetrieveLoadBalancerEndpoint(ctx, project) -} - -// UID retrieves the UID of the constellation. -func (m *Metadata) UID(ctx context.Context) (string, error) { - return m.api.UID(ctx) -} - -// Supported is used to determine if metadata API is implemented for this cloud provider. -func (m *Metadata) Supported() bool { - return true -} diff --git a/internal/cloud/gcp/metadata_test.go b/internal/cloud/gcp/metadata_test.go deleted file mode 100644 index 21fbe2f42..000000000 --- a/internal/cloud/gcp/metadata_test.go +++ /dev/null @@ -1,326 +0,0 @@ -/* -Copyright (c) Edgeless Systems GmbH - -SPDX-License-Identifier: AGPL-3.0-only -*/ - -package gcp - -import ( - "context" - "errors" - "testing" - - "github.com/edgelesssys/constellation/v2/internal/cloud" - "github.com/edgelesssys/constellation/v2/internal/cloud/metadata" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestList(t *testing.T) { - err := errors.New("some err") - uid := "1234" - instancesGenerator := func() *[]metadata.InstanceMetadata { - return &[]metadata.InstanceMetadata{ - { - Name: "someInstance", - ProviderID: "gce://someProject/someZone/someInstance", - VPCIP: "192.0.2.0", - }, - } - } - - testCases := map[string]struct { - client stubGCPClient - instancesGenerator func() *[]metadata.InstanceMetadata - instancesMutator func(*[]metadata.InstanceMetadata) - wantErr bool - wantInstances []metadata.InstanceMetadata - }{ - "retrieve works": { - client: stubGCPClient{ - projectID: "someProjectID", - zone: "someZone", - retrieveInstanceMetadaValues: map[string]string{ - cloud.TagUID: uid, - }, - }, - instancesGenerator: instancesGenerator, - wantInstances: []metadata.InstanceMetadata{ - { - Name: "someInstance", - ProviderID: "gce://someProject/someZone/someInstance", - VPCIP: "192.0.2.0", - }, - }, - }, - "retrieve error is detected": { - client: stubGCPClient{ - projectID: "someProjectID", - zone: "someZone", - retrieveInstanceMetadaValues: map[string]string{ - cloud.TagUID: uid, - }, - retrieveInstancesErr: err, - }, - instancesGenerator: instancesGenerator, - wantErr: true, - }, - "project metadata retrieval error is detected": { - client: stubGCPClient{ - retrieveProjectIDErr: err, - }, - instancesGenerator: instancesGenerator, - wantErr: true, - }, - "zone retrieval error is detected": { - client: stubGCPClient{ - retrieveZoneErr: err, - }, - instancesGenerator: instancesGenerator, - wantErr: true, - }, - } - - for name, tc := range testCases { - t.Run(name, func(t *testing.T) { - assert := assert.New(t) - require := require.New(t) - - tc.client.retrieveInstancesValues = *tc.instancesGenerator() - if tc.instancesMutator != nil { - tc.instancesMutator(&tc.client.retrieveInstancesValues) - } - metadata := New(&tc.client) - instances, err := metadata.List(context.Background()) - - if tc.wantErr { - assert.Error(err) - return - } - require.NoError(err) - assert.ElementsMatch(tc.wantInstances, instances) - }) - } -} - -func TestSelf(t *testing.T) { - err := errors.New("some err") - uid := "1234" - - testCases := map[string]struct { - client stubGCPClient - wantErr bool - wantInstance metadata.InstanceMetadata - }{ - "retrieve works": { - client: stubGCPClient{ - projectID: "someProjectID", - zone: "someZone", - retrieveInstanceValue: metadata.InstanceMetadata{ - Name: "someInstance", - ProviderID: "gce://someProject/someZone/someInstance", - VPCIP: "192.0.2.0", - }, - }, - wantInstance: metadata.InstanceMetadata{ - Name: "someInstance", - ProviderID: "gce://someProject/someZone/someInstance", - VPCIP: "192.0.2.0", - }, - }, - "retrieve error is detected": { - client: stubGCPClient{ - projectID: "someProjectID", - zone: "someZone", - retrieveInstanceMetadaValues: map[string]string{ - cloud.TagUID: uid, - }, - retrieveInstanceErr: err, - }, - wantErr: true, - }, - "project id retrieval error is detected": { - client: stubGCPClient{ - retrieveProjectIDErr: err, - }, - wantErr: true, - }, - "zone retrieval error is detected": { - client: stubGCPClient{ - retrieveZoneErr: err, - }, - wantErr: true, - }, - "instance name retrieval error is detected": { - client: stubGCPClient{ - retrieveInstanceNameErr: err, - }, - wantErr: true, - }, - } - - for name, tc := range testCases { - t.Run(name, func(t *testing.T) { - assert := assert.New(t) - require := require.New(t) - - cloud := New(&tc.client) - instance, err := cloud.Self(context.Background()) - - if tc.wantErr { - assert.Error(err) - return - } - require.NoError(err) - assert.Equal(tc.wantInstance, instance) - }) - } -} - -func TestGetInstance(t *testing.T) { - err := errors.New("some err") - - testCases := map[string]struct { - providerID string - client stubGCPClient - wantErr bool - wantInstance metadata.InstanceMetadata - }{ - "retrieve works": { - providerID: "gce://someProject/someZone/someInstance", - client: stubGCPClient{ - retrieveInstanceValue: metadata.InstanceMetadata{ - Name: "someInstance", - ProviderID: "gce://someProject/someZone/someInstance", - VPCIP: "192.0.2.0", - }, - }, - wantInstance: metadata.InstanceMetadata{ - Name: "someInstance", - ProviderID: "gce://someProject/someZone/someInstance", - VPCIP: "192.0.2.0", - }, - }, - "retrieve error is detected": { - providerID: "gce://someProject/someZone/someInstance", - client: stubGCPClient{ - retrieveInstanceErr: err, - }, - wantErr: true, - }, - "malformed providerID with too many fields is detected": { - providerID: "gce://someProject/someZone/someInstance/tooMany/fields", - wantErr: true, - }, - "malformed providerID with too few fields is detected": { - providerID: "gce://someProject", - wantErr: true, - }, - } - - for name, tc := range testCases { - t.Run(name, func(t *testing.T) { - assert := assert.New(t) - require := require.New(t) - - cloud := New(&tc.client) - instance, err := cloud.GetInstance(context.Background(), tc.providerID) - - if tc.wantErr { - assert.Error(err) - return - } - require.NoError(err) - assert.Equal(tc.wantInstance, instance) - }) - } -} - -type stubGCPClient struct { - retrieveUIDValue string - retrieveUIDErr error - retrieveInstanceValue metadata.InstanceMetadata - retrieveInstanceErr error - retrieveInstancesValues []metadata.InstanceMetadata - retrieveInstancesErr error - retrieveInstanceMetadaValues map[string]string - retrieveInstanceMetadataErr error - retrieveSubnetworkAliasErr error - projectID string - zone string - instanceName string - loadBalancerIP string - retrieveProjectIDErr error - retrieveZoneErr error - retrieveInstanceNameErr error - setInstanceMetadataErr error - unsetInstanceMetadataErr error - retrieveLoadBalancerErr error - - instanceMetadataProjects []string - instanceMetadataZones []string - instanceMetadataInstanceNames []string - instanceMetadataKeys []string - instanceMetadataValues []string - - unsetMetadataProjects []string - unsetMetadataZones []string - unsetMetadataInstanceNames []string - unsetMetadataKeys []string -} - -func (s *stubGCPClient) RetrieveInstances(ctx context.Context, project, zone string) ([]metadata.InstanceMetadata, error) { - return s.retrieveInstancesValues, s.retrieveInstancesErr -} - -func (s *stubGCPClient) RetrieveInstance(ctx context.Context, project, zone string, instanceName string) (metadata.InstanceMetadata, error) { - return s.retrieveInstanceValue, s.retrieveInstanceErr -} - -func (s *stubGCPClient) RetrieveInstanceMetadata(attr string) (string, error) { - return s.retrieveInstanceMetadaValues[attr], s.retrieveInstanceMetadataErr -} - -func (s *stubGCPClient) RetrieveProjectID() (string, error) { - return s.projectID, s.retrieveProjectIDErr -} - -func (s *stubGCPClient) RetrieveZone() (string, error) { - return s.zone, s.retrieveZoneErr -} - -func (s *stubGCPClient) RetrieveInstanceName() (string, error) { - return s.instanceName, s.retrieveInstanceNameErr -} - -func (s *stubGCPClient) RetrieveLoadBalancerEndpoint(ctx context.Context, project string) (string, error) { - return s.loadBalancerIP, s.retrieveLoadBalancerErr -} - -func (s *stubGCPClient) UID(context.Context) (string, error) { - return s.retrieveUIDValue, s.retrieveUIDErr -} - -func (s *stubGCPClient) SetInstanceMetadata(ctx context.Context, project, zone, instanceName, key, value string) error { - s.instanceMetadataProjects = append(s.instanceMetadataProjects, project) - s.instanceMetadataZones = append(s.instanceMetadataZones, zone) - s.instanceMetadataInstanceNames = append(s.instanceMetadataInstanceNames, instanceName) - s.instanceMetadataKeys = append(s.instanceMetadataKeys, key) - s.instanceMetadataValues = append(s.instanceMetadataValues, value) - - return s.setInstanceMetadataErr -} - -func (s *stubGCPClient) UnsetInstanceMetadata(ctx context.Context, project, zone, instanceName, key string) error { - s.unsetMetadataProjects = append(s.unsetMetadataProjects, project) - s.unsetMetadataZones = append(s.unsetMetadataZones, zone) - s.unsetMetadataInstanceNames = append(s.unsetMetadataInstanceNames, instanceName) - s.unsetMetadataKeys = append(s.unsetMetadataKeys, key) - - return s.unsetInstanceMetadataErr -} - -func (s *stubGCPClient) RetrieveSubnetworkAliasCIDR(ctx context.Context, project, zone, instanceName string) (string, error) { - return "", s.retrieveSubnetworkAliasErr -} diff --git a/internal/cloud/gcp/wrappers.go b/internal/cloud/gcp/wrappers.go index baf32ea4b..384849f65 100644 --- a/internal/cloud/gcp/wrappers.go +++ b/internal/cloud/gcp/wrappers.go @@ -10,43 +10,16 @@ import ( "context" compute "cloud.google.com/go/compute/apiv1" - "cloud.google.com/go/compute/metadata" "github.com/googleapis/gax-go/v2" computepb "google.golang.org/genproto/googleapis/cloud/compute/v1" ) -type instanceClient struct { - *compute.InstancesClient +type forwardingRuleIterator interface { + Next() (*computepb.ForwardingRule, error) } -func (c *instanceClient) Close() error { - return c.InstancesClient.Close() -} - -func (c *instanceClient) List(ctx context.Context, req *computepb.ListInstancesRequest, - opts ...gax.CallOption, -) InstanceIterator { - return c.InstancesClient.List(ctx, req) -} - -type subnetworkClient struct { - *compute.SubnetworksClient -} - -func (c *subnetworkClient) Close() error { - return c.SubnetworksClient.Close() -} - -func (c *subnetworkClient) List(ctx context.Context, req *computepb.ListSubnetworksRequest, - opts ...gax.CallOption, -) SubnetworkIterator { - return c.SubnetworksClient.List(ctx, req) -} - -func (c *subnetworkClient) Get(ctx context.Context, req *computepb.GetSubnetworkRequest, - opts ...gax.CallOption, -) (*computepb.Subnetwork, error) { - return c.SubnetworksClient.Get(ctx, req) +type instanceIterator interface { + Next() (*computepb.Instance, error) } type forwardingRulesClient struct { @@ -59,32 +32,20 @@ func (c *forwardingRulesClient) Close() error { func (c *forwardingRulesClient) List(ctx context.Context, req *computepb.ListGlobalForwardingRulesRequest, opts ...gax.CallOption, -) ForwardingRuleIterator { +) forwardingRuleIterator { return c.GlobalForwardingRulesClient.List(ctx, req) } -type metadataClient struct{} - -func (c *metadataClient) InstanceAttributeValue(attr string) (string, error) { - return metadata.InstanceAttributeValue(attr) +type instanceClient struct { + *compute.InstancesClient } -func (c *metadataClient) InstanceID() (string, error) { - return metadata.InstanceID() +func (c *instanceClient) Close() error { + return c.InstancesClient.Close() } -func (c *metadataClient) ProjectID() (string, error) { - return metadata.ProjectID() -} - -func (c *metadataClient) Zone() (string, error) { - return metadata.Zone() -} - -func (c *metadataClient) InstanceName() (string, error) { - return metadata.InstanceName() -} - -func (c *metadataClient) ProjectAttributeValue(attr string) (string, error) { - return metadata.ProjectAttributeValue(attr) +func (c *instanceClient) List(ctx context.Context, req *computepb.ListInstancesRequest, + opts ...gax.CallOption, +) instanceIterator { + return c.InstancesClient.List(ctx, req) } diff --git a/internal/cloud/gcp/writer.go b/internal/cloud/gcp/writer.go deleted file mode 100644 index 095f8b96c..000000000 --- a/internal/cloud/gcp/writer.go +++ /dev/null @@ -1,26 +0,0 @@ -/* -Copyright (c) Edgeless Systems GmbH - -SPDX-License-Identifier: AGPL-3.0-only -*/ - -package gcp - -import ( - "fmt" - - "github.com/spf13/afero" -) - -// Writer implements ConfigWriter. -type Writer struct { - fs afero.Afero -} - -// WriteGCEConf persists the GCE config on disk. -func (w *Writer) WriteGCEConf(config string) error { - if err := w.fs.WriteFile("/etc/gce.conf", []byte(config), 0o644); err != nil { - return fmt.Errorf("writing gce config: %w", err) - } - return nil -} diff --git a/internal/cloud/gcp/writer_test.go b/internal/cloud/gcp/writer_test.go deleted file mode 100644 index 9608b54b2..000000000 --- a/internal/cloud/gcp/writer_test.go +++ /dev/null @@ -1,60 +0,0 @@ -/* -Copyright (c) Edgeless Systems GmbH - -SPDX-License-Identifier: AGPL-3.0-only -*/ - -package gcp - -import ( - "testing" - - "github.com/spf13/afero" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestWriteGCEConf(t *testing.T) { - config := "someConfig" - - testCases := map[string]struct { - fs afero.Afero - wantValue string - wantErr bool - }{ - "write works": { - fs: afero.Afero{ - Fs: afero.NewMemMapFs(), - }, - wantValue: config, - wantErr: false, - }, - "write fails": { - fs: afero.Afero{ - Fs: afero.NewReadOnlyFs(afero.NewMemMapFs()), - }, - wantErr: true, - }, - } - - for name, tc := range testCases { - t.Run(name, func(t *testing.T) { - assert := assert.New(t) - require := require.New(t) - - writer := Writer{ - fs: tc.fs, - } - err := writer.WriteGCEConf(config) - - if tc.wantErr { - assert.Error(err) - return - } - require.NoError(err) - value, err := tc.fs.ReadFile("/etc/gce.conf") - assert.NoError(err) - assert.Equal(tc.wantValue, string(value)) - }) - } -} diff --git a/internal/cloud/gcpshared/doc.go b/internal/cloud/gcpshared/doc.go new file mode 100644 index 000000000..93b4a41fa --- /dev/null +++ b/internal/cloud/gcpshared/doc.go @@ -0,0 +1,15 @@ +/* +Copyright (c) Edgeless Systems GmbH +SPDX-License-Identifier: AGPL-3.0-only +*/ + +/* +Package gcpshared contains code to parse and define data types +relevant for Google Cloud Platform. + +This package is intended to have a minimal size and surface. If you +have GCP related code that is not shared by multiple applications, +or if the code interacts with the GCP API, please keep the code in +the application's internal package or add it to the GCP cloud package. +*/ +package gcpshared diff --git a/internal/gcpshared/metadata.go b/internal/cloud/gcpshared/providerid.go similarity index 81% rename from internal/gcpshared/metadata.go rename to internal/cloud/gcpshared/providerid.go index 0fcb00e3c..70c26334c 100644 --- a/internal/gcpshared/metadata.go +++ b/internal/cloud/gcpshared/providerid.go @@ -13,8 +13,8 @@ import ( var providerIDRegex = regexp.MustCompile(`^gce://([^/]+)/([^/]+)/([^/]+)$`) -// SplitProviderID splits a provider's id into core components. -// A providerID is build after the schema 'gce:////' +// SplitProviderID splits a k8s provider ID for GCP instances into its core components. +// A provider ID is build after the schema 'gce:////' func SplitProviderID(providerID string) (project, zone, instance string, err error) { matches := providerIDRegex.FindStringSubmatch(providerID) if len(matches) != 4 { diff --git a/internal/gcpshared/metadata_test.go b/internal/cloud/gcpshared/providerid_test.go similarity index 95% rename from internal/gcpshared/metadata_test.go rename to internal/cloud/gcpshared/providerid_test.go index ca9249027..61fbdb2f5 100644 --- a/internal/gcpshared/metadata_test.go +++ b/internal/cloud/gcpshared/providerid_test.go @@ -10,13 +10,8 @@ import ( "testing" "github.com/stretchr/testify/assert" - "go.uber.org/goleak" ) -func TestMain(m *testing.M) { - goleak.VerifyTestMain(m) -} - func TestSplitProviderID(t *testing.T) { testCases := map[string]struct { providerID string diff --git a/internal/gcpshared/serviceaccountkey.go b/internal/cloud/gcpshared/serviceaccountkey.go similarity index 100% rename from internal/gcpshared/serviceaccountkey.go rename to internal/cloud/gcpshared/serviceaccountkey.go diff --git a/internal/gcpshared/serviceaccountkey_test.go b/internal/cloud/gcpshared/serviceaccountkey_test.go similarity index 100% rename from internal/gcpshared/serviceaccountkey_test.go rename to internal/cloud/gcpshared/serviceaccountkey_test.go diff --git a/internal/cloud/metadata/metadata.go b/internal/cloud/metadata/metadata.go index 4a18b27fc..dacad4836 100644 --- a/internal/cloud/metadata/metadata.go +++ b/internal/cloud/metadata/metadata.go @@ -23,11 +23,9 @@ type InstanceMetadata struct { Role role.Role // VPCIP is the primary IP address of the instance in the VPC. VPCIP string - // PublicIP is the primary public IP of the instance, if available, empty string otherwise. - PublicIP string // SSHKeys maps usernames to ssh public keys. + // TODO: remove everywhere. SSHKeys map[string][]string - // SecondaryIPRange is the VPC wide CIDR from which subnets are attached to VMs as AliasIPRanges. // May be empty on certain CSPs. SecondaryIPRange string diff --git a/internal/cloud/qemu/ccm.go b/internal/cloud/qemu/ccm.go deleted file mode 100644 index 6763a674d..000000000 --- a/internal/cloud/qemu/ccm.go +++ /dev/null @@ -1,79 +0,0 @@ -/* -Copyright (c) Edgeless Systems GmbH - -SPDX-License-Identifier: AGPL-3.0-only -*/ - -package qemu - -import ( - "context" - - "github.com/edgelesssys/constellation/v2/internal/cloud/metadata" - "github.com/edgelesssys/constellation/v2/internal/kubernetes" - "github.com/edgelesssys/constellation/v2/internal/versions" - k8s "k8s.io/api/core/v1" -) - -// CloudControllerManager holds the QEMU cloud-controller-manager configuration. -type CloudControllerManager struct{} - -// Image returns the container image used to provide cloud-controller-manager for the cloud-provider. -func (c CloudControllerManager) Image(k8sVersion versions.ValidK8sVersion) (string, error) { - return "", nil -} - -// Path returns the path used by cloud-controller-manager executable within the container image. -func (c CloudControllerManager) Path() string { - return "/qemu-cloud-controller-manager" -} - -// Name returns the cloud-provider name as used by k8s cloud-controller-manager (k8s.gcr.io/cloud-controller-manager). -func (c CloudControllerManager) Name() string { - return "qemu" -} - -// ExtraArgs returns a list of arguments to append to the cloud-controller-manager command. -func (c CloudControllerManager) ExtraArgs() []string { - return []string{} -} - -// ConfigMaps returns a list of ConfigMaps to deploy together with the k8s cloud-controller-manager -// Reference: https://kubernetes.io/docs/concepts/configuration/configmap/ . -func (c CloudControllerManager) ConfigMaps() (kubernetes.ConfigMaps, error) { - return kubernetes.ConfigMaps{}, nil -} - -// Secrets returns a list of secrets to deploy together with the k8s cloud-controller-manager. -// Reference: https://kubernetes.io/docs/concepts/configuration/secret/ . -func (c CloudControllerManager) Secrets(ctx context.Context, providerID, cloudServiceAccountURI string) (kubernetes.Secrets, error) { - return kubernetes.Secrets{}, nil -} - -// Volumes returns a list of volumes to deploy together with the k8s cloud-controller-manager. -// Reference: https://kubernetes.io/docs/concepts/storage/volumes/ . -func (c CloudControllerManager) Volumes() []k8s.Volume { - return []k8s.Volume{} -} - -// VolumeMounts a list of of volume mounts to deploy together with the k8s cloud-controller-manager. -func (c CloudControllerManager) VolumeMounts() []k8s.VolumeMount { - return []k8s.VolumeMount{} -} - -// Env returns a list of k8s environment key-value pairs to deploy together with the k8s cloud-controller-manager. -func (c CloudControllerManager) Env() []k8s.EnvVar { - return []k8s.EnvVar{} -} - -// PrepareInstance is called on every instance before deploying the cloud-controller-manager. -// Allows for cloud-provider specific hooks. -func (c CloudControllerManager) PrepareInstance(instance metadata.InstanceMetadata, vpnIP string) error { - // no specific hook required. - return nil -} - -// Supported is used to determine if cloud controller manager is implemented for this cloud provider. -func (c CloudControllerManager) Supported() bool { - return false -} diff --git a/internal/cloud/qemu/cloudnodemanager.go b/internal/cloud/qemu/cloudnodemanager.go deleted file mode 100644 index 6983d2dfc..000000000 --- a/internal/cloud/qemu/cloudnodemanager.go +++ /dev/null @@ -1,35 +0,0 @@ -/* -Copyright (c) Edgeless Systems GmbH - -SPDX-License-Identifier: AGPL-3.0-only -*/ - -package qemu - -import "github.com/edgelesssys/constellation/v2/internal/versions" - -// CloudNodeManager holds the QEMU cloud-node-manager configuration. -type CloudNodeManager struct{} - -// Image returns the container image used to provide cloud-node-manager for the cloud-provider. -// Not used on QEMU. -func (c *CloudNodeManager) Image(k8sVersion versions.ValidK8sVersion) (string, error) { - return "", nil -} - -// Path returns the path used by cloud-node-manager executable within the container image. -// Not used on QEMU. -func (c *CloudNodeManager) Path() string { - return "" -} - -// ExtraArgs returns a list of arguments to append to the cloud-node-manager command. -// Not used on QEMU. -func (c *CloudNodeManager) ExtraArgs() []string { - return []string{} -} - -// Supported is used to determine if cloud node manager is implemented for this cloud provider. -func (c *CloudNodeManager) Supported() bool { - return false -} diff --git a/internal/cloud/qemu/metadata.go b/internal/cloud/qemu/metadata.go index d596d206a..54bce369b 100644 --- a/internal/cloud/qemu/metadata.go +++ b/internal/cloud/qemu/metadata.go @@ -22,11 +22,6 @@ const qemuMetadataEndpoint = "10.42.0.1:8080" // Metadata implements core.ProviderMetadata interface for QEMU. type Metadata struct{} -// Supported is used to determine if metadata API is implemented for this cloud provider. -func (m *Metadata) Supported() bool { - return true -} - // List retrieves all instances belonging to the current constellation. func (m *Metadata) List(ctx context.Context) ([]metadata.InstanceMetadata, error) { instancesRaw, err := m.retrieveMetadata(ctx, "/peers") diff --git a/internal/gcpshared/doc.go b/internal/gcpshared/doc.go deleted file mode 100644 index 9fd1685ea..000000000 --- a/internal/gcpshared/doc.go +++ /dev/null @@ -1,16 +0,0 @@ -/* -Copyright (c) Edgeless Systems GmbH - -SPDX-License-Identifier: AGPL-3.0-only -*/ - -package gcpshared - -/* -Package gcpshared contains code that is related to Google Cloud Platform -and is used by multiple microservices. - -This package is intended to have a minimal size and surface. If you -have GCP related code that is not shared by multiple microservices, -please keep the code in the microservice's internal package. -*/ diff --git a/joinservice/cmd/main.go b/joinservice/cmd/main.go index 05f6d6873..d07b4ba9b 100644 --- a/joinservice/cmd/main.go +++ b/joinservice/cmd/main.go @@ -130,11 +130,12 @@ func getVPCIP(ctx context.Context, provider string) (string, error) { return "", err } case cloudprovider.GCP: - gcpClient, err := gcpcloud.NewClient(ctx) + gcpMeta, err := gcpcloud.New(ctx) if err != nil { return "", err } - metadata = gcpcloud.New(gcpClient) + defer gcpMeta.Close() + metadata = gcpMeta case cloudprovider.QEMU: metadata = &qemucloud.Metadata{} default: