Git-Mirrors/constellation
1
0
Fork 0
mirror of https://github.com/edgelesssys/constellation.git synced 2024-12-24 23:19:39 -05:00
Code Issues Packages Projects Releases Wiki Activity

AB#2523 Refactor GCP metadata/cloud API (#387)

Browse Source 
* Refactor GCP metadata/cloud API

* Remove cloud controller manager from metadata package

* Remove PublicIP

* Move shared cloud packages

* Remove dead code

Signed-off-by: Daniel Weiße <dw@edgeless.systems>
This commit is contained in:
Daniel Weiße 2022-11-09 14:43:48 +01:00 committed by GitHub
parent e9fecec0bc
commit c9873f2bfb
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
54 changed files with 1587 additions and 3791 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

25
bootstrapper/cmd/bootstrapper/main.go
View File

@ -105,9 +105,8 @@ func main() {
log.With(zap.Error(err)).Fatalf("Failed to set up cloud logger") log.With(zap.Error(err)).Fatalf("Failed to set up cloud logger")
} }
cloudControllerManager := &awscloud.CloudControllerManager{}
clusterInitJoiner = kubernetes.New( clusterInitJoiner = kubernetes.New(
"aws", k8sapi.NewKubernetesUtil(), &k8sapi.KubdeadmConfiguration{}, kubectl.New(), cloudControllerManager, "aws", k8sapi.NewKubernetesUtil(), &k8sapi.KubdeadmConfiguration{}, kubectl.New(),
metadata, pcrsJSON, helmClient, &kubewaiter.CloudKubeAPIWaiter{}, metadata, pcrsJSON, helmClient, &kubewaiter.CloudKubeAPIWaiter{},
) )
openTPM = vtpm.OpenVTPM openTPM = vtpm.OpenVTPM
@ -121,30 +120,24 @@ func main() {
issuer = initserver.NewIssuerWrapper(gcp.NewIssuer(), vmtype.Unknown, nil) issuer = initserver.NewIssuerWrapper(gcp.NewIssuer(), vmtype.Unknown, nil)
gcpClient, err := gcpcloud.NewClient(ctx) metadata, err := gcpcloud.New(ctx)
if err != nil { if err != nil {
log.With(zap.Error(err)).Fatalf("Failed to create GCP metadata client") log.With(zap.Error(err)).Fatalf("Failed to create GCP metadata client")
} }
metadata := gcpcloud.New(gcpClient) defer metadata.Close()
descr, err := metadata.Self(ctx)
if err != nil { cloudLogger, err = gcpcloud.NewLogger(ctx, "constellation-boot-log")
log.With(zap.Error(err)).Fatalf("Failed to get instance metadata")
}
cloudLogger, err = gcpcloud.NewLogger(ctx, descr.ProviderID, "constellation-boot-log")
if err != nil { if err != nil {
log.With(zap.Error(err)).Fatalf("Failed to set up cloud logger") log.With(zap.Error(err)).Fatalf("Failed to set up cloud logger")
} }
metadataAPI = metadata metadataAPI = metadata
pcrsJSON, err := json.Marshal(pcrs) pcrsJSON, err := json.Marshal(pcrs)
if err != nil { if err != nil {
log.With(zap.Error(err)).Fatalf("Failed to marshal PCRs") log.With(zap.Error(err)).Fatalf("Failed to marshal PCRs")
} }
cloudControllerManager, err := gcpcloud.NewCloudControllerManager(ctx, metadata)
if err != nil {
log.With(zap.Error(err)).Fatalf("Failed to create cloud controller manager")
}
clusterInitJoiner = kubernetes.New( clusterInitJoiner = kubernetes.New(
"gcp", k8sapi.NewKubernetesUtil(), &k8sapi.KubdeadmConfiguration{}, kubectl.New(), cloudControllerManager, "gcp", k8sapi.NewKubernetesUtil(), &k8sapi.KubdeadmConfiguration{}, kubectl.New(),
metadata, pcrsJSON, helmClient, &kubewaiter.CloudKubeAPIWaiter{}, metadata, pcrsJSON, helmClient, &kubewaiter.CloudKubeAPIWaiter{},
) )
openTPM = vtpm.OpenVTPM openTPM = vtpm.OpenVTPM
@ -178,7 +171,7 @@ func main() {
log.With(zap.Error(err)).Fatalf("Failed to marshal PCRs") log.With(zap.Error(err)).Fatalf("Failed to marshal PCRs")
} }
clusterInitJoiner = kubernetes.New( clusterInitJoiner = kubernetes.New(
"azure", k8sapi.NewKubernetesUtil(), &k8sapi.KubdeadmConfiguration{}, kubectl.New(), azurecloud.NewCloudControllerManager(metadata), "azure", k8sapi.NewKubernetesUtil(), &k8sapi.KubdeadmConfiguration{}, kubectl.New(),
metadata, pcrsJSON, helmClient, &kubewaiter.CloudKubeAPIWaiter{}, metadata, pcrsJSON, helmClient, &kubewaiter.CloudKubeAPIWaiter{},
) )
@ -200,7 +193,7 @@ func main() {
log.With(zap.Error(err)).Fatalf("Failed to marshal PCRs") log.With(zap.Error(err)).Fatalf("Failed to marshal PCRs")
} }
clusterInitJoiner = kubernetes.New( clusterInitJoiner = kubernetes.New(
"qemu", k8sapi.NewKubernetesUtil(), &k8sapi.KubdeadmConfiguration{}, kubectl.New(), &qemucloud.CloudControllerManager{}, "qemu", k8sapi.NewKubernetesUtil(), &k8sapi.KubdeadmConfiguration{}, kubectl.New(),
metadata, pcrsJSON, helmClient, &kubewaiter.CloudKubeAPIWaiter{}, metadata, pcrsJSON, helmClient, &kubewaiter.CloudKubeAPIWaiter{},
) )
metadataAPI = metadata metadataAPI = metadata

12
bootstrapper/cmd/bootstrapper/test.go
View File

@ -52,18 +52,6 @@ func (f *providerMetadataFake) Self(ctx context.Context) (metadata.InstanceMetad
}, nil }, nil
} }
func (f *providerMetadataFake) SignalRole(ctx context.Context, role role.Role) error {
return nil
}
func (f *providerMetadataFake) SetVPNIP(ctx context.Context, vpnIP string) error {
return nil
}
func (f *providerMetadataFake) GetLoadBalancerEndpoint(ctx context.Context) (string, error) { func (f *providerMetadataFake) GetLoadBalancerEndpoint(ctx context.Context) (string, error) {
return "", nil return "", nil
} }
func (f *providerMetadataFake) Supported() bool {
return true
}

147
bootstrapper/internal/kubernetes/cloud_provider.go
View File

@ -10,9 +10,6 @@ import (
"context" "context"
"github.com/edgelesssys/constellation/v2/internal/cloud/metadata" "github.com/edgelesssys/constellation/v2/internal/cloud/metadata"
"github.com/edgelesssys/constellation/v2/internal/kubernetes"
"github.com/edgelesssys/constellation/v2/internal/versions"
k8s "k8s.io/api/core/v1"
) )
// ProviderMetadata implementers read/write cloud provider metadata. // ProviderMetadata implementers read/write cloud provider metadata.
@ -27,63 +24,6 @@ type ProviderMetadata interface {
GetLoadBalancerEndpoint(ctx context.Context) (string, error) GetLoadBalancerEndpoint(ctx context.Context) (string, error)
// GetInstance retrieves an instance using its providerID. // GetInstance retrieves an instance using its providerID.
GetInstance(ctx context.Context, providerID string) (metadata.InstanceMetadata, error) GetInstance(ctx context.Context, providerID string) (metadata.InstanceMetadata, error)
// Supported is used to determine if metadata API is implemented for this cloud provider.
Supported() bool
}
// CloudControllerManager implementers provide configuration for the k8s cloud-controller-manager.
type CloudControllerManager interface {
// Image returns the container image used to provide cloud-controller-manager for the cloud-provider.
Image(k8sVersion versions.ValidK8sVersion) (string, error)
// Path returns the path used by cloud-controller-manager executable within the container image.
Path() string
// Name returns the cloud-provider name as used by k8s cloud-controller-manager (k8s.gcr.io/cloud-controller-manager).
Name() string
// ExtraArgs returns a list of arguments to append to the cloud-controller-manager command.
ExtraArgs() []string
// ConfigMaps returns a list of ConfigMaps to deploy together with the k8s cloud-controller-manager
// Reference: https://kubernetes.io/docs/concepts/configuration/configmap/ .
ConfigMaps() (kubernetes.ConfigMaps, error)
// Secrets returns a list of secrets to deploy together with the k8s cloud-controller-manager.
// Reference: https://kubernetes.io/docs/concepts/configuration/secret/ .
Secrets(ctx context.Context, providerID, cloudServiceAccountURI string) (kubernetes.Secrets, error)
// Volumes returns a list of volumes to deploy together with the k8s cloud-controller-manager.
// Reference: https://kubernetes.io/docs/concepts/storage/volumes/ .
Volumes() []k8s.Volume
// VolumeMounts a list of of volume mounts to deploy together with the k8s cloud-controller-manager.
VolumeMounts() []k8s.VolumeMount
// Env returns a list of k8s environment key-value pairs to deploy together with the k8s cloud-controller-manager.
Env() []k8s.EnvVar
// Supported is used to determine if cloud controller manager is implemented for this cloud provider.
Supported() bool
}
// CloudNodeManager implementers provide configuration for the k8s cloud-node-manager.
type CloudNodeManager interface {
// Image returns the container image used to provide cloud-node-manager for the cloud-provider.
Image(k8sVersion versions.ValidK8sVersion) (string, error)
// Path returns the path used by cloud-node-manager executable within the container image.
Path() string
// ExtraArgs returns a list of arguments to append to the cloud-node-manager command.
ExtraArgs() []string
// Supported is used to determine if cloud node manager is implemented for this cloud provider.
Supported() bool
}
// ClusterAutoscaler implementers provide configuration for the k8s cluster-autoscaler.
type ClusterAutoscaler interface {
// Name returns the cloud-provider name as used by k8s cluster-autoscaler.
Name() string
// Secrets returns a list of secrets to deploy together with the k8s cluster-autoscaler.
Secrets(providerID, cloudServiceAccountURI string) (kubernetes.Secrets, error)
// Volumes returns a list of volumes to deploy together with the k8s cluster-autoscaler.
Volumes() []k8s.Volume
// VolumeMounts returns a list of volume mounts to deploy together with the k8s cluster-autoscaler.
VolumeMounts() []k8s.VolumeMount
// Env returns a list of k8s environment key-value pairs to deploy together with the k8s cluster-autoscaler.
Env() []k8s.EnvVar
// Supported is used to determine if cluster autoscaler is implemented for this cloud provider.
Supported() bool
} }
type stubProviderMetadata struct { type stubProviderMetadata struct {
@ -99,9 +39,6 @@ type stubProviderMetadata struct {
GetInstanceErr error GetInstanceErr error
GetInstanceResp metadata.InstanceMetadata GetInstanceResp metadata.InstanceMetadata
SupportedResp bool
SupportsLoadBalancerResp bool
UIDErr error UIDErr error
UIDResp string UIDResp string
} }
@ -122,90 +59,6 @@ func (m *stubProviderMetadata) GetInstance(ctx context.Context, providerID strin
return m.GetInstanceResp, m.GetInstanceErr return m.GetInstanceResp, m.GetInstanceErr
} }
func (m *stubProviderMetadata) Supported() bool {
return m.SupportedResp
}
func (m *stubProviderMetadata) SupportsLoadBalancer() bool {
return m.SupportsLoadBalancerResp
}
func (m *stubProviderMetadata) UID(ctx context.Context) (string, error) { func (m *stubProviderMetadata) UID(ctx context.Context) (string, error) {
return m.UIDResp, m.UIDErr return m.UIDResp, m.UIDErr
} }
type stubCloudControllerManager struct {
SupportedResp bool
}
func (m *stubCloudControllerManager) Image(k8sVersion versions.ValidK8sVersion) (string, error) {
return "stub-image:latest", nil
}
func (m *stubCloudControllerManager) Path() string {
return "/stub-controller-manager"
}
func (m *stubCloudControllerManager) Name() string {
return "stub"
}
func (m *stubCloudControllerManager) ExtraArgs() []string {
return []string{}
}
func (m *stubCloudControllerManager) ConfigMaps() (kubernetes.ConfigMaps, error) {
return []*k8s.ConfigMap{}, nil
}
func (m *stubCloudControllerManager) Secrets(ctx context.Context, instance, cloudServiceAccountURI string) (kubernetes.Secrets, error) {
return []*k8s.Secret{}, nil
}
func (m *stubCloudControllerManager) Volumes() []k8s.Volume {
return []k8s.Volume{}
}
func (m *stubCloudControllerManager) VolumeMounts() []k8s.VolumeMount {
return []k8s.VolumeMount{}
}
func (m *stubCloudControllerManager) Env() []k8s.EnvVar {
return []k8s.EnvVar{}
}
func (m *stubCloudControllerManager) Supported() bool {
return m.SupportedResp
}
type stubClusterAutoscaler struct {
SupportedResp bool
}
func (a *stubClusterAutoscaler) Name() string {
return "stub"
}
// Secrets returns a list of secrets to deploy together with the k8s cluster-autoscaler.
func (a *stubClusterAutoscaler) Secrets(instance, cloudServiceAccountURI string) (kubernetes.Secrets, error) {
return kubernetes.Secrets{}, nil
}
// Volumes returns a list of volumes to deploy together with the k8s cluster-autoscaler.
func (a *stubClusterAutoscaler) Volumes() []k8s.Volume {
return []k8s.Volume{}
}
// VolumeMounts returns a list of volume mounts to deploy together with the k8s cluster-autoscaler.
func (a *stubClusterAutoscaler) VolumeMounts() []k8s.VolumeMount {
return []k8s.VolumeMount{}
}
// Env returns a list of k8s environment key-value pairs to deploy together with the k8s cluster-autoscaler.
func (a *stubClusterAutoscaler) Env() []k8s.EnvVar {
return []k8s.EnvVar{}
}
func (a *stubClusterAutoscaler) Supported() bool {
return a.SupportedResp
}

253
bootstrapper/internal/kubernetes/kubernetes.go
View File

@ -21,12 +21,11 @@ import (
"github.com/edgelesssys/constellation/v2/bootstrapper/internal/kubernetes/k8sapi" "github.com/edgelesssys/constellation/v2/bootstrapper/internal/kubernetes/k8sapi"
"github.com/edgelesssys/constellation/v2/bootstrapper/internal/kubernetes/k8sapi/resources" "github.com/edgelesssys/constellation/v2/bootstrapper/internal/kubernetes/k8sapi/resources"
kubewaiter "github.com/edgelesssys/constellation/v2/bootstrapper/internal/kubernetes/kubeWaiter" kubewaiter "github.com/edgelesssys/constellation/v2/bootstrapper/internal/kubernetes/kubeWaiter"
"github.com/edgelesssys/constellation/v2/internal/azureshared" "github.com/edgelesssys/constellation/v2/internal/cloud/azureshared"
"github.com/edgelesssys/constellation/v2/internal/cloud/cloudprovider" "github.com/edgelesssys/constellation/v2/internal/cloud/cloudprovider"
"github.com/edgelesssys/constellation/v2/internal/cloud/metadata" "github.com/edgelesssys/constellation/v2/internal/cloud/gcpshared"
"github.com/edgelesssys/constellation/v2/internal/constants" "github.com/edgelesssys/constellation/v2/internal/constants"
"github.com/edgelesssys/constellation/v2/internal/deploy/helm" "github.com/edgelesssys/constellation/v2/internal/deploy/helm"
"github.com/edgelesssys/constellation/v2/internal/gcpshared"
"github.com/edgelesssys/constellation/v2/internal/logger" "github.com/edgelesssys/constellation/v2/internal/logger"
"github.com/edgelesssys/constellation/v2/internal/role" "github.com/edgelesssys/constellation/v2/internal/role"
"github.com/edgelesssys/constellation/v2/internal/versions" "github.com/edgelesssys/constellation/v2/internal/versions"
@ -61,14 +60,13 @@ type KubeWrapper struct {
configProvider configurationProvider configProvider configurationProvider
client k8sapi.Client client k8sapi.Client
kubeconfigReader configReader kubeconfigReader configReader
cloudControllerManager CloudControllerManager
providerMetadata ProviderMetadata providerMetadata ProviderMetadata
initialMeasurementsJSON []byte initialMeasurementsJSON []byte
getIPAddr func() (string, error) getIPAddr func() (string, error)
} }
// New creates a new KubeWrapper with real values. // New creates a new KubeWrapper with real values.
func New(cloudProvider string, clusterUtil clusterUtil, configProvider configurationProvider, client k8sapi.Client, cloudControllerManager CloudControllerManager, func New(cloudProvider string, clusterUtil clusterUtil, configProvider configurationProvider, client k8sapi.Client,
providerMetadata ProviderMetadata, initialMeasurementsJSON []byte, helmClient helmClient, kubeAPIWaiter kubeAPIWaiter, providerMetadata ProviderMetadata, initialMeasurementsJSON []byte, helmClient helmClient, kubeAPIWaiter kubeAPIWaiter,
) *KubeWrapper { ) *KubeWrapper {
return &KubeWrapper{ return &KubeWrapper{
@ -79,7 +77,6 @@ func New(cloudProvider string, clusterUtil clusterUtil, configProvider configura
configProvider: configProvider, configProvider: configProvider,
client: client, client: client,
kubeconfigReader: &KubeconfigReader{fs: afero.Afero{Fs: afero.NewOsFs()}}, kubeconfigReader: &KubeconfigReader{fs: afero.Afero{Fs: afero.NewOsFs()}},
cloudControllerManager: cloudControllerManager,
providerMetadata: providerMetadata, providerMetadata: providerMetadata,
initialMeasurementsJSON: initialMeasurementsJSON, initialMeasurementsJSON: initialMeasurementsJSON,
getIPAddr: getIPAddr, getIPAddr: getIPAddr,
@ -101,56 +98,47 @@ func (k *KubeWrapper) InitCluster(
return nil, err return nil, err
} }
ip, err := k.getIPAddr()
if err != nil {
return nil, err
}
nodeName := ip
var providerID string
var instance metadata.InstanceMetadata
var nodePodCIDR string var nodePodCIDR string
var subnetworkPodCIDR string
var controlPlaneEndpoint string // this is the endpoint in "kubeadm init --control-plane-endpoint=<IP/DNS>:<port>"
var nodeIP string
var validIPs []net.IP var validIPs []net.IP
// Step 1: retrieve cloud metadata for Kubernetes configuration // Step 1: retrieve cloud metadata for Kubernetes configuration
if k.providerMetadata.Supported() { log.Infof("Retrieving node metadata")
log.Infof("Retrieving node metadata") instance, err := k.providerMetadata.Self(ctx)
instance, err = k.providerMetadata.Self(ctx) if err != nil {
if err != nil { return nil, fmt.Errorf("retrieving own instance metadata: %w", err)
return nil, fmt.Errorf("retrieving own instance metadata: %w", err)
}
if instance.VPCIP != "" {
validIPs = append(validIPs, net.ParseIP(instance.VPCIP))
}
nodeName = k8sCompliantHostname(instance.Name)
providerID = instance.ProviderID
nodeIP = instance.VPCIP
subnetworkPodCIDR = instance.SecondaryIPRange
if len(instance.AliasIPRanges) > 0 {
nodePodCIDR = instance.AliasIPRanges[0]
}
controlPlaneEndpoint, err = k.providerMetadata.GetLoadBalancerEndpoint(ctx)
if err != nil {
return nil, fmt.Errorf("retrieving load balancer endpoint: %w", err)
}
} }
if instance.VPCIP != "" {
validIPs = append(validIPs, net.ParseIP(instance.VPCIP))
}
nodeName := k8sCompliantHostname(instance.Name)
nodeIP := instance.VPCIP
subnetworkPodCIDR := instance.SecondaryIPRange
if len(instance.AliasIPRanges) > 0 {
nodePodCIDR = instance.AliasIPRanges[0]
}
// this is the endpoint in "kubeadm init --control-plane-endpoint=<IP/DNS>:<port>"
controlPlaneEndpoint, err := k.providerMetadata.GetLoadBalancerEndpoint(ctx)
if err != nil {
return nil, fmt.Errorf("retrieving load balancer endpoint: %w", err)
}
log.With( log.With(
zap.String("nodeName", nodeName), zap.String("nodeName", nodeName),
zap.String("providerID", providerID), zap.String("providerID", instance.ProviderID),
zap.String("nodeIP", nodeIP), zap.String("nodeIP", nodeIP),
zap.String("controlPlaneEndpoint", controlPlaneEndpoint), zap.String("controlPlaneEndpoint", controlPlaneEndpoint),
zap.String("podCIDR", subnetworkPodCIDR), zap.String("podCIDR", subnetworkPodCIDR),
).Infof("Setting information for node") ).Infof("Setting information for node")
// Step 2: configure kubeadm init config // Step 2: configure kubeadm init config
initConfig := k.configProvider.InitConfiguration(k.cloudControllerManager.Supported(), k8sVersion) ccmSupported := cloudprovider.FromString(k.cloudProvider) == cloudprovider.Azure ||
cloudprovider.FromString(k.cloudProvider) == cloudprovider.GCP
initConfig := k.configProvider.InitConfiguration(ccmSupported, k8sVersion)
initConfig.SetNodeIP(nodeIP) initConfig.SetNodeIP(nodeIP)
initConfig.SetCertSANs([]string{nodeIP}) initConfig.SetCertSANs([]string{nodeIP})
initConfig.SetNodeName(nodeName) initConfig.SetNodeName(nodeName)
initConfig.SetProviderID(providerID) initConfig.SetProviderID(instance.ProviderID)
initConfig.SetControlPlaneEndpoint(controlPlaneEndpoint) initConfig.SetControlPlaneEndpoint(controlPlaneEndpoint)
initConfigYAML, err := initConfig.Marshal() initConfigYAML, err := initConfig.Marshal()
if err != nil { if err != nil {
@ -262,28 +250,19 @@ func (k *KubeWrapper) JoinCluster(ctx context.Context, args *kubeadm.BootstrapTo
} }
// Step 1: retrieve cloud metadata for Kubernetes configuration // Step 1: retrieve cloud metadata for Kubernetes configuration
nodeInternalIP, err := k.getIPAddr() log.Infof("Retrieving node metadata")
instance, err := k.providerMetadata.Self(ctx)
if err != nil { if err != nil {
return err return fmt.Errorf("retrieving own instance metadata: %w", err)
} }
nodeName := nodeInternalIP providerID := instance.ProviderID
var providerID string nodeInternalIP := instance.VPCIP
var loadbalancerEndpoint string nodeName := k8sCompliantHostname(instance.Name)
if k.providerMetadata.Supported() {
log.Infof("Retrieving node metadata") loadbalancerEndpoint, err := k.providerMetadata.GetLoadBalancerEndpoint(ctx)
instance, err := k.providerMetadata.Self(ctx) if err != nil {
if err != nil { return fmt.Errorf("retrieving own instance metadata: %w", err)
return fmt.Errorf("retrieving own instance metadata: %w", err)
}
providerID = instance.ProviderID
nodeName = instance.Name
nodeInternalIP = instance.VPCIP
loadbalancerEndpoint, err = k.providerMetadata.GetLoadBalancerEndpoint(ctx)
if err != nil {
return fmt.Errorf("retrieving loadbalancer endpoint: %w", err)
}
} }
nodeName = k8sCompliantHostname(nodeName)
log.With( log.With(
zap.String("nodeName", nodeName), zap.String("nodeName", nodeName),
@ -292,7 +271,9 @@ func (k *KubeWrapper) JoinCluster(ctx context.Context, args *kubeadm.BootstrapTo
).Infof("Setting information for node") ).Infof("Setting information for node")
// Step 2: configure kubeadm join config // Step 2: configure kubeadm join config
joinConfig := k.configProvider.JoinConfiguration(k.cloudControllerManager.Supported()) ccmSupported := cloudprovider.FromString(k.cloudProvider) == cloudprovider.Azure ||
cloudprovider.FromString(k.cloudProvider) == cloudprovider.GCP
joinConfig := k.configProvider.JoinConfiguration(ccmSupported)
joinConfig.SetAPIServerEndpoint(args.APIServerEndpoint) joinConfig.SetAPIServerEndpoint(args.APIServerEndpoint)
joinConfig.SetToken(args.Token) joinConfig.SetToken(args.Token)
joinConfig.AppendDiscoveryTokenCaCertHash(args.CACertHashes[0]) joinConfig.AppendDiscoveryTokenCaCertHash(args.CACertHashes[0])
@ -443,85 +424,85 @@ func (k *KubeWrapper) setupExtraVals(ctx context.Context, initialMeasurementsJSO
switch cloudprovider.FromString(k.cloudProvider) { switch cloudprovider.FromString(k.cloudProvider) {
case cloudprovider.GCP: case cloudprovider.GCP:
{ uid, err := k.providerMetadata.UID(ctx)
uid, err := k.providerMetadata.UID(ctx) if err != nil {
if err != nil { return nil, fmt.Errorf("getting uid: %w", err)
return nil, fmt.Errorf("getting uid: %w", err)
}
projectID, _, _, err := gcpshared.SplitProviderID(instance.ProviderID)
if err != nil {
return nil, fmt.Errorf("splitting providerID: %w", err)
}
serviceAccountKey, err := gcpshared.ServiceAccountKeyFromURI(cloudServiceAccountURI)
if err != nil {
return nil, fmt.Errorf("getting service account key: %w", err)
}
rawKey, err := json.Marshal(serviceAccountKey)
if err != nil {
return nil, fmt.Errorf("marshaling service account key: %w", err)
}
ccmVals, ok := extraVals["ccm"].(map[string]any)
if !ok {
return nil, errors.New("invalid ccm values")
}
ccmVals["GCP"] = map[string]any{
"projectID": projectID,
"uid": uid,
"secretData": string(rawKey),
"subnetworkPodCIDR": subnetworkPodCIDR,
}
} }
projectID, _, _, err := gcpshared.SplitProviderID(instance.ProviderID)
if err != nil {
return nil, fmt.Errorf("splitting providerID: %w", err)
}
serviceAccountKey, err := gcpshared.ServiceAccountKeyFromURI(cloudServiceAccountURI)
if err != nil {
return nil, fmt.Errorf("getting service account key: %w", err)
}
rawKey, err := json.Marshal(serviceAccountKey)
if err != nil {
return nil, fmt.Errorf("marshaling service account key: %w", err)
}
ccmVals, ok := extraVals["ccm"].(map[string]any)
if !ok {
return nil, errors.New("invalid ccm values")
}
ccmVals["GCP"] = map[string]any{
"projectID": projectID,
"uid": uid,
"secretData": string(rawKey),
"subnetworkPodCIDR": subnetworkPodCIDR,
}
case cloudprovider.Azure: case cloudprovider.Azure:
{ ccmAzure, ok := k.providerMetadata.(ccmConfigGetter)
// TODO: After refactoring the ProviderMetadata interface this section should be rewritten. if !ok {
// Currently, we have to rely on the Secrets(..) method, as GetNetworkSecurityGroupName & GetLoadBalancerName return nil, errors.New("invalid cloud provider metadata for Azure")
// rely on Azure specific API endpoints.
ccmSecrets, err := k.cloudControllerManager.Secrets(ctx, instance.ProviderID, cloudServiceAccountURI)
if err != nil {
return nil, fmt.Errorf("creating ccm secret: %w", err)
}
if len(ccmSecrets) < 1 {
return nil, errors.New("missing secret")
}
rawConfig := ccmSecrets[0].Data["azure.json"]
ccmVals, ok := extraVals["ccm"].(map[string]any)
if !ok {
return nil, errors.New("invalid ccm values")
}
ccmVals["Azure"] = map[string]any{
"azureConfig": string(rawConfig),
"subnetworkPodCIDR": subnetworkPodCIDR,
}
joinVals, ok := extraVals["join-service"].(map[string]any)
if !ok {
return nil, errors.New("invalid join-service values")
}
joinVals["idkeydigest"] = hex.EncodeToString(idkeydigest)
subscriptionID, resourceGroup, err := azureshared.BasicsFromProviderID(instance.ProviderID)
if err != nil {
return nil, err
}
creds, err := azureshared.ApplicationCredentialsFromURI(cloudServiceAccountURI)
if err != nil {
return nil, err
}
extraVals["autoscaler"] = map[string]any{
"Azure": map[string]any{
"clientID": creds.AppClientID,
"clientSecret": creds.ClientSecretValue,
"resourceGroup": resourceGroup,
"subscriptionID": subscriptionID,
"tenantID": creds.TenantID,
},
}
} }
ccmConfig, err := ccmAzure.GetCCMConfig(ctx, instance.ProviderID, cloudServiceAccountURI)
if err != nil {
return nil, fmt.Errorf("creating ccm secret: %w", err)
}
ccmVals, ok := extraVals["ccm"].(map[string]any)
if !ok {
return nil, errors.New("invalid ccm values")
}
ccmVals["Azure"] = map[string]any{
"azureConfig": string(ccmConfig),
"subnetworkPodCIDR": subnetworkPodCIDR,
}
joinVals, ok := extraVals["join-service"].(map[string]any)
if !ok {
return nil, errors.New("invalid join-service values")
}
joinVals["idkeydigest"] = hex.EncodeToString(idkeydigest)
subscriptionID, resourceGroup, err := azureshared.BasicsFromProviderID(instance.ProviderID)
if err != nil {
return nil, err
}
creds, err := azureshared.ApplicationCredentialsFromURI(cloudServiceAccountURI)
if err != nil {
return nil, err
}
extraVals["autoscaler"] = map[string]any{
"Azure": map[string]any{
"clientID": creds.AppClientID,
"clientSecret": creds.ClientSecretValue,
"resourceGroup": resourceGroup,
"subscriptionID": subscriptionID,
"tenantID": creds.TenantID,
},
}
} }
return extraVals, nil return extraVals, nil
} }
type ccmConfigGetter interface {
GetCCMConfig(ctx context.Context, providerID, cloudServiceAccountURI string) ([]byte, error)
}

243
bootstrapper/internal/kubernetes/kubernetes_test.go
View File

@ -45,41 +45,16 @@ func TestInitCluster(t *testing.T) {
aliasIPRange := "192.0.2.0/24" aliasIPRange := "192.0.2.0/24"
testCases := map[string]struct { testCases := map[string]struct {
clusterUtil stubClusterUtil clusterUtil stubClusterUtil
helmClient stubHelmClient helmClient stubHelmClient
kubectl stubKubectl kubectl stubKubectl
kubeAPIWaiter stubKubeAPIWaiter kubeAPIWaiter stubKubeAPIWaiter
providerMetadata ProviderMetadata providerMetadata ProviderMetadata
CloudControllerManager CloudControllerManager kubeconfigReader configReader
ClusterAutoscaler ClusterAutoscaler wantConfig k8sapi.KubeadmInitYAML
kubeconfigReader configReader wantErr bool
wantConfig k8sapi.KubeadmInitYAML k8sVersion versions.ValidK8sVersion
wantErr bool
k8sVersion versions.ValidK8sVersion
}{ }{
"kubeadm init works without metadata": {
clusterUtil: stubClusterUtil{},
kubeconfigReader: &stubKubeconfigReader{
Kubeconfig: []byte("someKubeconfig"),
},
kubeAPIWaiter: stubKubeAPIWaiter{},
providerMetadata: &stubProviderMetadata{SupportedResp: false},
CloudControllerManager: &stubCloudControllerManager{},
ClusterAutoscaler: &stubClusterAutoscaler{},
wantConfig: k8sapi.KubeadmInitYAML{
InitConfiguration: kubeadm.InitConfiguration{
NodeRegistration: kubeadm.NodeRegistrationOptions{
KubeletExtraArgs: map[string]string{
"node-ip": "",
"provider-id": "",
},
Name: privateIP,
},
},
ClusterConfiguration: kubeadm.ClusterConfiguration{},
},
k8sVersion: versions.Default,
},
"kubeadm init works with metadata and loadbalancer": { "kubeadm init works with metadata and loadbalancer": {
clusterUtil: stubClusterUtil{}, clusterUtil: stubClusterUtil{},
kubeconfigReader: &stubKubeconfigReader{ kubeconfigReader: &stubKubeconfigReader{
@ -87,7 +62,6 @@ func TestInitCluster(t *testing.T) {
}, },
kubeAPIWaiter: stubKubeAPIWaiter{}, kubeAPIWaiter: stubKubeAPIWaiter{},
providerMetadata: &stubProviderMetadata{ providerMetadata: &stubProviderMetadata{
SupportedResp: true,
SelfResp: metadata.InstanceMetadata{ SelfResp: metadata.InstanceMetadata{
Name: nodeName, Name: nodeName,
ProviderID: providerID, ProviderID: providerID,
@ -95,10 +69,7 @@ func TestInitCluster(t *testing.T) {
AliasIPRanges: []string{aliasIPRange}, AliasIPRanges: []string{aliasIPRange},
}, },
GetLoadBalancerEndpointResp: loadbalancerIP, GetLoadBalancerEndpointResp: loadbalancerIP,
SupportsLoadBalancerResp: true,
}, },
CloudControllerManager: &stubCloudControllerManager{},
ClusterAutoscaler: &stubClusterAutoscaler{},
wantConfig: k8sapi.KubeadmInitYAML{ wantConfig: k8sapi.KubeadmInitYAML{
InitConfiguration: kubeadm.InitConfiguration{ InitConfiguration: kubeadm.InitConfiguration{
NodeRegistration: kubeadm.NodeRegistrationOptions{ NodeRegistration: kubeadm.NodeRegistrationOptions{
@ -126,13 +97,10 @@ func TestInitCluster(t *testing.T) {
}, },
kubeAPIWaiter: stubKubeAPIWaiter{}, kubeAPIWaiter: stubKubeAPIWaiter{},
providerMetadata: &stubProviderMetadata{ providerMetadata: &stubProviderMetadata{
SelfErr: someErr, SelfErr: someErr,
SupportedResp: true,
}, },
CloudControllerManager: &stubCloudControllerManager{}, wantErr: true,
ClusterAutoscaler: &stubClusterAutoscaler{}, k8sVersion: versions.Default,
wantErr: true,
k8sVersion: versions.Default,
}, },
"kubeadm init fails when retrieving metadata loadbalancer ip": { "kubeadm init fails when retrieving metadata loadbalancer ip": {
clusterUtil: stubClusterUtil{}, clusterUtil: stubClusterUtil{},
@ -141,25 +109,19 @@ func TestInitCluster(t *testing.T) {
}, },
providerMetadata: &stubProviderMetadata{ providerMetadata: &stubProviderMetadata{
GetLoadBalancerEndpointErr: someErr, GetLoadBalancerEndpointErr: someErr,
SupportsLoadBalancerResp: true,
SupportedResp: true,
}, },
CloudControllerManager: &stubCloudControllerManager{}, wantErr: true,
ClusterAutoscaler: &stubClusterAutoscaler{}, k8sVersion: versions.Default,
wantErr: true,
k8sVersion: versions.Default,
}, },
"kubeadm init fails when applying the init config": { "kubeadm init fails when applying the init config": {
clusterUtil: stubClusterUtil{initClusterErr: someErr}, clusterUtil: stubClusterUtil{initClusterErr: someErr},
kubeconfigReader: &stubKubeconfigReader{ kubeconfigReader: &stubKubeconfigReader{
Kubeconfig: []byte("someKubeconfig"), Kubeconfig: []byte("someKubeconfig"),
}, },
kubeAPIWaiter: stubKubeAPIWaiter{}, kubeAPIWaiter: stubKubeAPIWaiter{},
providerMetadata: &stubProviderMetadata{}, providerMetadata: &stubProviderMetadata{},
CloudControllerManager: &stubCloudControllerManager{}, wantErr: true,
ClusterAutoscaler: &stubClusterAutoscaler{}, k8sVersion: versions.Default,
wantErr: true,
k8sVersion: versions.Default,
}, },
"kubeadm init fails when deploying cilium": { "kubeadm init fails when deploying cilium": {
clusterUtil: stubClusterUtil{}, clusterUtil: stubClusterUtil{},
@ -167,11 +129,9 @@ func TestInitCluster(t *testing.T) {
kubeconfigReader: &stubKubeconfigReader{ kubeconfigReader: &stubKubeconfigReader{
Kubeconfig: []byte("someKubeconfig"), Kubeconfig: []byte("someKubeconfig"),
}, },
providerMetadata: &stubProviderMetadata{}, providerMetadata: &stubProviderMetadata{},
CloudControllerManager: &stubCloudControllerManager{}, wantErr: true,
ClusterAutoscaler: &stubClusterAutoscaler{}, k8sVersion: versions.Default,
wantErr: true,
k8sVersion: versions.Default,
}, },
"kubeadm init fails when setting up constellation-services chart": { "kubeadm init fails when setting up constellation-services chart": {
clusterUtil: stubClusterUtil{}, clusterUtil: stubClusterUtil{},
@ -179,12 +139,10 @@ func TestInitCluster(t *testing.T) {
kubeconfigReader: &stubKubeconfigReader{ kubeconfigReader: &stubKubeconfigReader{
Kubeconfig: []byte("someKubeconfig"), Kubeconfig: []byte("someKubeconfig"),
}, },
kubeAPIWaiter: stubKubeAPIWaiter{}, kubeAPIWaiter: stubKubeAPIWaiter{},
providerMetadata: &stubProviderMetadata{}, providerMetadata: &stubProviderMetadata{},
CloudControllerManager: &stubCloudControllerManager{SupportedResp: true}, wantErr: true,
ClusterAutoscaler: &stubClusterAutoscaler{}, k8sVersion: versions.Default,
wantErr: true,
k8sVersion: versions.Default,
}, },
"kubeadm init fails when setting the cloud node manager": { "kubeadm init fails when setting the cloud node manager": {
clusterUtil: stubClusterUtil{}, clusterUtil: stubClusterUtil{},
@ -192,12 +150,10 @@ func TestInitCluster(t *testing.T) {
kubeconfigReader: &stubKubeconfigReader{ kubeconfigReader: &stubKubeconfigReader{
Kubeconfig: []byte("someKubeconfig"), Kubeconfig: []byte("someKubeconfig"),
}, },
kubeAPIWaiter: stubKubeAPIWaiter{}, kubeAPIWaiter: stubKubeAPIWaiter{},
providerMetadata: &stubProviderMetadata{}, providerMetadata: &stubProviderMetadata{},
CloudControllerManager: &stubCloudControllerManager{}, wantErr: true,
ClusterAutoscaler: &stubClusterAutoscaler{}, k8sVersion: versions.Default,
wantErr: true,
k8sVersion: versions.Default,
}, },
"kubeadm init fails when setting the cluster autoscaler": { "kubeadm init fails when setting the cluster autoscaler": {
clusterUtil: stubClusterUtil{}, clusterUtil: stubClusterUtil{},
@ -205,72 +161,60 @@ func TestInitCluster(t *testing.T) {
kubeconfigReader: &stubKubeconfigReader{ kubeconfigReader: &stubKubeconfigReader{
Kubeconfig: []byte("someKubeconfig"), Kubeconfig: []byte("someKubeconfig"),
}, },
kubeAPIWaiter: stubKubeAPIWaiter{}, kubeAPIWaiter: stubKubeAPIWaiter{},
providerMetadata: &stubProviderMetadata{}, providerMetadata: &stubProviderMetadata{},
CloudControllerManager: &stubCloudControllerManager{}, wantErr: true,
ClusterAutoscaler: &stubClusterAutoscaler{SupportedResp: true}, k8sVersion: versions.Default,
wantErr: true,
k8sVersion: versions.Default,
}, },
"kubeadm init fails when reading kubeconfig": { "kubeadm init fails when reading kubeconfig": {
clusterUtil: stubClusterUtil{}, clusterUtil: stubClusterUtil{},
kubeconfigReader: &stubKubeconfigReader{ kubeconfigReader: &stubKubeconfigReader{
ReadErr: someErr, ReadErr: someErr,
}, },
kubeAPIWaiter: stubKubeAPIWaiter{}, kubeAPIWaiter: stubKubeAPIWaiter{},
providerMetadata: &stubProviderMetadata{}, providerMetadata: &stubProviderMetadata{},
CloudControllerManager: &stubCloudControllerManager{}, wantErr: true,
ClusterAutoscaler: &stubClusterAutoscaler{}, k8sVersion: versions.Default,
wantErr: true,
k8sVersion: versions.Default,
}, },
"kubeadm init fails when setting up konnectivity": { "kubeadm init fails when setting up konnectivity": {
clusterUtil: stubClusterUtil{setupKonnectivityError: someErr}, clusterUtil: stubClusterUtil{setupKonnectivityError: someErr},
kubeconfigReader: &stubKubeconfigReader{ kubeconfigReader: &stubKubeconfigReader{
Kubeconfig: []byte("someKubeconfig"), Kubeconfig: []byte("someKubeconfig"),
}, },
kubeAPIWaiter: stubKubeAPIWaiter{}, kubeAPIWaiter: stubKubeAPIWaiter{},
providerMetadata: &stubProviderMetadata{SupportedResp: false}, providerMetadata: &stubProviderMetadata{},
CloudControllerManager: &stubCloudControllerManager{}, wantErr: true,
ClusterAutoscaler: &stubClusterAutoscaler{}, k8sVersion: versions.Default,
wantErr: true,
k8sVersion: versions.Default,
}, },
"kubeadm init fails when setting up verification service": { "kubeadm init fails when setting up verification service": {
clusterUtil: stubClusterUtil{setupVerificationServiceErr: someErr}, clusterUtil: stubClusterUtil{setupVerificationServiceErr: someErr},
kubeconfigReader: &stubKubeconfigReader{ kubeconfigReader: &stubKubeconfigReader{
Kubeconfig: []byte("someKubeconfig"), Kubeconfig: []byte("someKubeconfig"),
}, },
kubeAPIWaiter: stubKubeAPIWaiter{}, kubeAPIWaiter: stubKubeAPIWaiter{},
providerMetadata: &stubProviderMetadata{SupportedResp: false}, providerMetadata: &stubProviderMetadata{},
CloudControllerManager: &stubCloudControllerManager{}, wantErr: true,
ClusterAutoscaler: &stubClusterAutoscaler{}, k8sVersion: versions.Default,
wantErr: true,
k8sVersion: versions.Default,
}, },
"kubeadm init fails when waiting for kubeAPI server": { "kubeadm init fails when waiting for kubeAPI server": {
clusterUtil: stubClusterUtil{}, clusterUtil: stubClusterUtil{},
kubeconfigReader: &stubKubeconfigReader{ kubeconfigReader: &stubKubeconfigReader{
Kubeconfig: []byte("someKubeconfig"), Kubeconfig: []byte("someKubeconfig"),
}, },
kubeAPIWaiter: stubKubeAPIWaiter{waitErr: someErr}, kubeAPIWaiter: stubKubeAPIWaiter{waitErr: someErr},
providerMetadata: &stubProviderMetadata{SupportedResp: false}, providerMetadata: &stubProviderMetadata{},
CloudControllerManager: &stubCloudControllerManager{}, k8sVersion: versions.Default,
ClusterAutoscaler: &stubClusterAutoscaler{}, wantErr: true,
k8sVersion: versions.Default,
wantErr: true,
}, },
"unsupported k8sVersion fails cluster creation": { "unsupported k8sVersion fails cluster creation": {
clusterUtil: stubClusterUtil{}, clusterUtil: stubClusterUtil{},
kubeconfigReader: &stubKubeconfigReader{ kubeconfigReader: &stubKubeconfigReader{
Kubeconfig: []byte("someKubeconfig"), Kubeconfig: []byte("someKubeconfig"),
}, },
kubeAPIWaiter: stubKubeAPIWaiter{}, kubeAPIWaiter: stubKubeAPIWaiter{},
providerMetadata: &stubProviderMetadata{}, providerMetadata: &stubProviderMetadata{},
CloudControllerManager: &stubCloudControllerManager{}, k8sVersion: "1.19",
ClusterAutoscaler: &stubClusterAutoscaler{}, wantErr: true,
k8sVersion: "1.19",
wantErr: true,
}, },
} }
@ -280,15 +224,14 @@ func TestInitCluster(t *testing.T) {
require := require.New(t) require := require.New(t)
kube := KubeWrapper{ kube := KubeWrapper{
clusterUtil: &tc.clusterUtil, clusterUtil: &tc.clusterUtil,
helmClient: &tc.helmClient, helmClient: &tc.helmClient,
providerMetadata: tc.providerMetadata, providerMetadata: tc.providerMetadata,
kubeAPIWaiter: &tc.kubeAPIWaiter, kubeAPIWaiter: &tc.kubeAPIWaiter,
cloudControllerManager: tc.CloudControllerManager, configProvider: &stubConfigProvider{InitConfig: k8sapi.KubeadmInitYAML{}},
configProvider: &stubConfigProvider{InitConfig: k8sapi.KubeadmInitYAML{}}, client: &tc.kubectl,
client: &tc.kubectl, kubeconfigReader: tc.kubeconfigReader,
kubeconfigReader: tc.kubeconfigReader, getIPAddr: func() (string, error) { return privateIP, nil },
getIPAddr: func() (string, error) { return privateIP, nil },
} }
_, err := kube.InitCluster( _, err := kube.InitCluster(
@ -322,40 +265,22 @@ func TestJoinCluster(t *testing.T) {
k8sVersion := versions.Default k8sVersion := versions.Default
testCases := map[string]struct { testCases := map[string]struct {
clusterUtil stubClusterUtil clusterUtil stubClusterUtil
providerMetadata ProviderMetadata providerMetadata ProviderMetadata
CloudControllerManager CloudControllerManager wantConfig kubeadm.JoinConfiguration
wantConfig kubeadm.JoinConfiguration role role.Role
role role.Role wantErr bool
wantErr bool
}{ }{
"kubeadm join worker works without metadata": {
clusterUtil: stubClusterUtil{},
providerMetadata: &stubProviderMetadata{},
CloudControllerManager: &stubCloudControllerManager{},
role: role.Worker,
wantConfig: kubeadm.JoinConfiguration{
Discovery: kubeadm.Discovery{
BootstrapToken: joinCommand,
},
NodeRegistration: kubeadm.NodeRegistrationOptions{
Name: privateIP,
KubeletExtraArgs: map[string]string{"node-ip": privateIP},
},
},
},
"kubeadm join worker works with metadata": { "kubeadm join worker works with metadata": {
clusterUtil: stubClusterUtil{}, clusterUtil: stubClusterUtil{},
providerMetadata: &stubProviderMetadata{ providerMetadata: &stubProviderMetadata{
SupportedResp: true,
SelfResp: metadata.InstanceMetadata{ SelfResp: metadata.InstanceMetadata{
ProviderID: "provider-id", ProviderID: "provider-id",
Name: "metadata-name", Name: "metadata-name",
VPCIP: "192.0.2.1", VPCIP: "192.0.2.1",
}, },
}, },
CloudControllerManager: &stubCloudControllerManager{}, role: role.Worker,
role: role.Worker,
wantConfig: kubeadm.JoinConfiguration{ wantConfig: kubeadm.JoinConfiguration{
Discovery: kubeadm.Discovery{ Discovery: kubeadm.Discovery{
BootstrapToken: joinCommand, BootstrapToken: joinCommand,
@ -369,16 +294,12 @@ func TestJoinCluster(t *testing.T) {
"kubeadm join worker works with metadata and cloud controller manager": { "kubeadm join worker works with metadata and cloud controller manager": {
clusterUtil: stubClusterUtil{}, clusterUtil: stubClusterUtil{},
providerMetadata: &stubProviderMetadata{ providerMetadata: &stubProviderMetadata{
SupportedResp: true,
SelfResp: metadata.InstanceMetadata{ SelfResp: metadata.InstanceMetadata{
ProviderID: "provider-id", ProviderID: "provider-id",
Name: "metadata-name", Name: "metadata-name",
VPCIP: "192.0.2.1", VPCIP: "192.0.2.1",
}, },
}, },
CloudControllerManager: &stubCloudControllerManager{
SupportedResp: true,
},
role: role.Worker, role: role.Worker,
wantConfig: kubeadm.JoinConfiguration{ wantConfig: kubeadm.JoinConfiguration{
Discovery: kubeadm.Discovery{ Discovery: kubeadm.Discovery{
@ -393,15 +314,13 @@ func TestJoinCluster(t *testing.T) {
"kubeadm join control-plane node works with metadata": { "kubeadm join control-plane node works with metadata": {
clusterUtil: stubClusterUtil{}, clusterUtil: stubClusterUtil{},
providerMetadata: &stubProviderMetadata{ providerMetadata: &stubProviderMetadata{
SupportedResp: true,
SelfResp: metadata.InstanceMetadata{ SelfResp: metadata.InstanceMetadata{
ProviderID: "provider-id", ProviderID: "provider-id",
Name: "metadata-name", Name: "metadata-name",
VPCIP: "192.0.2.1", VPCIP: "192.0.2.1",
}, },
}, },
CloudControllerManager: &stubCloudControllerManager{}, role: role.ControlPlane,
role: role.ControlPlane,
wantConfig: kubeadm.JoinConfiguration{ wantConfig: kubeadm.JoinConfiguration{
Discovery: kubeadm.Discovery{ Discovery: kubeadm.Discovery{
BootstrapToken: joinCommand, BootstrapToken: joinCommand,
@ -422,19 +341,16 @@ func TestJoinCluster(t *testing.T) {
"kubeadm join worker fails when retrieving self metadata": { "kubeadm join worker fails when retrieving self metadata": {
clusterUtil: stubClusterUtil{}, clusterUtil: stubClusterUtil{},
providerMetadata: &stubProviderMetadata{ providerMetadata: &stubProviderMetadata{
SupportedResp: true, SelfErr: someErr,
SelfErr: someErr,
}, },
CloudControllerManager: &stubCloudControllerManager{}, role: role.Worker,
role: role.Worker, wantErr: true,
wantErr: true,
}, },
"kubeadm join worker fails when applying the join config": { "kubeadm join worker fails when applying the join config": {
clusterUtil: stubClusterUtil{joinClusterErr: someErr}, clusterUtil: stubClusterUtil{joinClusterErr: someErr},
providerMetadata: &stubProviderMetadata{}, providerMetadata: &stubProviderMetadata{},
CloudControllerManager: &stubCloudControllerManager{}, role: role.Worker,
role: role.Worker, wantErr: true,
wantErr: true,
}, },
} }
@ -444,11 +360,10 @@ func TestJoinCluster(t *testing.T) {
require := require.New(t) require := require.New(t)
kube := KubeWrapper{ kube := KubeWrapper{
clusterUtil: &tc.clusterUtil, clusterUtil: &tc.clusterUtil,
providerMetadata: tc.providerMetadata, providerMetadata: tc.providerMetadata,
cloudControllerManager: tc.CloudControllerManager, configProvider: &stubConfigProvider{},
configProvider: &stubConfigProvider{}, getIPAddr: func() (string, error) { return privateIP, nil },
getIPAddr: func() (string, error) { return privateIP, nil },
} }
err := kube.JoinCluster(context.Background(), joinCommand, tc.role, string(k8sVersion), logger.NewTest(t)) err := kube.JoinCluster(context.Background(), joinCommand, tc.role, string(k8sVersion), logger.NewTest(t))

4
cli/internal/cmd/init.go
View File

@ -20,14 +20,14 @@ import (
"github.com/edgelesssys/constellation/v2/cli/internal/cloudcmd" "github.com/edgelesssys/constellation/v2/cli/internal/cloudcmd"
"github.com/edgelesssys/constellation/v2/cli/internal/clusterid" "github.com/edgelesssys/constellation/v2/cli/internal/clusterid"
"github.com/edgelesssys/constellation/v2/cli/internal/helm" "github.com/edgelesssys/constellation/v2/cli/internal/helm"
"github.com/edgelesssys/constellation/v2/internal/azureshared" "github.com/edgelesssys/constellation/v2/internal/cloud/azureshared"
"github.com/edgelesssys/constellation/v2/internal/cloud/cloudprovider" "github.com/edgelesssys/constellation/v2/internal/cloud/cloudprovider"
"github.com/edgelesssys/constellation/v2/internal/cloud/gcpshared"
"github.com/edgelesssys/constellation/v2/internal/config" "github.com/edgelesssys/constellation/v2/internal/config"
"github.com/edgelesssys/constellation/v2/internal/constants" "github.com/edgelesssys/constellation/v2/internal/constants"
"github.com/edgelesssys/constellation/v2/internal/crypto" "github.com/edgelesssys/constellation/v2/internal/crypto"
"github.com/edgelesssys/constellation/v2/internal/deploy/ssh" "github.com/edgelesssys/constellation/v2/internal/deploy/ssh"
"github.com/edgelesssys/constellation/v2/internal/file" "github.com/edgelesssys/constellation/v2/internal/file"
"github.com/edgelesssys/constellation/v2/internal/gcpshared"
"github.com/edgelesssys/constellation/v2/internal/grpc/dialer" "github.com/edgelesssys/constellation/v2/internal/grpc/dialer"
grpcRetry "github.com/edgelesssys/constellation/v2/internal/grpc/retry" grpcRetry "github.com/edgelesssys/constellation/v2/internal/grpc/retry"
"github.com/edgelesssys/constellation/v2/internal/license" "github.com/edgelesssys/constellation/v2/internal/license"

2
cli/internal/cmd/init_test.go
View File

@ -22,10 +22,10 @@ import (
"github.com/edgelesssys/constellation/v2/cli/internal/cloudcmd" "github.com/edgelesssys/constellation/v2/cli/internal/cloudcmd"
"github.com/edgelesssys/constellation/v2/cli/internal/clusterid" "github.com/edgelesssys/constellation/v2/cli/internal/clusterid"
"github.com/edgelesssys/constellation/v2/internal/cloud/cloudprovider" "github.com/edgelesssys/constellation/v2/internal/cloud/cloudprovider"
"github.com/edgelesssys/constellation/v2/internal/cloud/gcpshared"
"github.com/edgelesssys/constellation/v2/internal/config" "github.com/edgelesssys/constellation/v2/internal/config"
"github.com/edgelesssys/constellation/v2/internal/constants" "github.com/edgelesssys/constellation/v2/internal/constants"
"github.com/edgelesssys/constellation/v2/internal/file" "github.com/edgelesssys/constellation/v2/internal/file"
"github.com/edgelesssys/constellation/v2/internal/gcpshared"
"github.com/edgelesssys/constellation/v2/internal/grpc/atlscredentials" "github.com/edgelesssys/constellation/v2/internal/grpc/atlscredentials"
"github.com/edgelesssys/constellation/v2/internal/grpc/dialer" "github.com/edgelesssys/constellation/v2/internal/grpc/dialer"
"github.com/edgelesssys/constellation/v2/internal/grpc/testdialer" "github.com/edgelesssys/constellation/v2/internal/grpc/testdialer"

38
debugd/cmd/debugd/debugd.go
View File

@ -20,11 +20,16 @@ import (
"github.com/edgelesssys/constellation/v2/debugd/internal/debugd/metadata/cloudprovider" "github.com/edgelesssys/constellation/v2/debugd/internal/debugd/metadata/cloudprovider"
"github.com/edgelesssys/constellation/v2/debugd/internal/debugd/metadata/fallback" "github.com/edgelesssys/constellation/v2/debugd/internal/debugd/metadata/fallback"
"github.com/edgelesssys/constellation/v2/debugd/internal/debugd/server" "github.com/edgelesssys/constellation/v2/debugd/internal/debugd/server"
awscloud "github.com/edgelesssys/constellation/v2/internal/cloud/aws"
azurecloud "github.com/edgelesssys/constellation/v2/internal/cloud/azure"
platform "github.com/edgelesssys/constellation/v2/internal/cloud/cloudprovider" platform "github.com/edgelesssys/constellation/v2/internal/cloud/cloudprovider"
gcpcloud "github.com/edgelesssys/constellation/v2/internal/cloud/gcp"
qemucloud "github.com/edgelesssys/constellation/v2/internal/cloud/qemu"
"github.com/edgelesssys/constellation/v2/internal/deploy/ssh" "github.com/edgelesssys/constellation/v2/internal/deploy/ssh"
"github.com/edgelesssys/constellation/v2/internal/deploy/user" "github.com/edgelesssys/constellation/v2/internal/deploy/user"
"github.com/edgelesssys/constellation/v2/internal/logger" "github.com/edgelesssys/constellation/v2/internal/logger"
"github.com/spf13/afero" "github.com/spf13/afero"
"go.uber.org/zap"
) )
const debugBanner = ` const debugBanner = `
@ -56,34 +61,39 @@ func main() {
csp := os.Getenv("CONSTEL_CSP") csp := os.Getenv("CONSTEL_CSP")
switch platform.FromString(csp) { switch platform.FromString(csp) {
case platform.AWS: case platform.AWS:
awsFetcher, err := cloudprovider.NewAWS(ctx) meta, err := awscloud.New(ctx)
if err != nil { if err != nil {
log.Fatalf("%s", err) log.With(zap.Error(err)).Fatalf("Failed to initialize AWS metadata")
} }
fetcher = awsFetcher fetcher = cloudprovider.New(meta)
case platform.Azure: case platform.Azure:
azureFetcher, err := cloudprovider.NewAzure(ctx) meta, err := azurecloud.NewMetadata(ctx)
if err != nil { if err != nil {
log.Fatalf("%s", err) log.With(zap.Error(err)).Fatalf("Failed to initialize Azure metadata")
} }
fetcher = azureFetcher fetcher = cloudprovider.New(meta)
case platform.GCP: case platform.GCP:
gcpFetcher, err := cloudprovider.NewGCP(ctx) meta, err := gcpcloud.New(ctx)
if err != nil { if err != nil {
log.Fatalf("%s", err) log.With(zap.Error(err)).Fatalf("Failed to initialize GCP metadata")
} }
fetcher = gcpFetcher defer meta.Close()
log.Infof("Added load balancer IP to local routing table") fetcher = cloudprovider.New(meta)
case platform.QEMU: case platform.QEMU:
fetcher = cloudprovider.NewQEMU() fetcher = cloudprovider.New(&qemucloud.Metadata{})
default: default:
log.Errorf("Unknown / unimplemented cloud provider CONSTEL_CSP=%v. Using fallback", csp) log.Errorf("Unknown / unimplemented cloud provider CONSTEL_CSP=%v. Using fallback", csp)
fetcher = fallback.Fetcher{} fetcher = fallback.Fetcher{}
} }
sched := metadata.NewScheduler(log.Named("scheduler"), fetcher, ssh, download) sched := metadata.NewScheduler(log.Named("scheduler"), fetcher, ssh, download)
serv := server.New(log.Named("server"), ssh, serviceManager, streamer) serv := server.New(log.Named("server"), ssh, serviceManager, streamer)
if err := deploy.DefaultServiceUnit(ctx, serviceManager); err != nil { if err := deploy.DefaultServiceUnit(ctx, serviceManager); err != nil {
log.Fatalf("%s", err) log.With(zap.Error(err)).Fatalf("Failed to create default service unit")
} }
writeDebugBanner(log) writeDebugBanner(log)
@ -101,11 +111,11 @@ func main() {
func writeDebugBanner(log *logger.Logger) { func writeDebugBanner(log *logger.Logger) {
tty, err := os.OpenFile("/dev/ttyS0", os.O_WRONLY, os.ModeAppend) tty, err := os.OpenFile("/dev/ttyS0", os.O_WRONLY, os.ModeAppend)
if err != nil { if err != nil {
log.Infof("Unable to open /dev/ttyS0 for printing banner: %v", err) log.With(zap.Error(err)).Errorf("Unable to open /dev/ttyS0 for printing banner")
return return
} }
defer tty.Close() defer tty.Close()
if _, err := fmt.Fprint(tty, debugBanner); err != nil { if _, err := fmt.Fprint(tty, debugBanner); err != nil {
log.Infof("Unable to print to /dev/ttyS0: %v", err) log.With(zap.Error(err)).Errorf("Unable to print to /dev/ttyS0")
} }
} }

45
debugd/internal/debugd/metadata/cloudprovider/cloudprovider.go
View File

@ -11,11 +11,7 @@ import (
"fmt" "fmt"
"net" "net"
awscloud "github.com/edgelesssys/constellation/v2/internal/cloud/aws"
azurecloud "github.com/edgelesssys/constellation/v2/internal/cloud/azure"
gcpcloud "github.com/edgelesssys/constellation/v2/internal/cloud/gcp"
"github.com/edgelesssys/constellation/v2/internal/cloud/metadata" "github.com/edgelesssys/constellation/v2/internal/cloud/metadata"
qemucloud "github.com/edgelesssys/constellation/v2/internal/cloud/qemu"
"github.com/edgelesssys/constellation/v2/internal/deploy/ssh" "github.com/edgelesssys/constellation/v2/internal/deploy/ssh"
"github.com/edgelesssys/constellation/v2/internal/role" "github.com/edgelesssys/constellation/v2/internal/role"
) )
@ -34,46 +30,9 @@ type Fetcher struct {
metaAPI providerMetadata metaAPI providerMetadata
} }
// NewGCP creates a new GCP fetcher. func New(cloud providerMetadata) *Fetcher {
func NewGCP(ctx context.Context) (*Fetcher, error) {
gcpClient, err := gcpcloud.NewClient(ctx)
if err != nil {
return nil, err
}
metaAPI := gcpcloud.New(gcpClient)
return &Fetcher{ return &Fetcher{
metaAPI: metaAPI, metaAPI: cloud,
}, nil
}
// NewAzure creates a new Azure fetcher.
func NewAWS(ctx context.Context) (*Fetcher, error) {
metaAPI, err := awscloud.New(ctx)
if err != nil {
return nil, err
}
return &Fetcher{
metaAPI: metaAPI,
}, nil
}
// NewAzure creates a new Azure fetcher.
func NewAzure(ctx context.Context) (*Fetcher, error) {
metaAPI, err := azurecloud.NewMetadata(ctx)
if err != nil {
return nil, err
}
return &Fetcher{
metaAPI: metaAPI,
}, nil
}
func NewQEMU() *Fetcher {
return &Fetcher{
metaAPI: &qemucloud.Metadata{},
} }
} }

5
debugd/internal/debugd/metadata/cloudprovider/cloudprovider_test.go
View File

@ -226,7 +226,6 @@ type stubMetadata struct {
getInstanceErr error getInstanceErr error
getLBEndpointRes string getLBEndpointRes string
getLBEndpointErr error getLBEndpointErr error
supportedRes bool
} }
func (m *stubMetadata) List(ctx context.Context) ([]metadata.InstanceMetadata, error) { func (m *stubMetadata) List(ctx context.Context) ([]metadata.InstanceMetadata, error) {
@ -244,7 +243,3 @@ func (m *stubMetadata) GetInstance(ctx context.Context, providerID string) (meta
func (m *stubMetadata) GetLoadBalancerEndpoint(ctx context.Context) (string, error) { func (m *stubMetadata) GetLoadBalancerEndpoint(ctx context.Context) (string, error) {
return m.getLBEndpointRes, m.getLBEndpointErr return m.getLBEndpointRes, m.getLBEndpointErr
} }
func (m *stubMetadata) Supported() bool {
return m.supportedRes
}

5
disk-mapper/cmd/main.go
View File

@ -101,11 +101,12 @@ func main() {
log.With(zap.Error(err)).Fatalf("Unable to resolve GCP state disk path") log.With(zap.Error(err)).Fatalf("Unable to resolve GCP state disk path")
} }
issuer = gcp.NewIssuer() issuer = gcp.NewIssuer()
gcpClient, err := gcpcloud.NewClient(context.Background()) gcpMeta, err := gcpcloud.New(context.Background())
if err != nil { if err != nil {
log.With(zap.Error).Fatalf("Failed to create GCP client") log.With(zap.Error).Fatalf("Failed to create GCP client")
} }
metadataAPI = gcpcloud.New(gcpClient) defer gcpMeta.Close()
metadataAPI = gcpMeta
case cloudprovider.QEMU: case cloudprovider.QEMU:
diskPath = qemuStateDiskPath diskPath = qemuStateDiskPath

16
internal/azureshared/doc.go
View File

@ -1,16 +0,0 @@
/*
Copyright (c) Edgeless Systems GmbH
SPDX-License-Identifier: AGPL-3.0-only
*/
package azureshared
/*
Package azureshared contains code that is related to Microsoft Azure
and is used by multiple microservices.
This package is intended to have a minimal size and surface. If you
have Azure related code that is not shared by multiple microservices,
please keep the code in the microservice's internal package.
*/

81
internal/cloud/aws/ccm.go
View File

@ -1,81 +0,0 @@
/*
Copyright (c) Edgeless Systems GmbH
SPDX-License-Identifier: AGPL-3.0-only
*/
package aws
import (
"context"
"github.com/edgelesssys/constellation/v2/internal/cloud/metadata"
"github.com/edgelesssys/constellation/v2/internal/kubernetes"
"github.com/edgelesssys/constellation/v2/internal/versions"
k8s "k8s.io/api/core/v1"
)
// TODO: Implement for AWS.
// CloudControllerManager holds the AWS cloud-controller-manager configuration.
type CloudControllerManager struct{}
// Image returns the container image used to provide cloud-controller-manager for the cloud-provider.
func (c CloudControllerManager) Image(k8sVersion versions.ValidK8sVersion) (string, error) {
return "", nil
}
// Path returns the path used by cloud-controller-manager executable within the container image.
func (c CloudControllerManager) Path() string {
return "/aws-cloud-controller-manager"
}
// Name returns the cloud-provider name as used by k8s cloud-controller-manager (k8s.gcr.io/cloud-controller-manager).
func (c CloudControllerManager) Name() string {
return "aws"
}
// ExtraArgs returns a list of arguments to append to the cloud-controller-manager command.
func (c CloudControllerManager) ExtraArgs() []string {
return []string{}
}
// ConfigMaps returns a list of ConfigMaps to deploy together with the k8s cloud-controller-manager
// Reference: https://kubernetes.io/docs/concepts/configuration/configmap/ .
func (c CloudControllerManager) ConfigMaps() (kubernetes.ConfigMaps, error) {
return kubernetes.ConfigMaps{}, nil
}
// Secrets returns a list of secrets to deploy together with the k8s cloud-controller-manager.
// Reference: https://kubernetes.io/docs/concepts/configuration/secret/ .
func (c CloudControllerManager) Secrets(ctx context.Context, providerID, cloudServiceAccountURI string) (kubernetes.Secrets, error) {
return kubernetes.Secrets{}, nil
}
// Volumes returns a list of volumes to deploy together with the k8s cloud-controller-manager.
// Reference: https://kubernetes.io/docs/concepts/storage/volumes/ .
func (c CloudControllerManager) Volumes() []k8s.Volume {
return []k8s.Volume{}
}
// VolumeMounts a list of of volume mounts to deploy together with the k8s cloud-controller-manager.
func (c CloudControllerManager) VolumeMounts() []k8s.VolumeMount {
return []k8s.VolumeMount{}
}
// Env returns a list of k8s environment key-value pairs to deploy together with the k8s cloud-controller-manager.
func (c CloudControllerManager) Env() []k8s.EnvVar {
return []k8s.EnvVar{}
}
// PrepareInstance is called on every instance before deploying the cloud-controller-manager.
// Allows for cloud-provider specific hooks.
func (c CloudControllerManager) PrepareInstance(instance metadata.InstanceMetadata, vpnIP string) error {
// no specific hook required.
return nil
}
// Supported is used to determine if cloud controller manager is implemented for this cloud provider.
func (c CloudControllerManager) Supported() bool {
return false
}

34
internal/cloud/aws/cloudnodemanager.go
View File

@ -1,34 +0,0 @@
/*
Copyright (c) Edgeless Systems GmbH
SPDX-License-Identifier: AGPL-3.0-only
*/
package aws
import "github.com/edgelesssys/constellation/v2/internal/versions"
// TODO: Implement for AWS.
// CloudNodeManager holds the AWS cloud-node-manager configuration.
type CloudNodeManager struct{}
// Image returns the container image used to provide cloud-node-manager for the cloud-provider.
func (c *CloudNodeManager) Image(k8sVersion versions.ValidK8sVersion) (string, error) {
return "", nil
}
// Path returns the path used by cloud-node-manager executable within the container image.
func (c *CloudNodeManager) Path() string {
return ""
}
// ExtraArgs returns a list of arguments to append to the cloud-node-manager command.
func (c *CloudNodeManager) ExtraArgs() []string {
return []string{}
}
// Supported is used to determine if cloud node manager is implemented for this cloud provider.
func (c *CloudNodeManager) Supported() bool {
return false
}

10
internal/cloud/aws/metadata.go
View File

@ -70,11 +70,6 @@ func New(ctx context.Context) (*Metadata, error) {
}, nil }, nil
} }
// Supported is used to determine if metadata API is implemented for this cloud provider.
func (m *Metadata) Supported() bool {
return true
}
// List retrieves all instances belonging to the current Constellation. // List retrieves all instances belonging to the current Constellation.
func (m *Metadata) List(ctx context.Context) ([]metadata.InstanceMetadata, error) { func (m *Metadata) List(ctx context.Context) ([]metadata.InstanceMetadata, error) {
uid, err := readInstanceTag(ctx, m.imds, cloud.TagUID) uid, err := readInstanceTag(ctx, m.imds, cloud.TagUID)
@ -142,11 +137,6 @@ func (m *Metadata) UID(ctx context.Context) (string, error) {
return readInstanceTag(ctx, m.imds, cloud.TagUID) return readInstanceTag(ctx, m.imds, cloud.TagUID)
} }
// SupportsLoadBalancer returns true if the cloud provider supports load balancers.
func (m *Metadata) SupportsLoadBalancer() bool {
return true
}
// GetLoadBalancerEndpoint returns the endpoint of the load balancer. // GetLoadBalancerEndpoint returns the endpoint of the load balancer.
func (m *Metadata) GetLoadBalancerEndpoint(ctx context.Context) (string, error) { func (m *Metadata) GetLoadBalancerEndpoint(ctx context.Context) (string, error) {
uid, err := readInstanceTag(ctx, m.imds, cloud.TagUID) uid, err := readInstanceTag(ctx, m.imds, cloud.TagUID)

188
internal/cloud/azure/ccm.go
View File

@ -1,188 +0,0 @@
/*
Copyright (c) Edgeless Systems GmbH
SPDX-License-Identifier: AGPL-3.0-only
*/
package azure
import (
"context"
"encoding/json"
"github.com/edgelesssys/constellation/v2/internal/azureshared"
"github.com/edgelesssys/constellation/v2/internal/kubernetes"
"github.com/edgelesssys/constellation/v2/internal/versions"
k8s "k8s.io/api/core/v1"
meta "k8s.io/apimachinery/pkg/apis/meta/v1"
)
type ccmMetadata interface {
GetNetworkSecurityGroupName(ctx context.Context) (string, error)
GetLoadBalancerName(ctx context.Context) (string, error)
}
// CloudControllerManager holds the Azure cloud-controller-manager configuration.
type CloudControllerManager struct {
metadata ccmMetadata
}
func NewCloudControllerManager(metadata ccmMetadata) *CloudControllerManager {
return &CloudControllerManager{
metadata: metadata,
}
}
// Image returns the container image used to provide cloud-controller-manager for the cloud-provider.
func (c *CloudControllerManager) Image(k8sVersion versions.ValidK8sVersion) (string, error) {
return versions.VersionConfigs[k8sVersion].CloudControllerManagerImageAzure, nil
}
// Path returns the path used by cloud-controller-manager executable within the container image.
func (c *CloudControllerManager) Path() string {
return "cloud-controller-manager"
}
// Name returns the cloud-provider name as used by k8s cloud-controller-manager (k8s.gcr.io/cloud-controller-manager).
func (c *CloudControllerManager) Name() string {
return "azure"
}
// ExtraArgs returns a list of arguments to append to the cloud-controller-manager command.
func (c *CloudControllerManager) ExtraArgs() []string {
return []string{
"--controllers=*,-cloud-node",
"--cloud-config=/etc/azure/azure.json",
"--allocate-node-cidrs=false",
"--configure-cloud-routes=true",
}
}
// ConfigMaps returns a list of ConfigMaps to deploy together with the k8s cloud-controller-manager
// Reference: https://kubernetes.io/docs/concepts/configuration/configmap/ .
func (c *CloudControllerManager) ConfigMaps() (kubernetes.ConfigMaps, error) {
return kubernetes.ConfigMaps{}, nil
}
// Secrets returns a list of secrets to deploy together with the k8s cloud-controller-manager.
// Reference: https://kubernetes.io/docs/concepts/configuration/secret/ .
func (c *CloudControllerManager) Secrets(ctx context.Context, providerID string, cloudServiceAccountURI string) (kubernetes.Secrets, error) {
// Azure CCM expects cloud provider config to contain cluster configuration and service principal client secrets
// reference: https://kubernetes-sigs.github.io/cloud-provider-azure/install/configs/
subscriptionID, resourceGroup, err := azureshared.BasicsFromProviderID(providerID)
if err != nil {
return kubernetes.Secrets{}, err
}
creds, err := azureshared.ApplicationCredentialsFromURI(cloudServiceAccountURI)
if err != nil {
return kubernetes.Secrets{}, err
}
vmType := "standard"
if _, _, _, _, err := azureshared.ScaleSetInformationFromProviderID(providerID); err == nil {
vmType = "vmss"
}
securityGroupName, err := c.metadata.GetNetworkSecurityGroupName(ctx)
if err != nil {
return kubernetes.Secrets{}, err
}
loadBalancerName, err := c.metadata.GetLoadBalancerName(ctx)
if err != nil {
return kubernetes.Secrets{}, err
}
config := cloudConfig{
Cloud: "AzurePublicCloud",
TenantID: creds.TenantID,
SubscriptionID: subscriptionID,
ResourceGroup: resourceGroup,
LoadBalancerSku: "standard",
SecurityGroupName: securityGroupName,
LoadBalancerName: loadBalancerName,
UseInstanceMetadata: true,
VMType: vmType,
Location: creds.Location,
AADClientID: creds.AppClientID,
AADClientSecret: creds.ClientSecretValue,
}
rawConfig, err := json.Marshal(config)
if err != nil {
return kubernetes.Secrets{}, err
}
return kubernetes.Secrets{
&k8s.Secret{
TypeMeta: meta.TypeMeta{
Kind: "Secret",
APIVersion: "v1",
},
ObjectMeta: meta.ObjectMeta{
Name: "azureconfig",
Namespace: "kube-system",
},
Data: map[string][]byte{
"azure.json": rawConfig,
},
},
}, nil
}
// Volumes returns a list of volumes to deploy together with the k8s cloud-controller-manager.
// Reference: https://kubernetes.io/docs/concepts/storage/volumes/ .
func (c *CloudControllerManager) Volumes() []k8s.Volume {
return []k8s.Volume{
{
Name: "azureconfig",
VolumeSource: k8s.VolumeSource{
Secret: &k8s.SecretVolumeSource{
SecretName: "azureconfig",
},
},
},
}
}
// VolumeMounts a list of of volume mounts to deploy together with the k8s cloud-controller-manager.
func (c *CloudControllerManager) VolumeMounts() []k8s.VolumeMount {
return []k8s.VolumeMount{
{
Name: "azureconfig",
ReadOnly: true,
MountPath: "/etc/azure",
},
}
}
// Env returns a list of k8s environment key-value pairs to deploy together with the k8s cloud-controller-manager.
func (c *CloudControllerManager) Env() []k8s.EnvVar {
return []k8s.EnvVar{}
}
// Supported is used to determine if cloud controller manager is implemented for this cloud provider.
func (c *CloudControllerManager) Supported() bool {
return true
}
type cloudConfig struct {
Cloud string `json:"cloud,omitempty"`
TenantID string `json:"tenantId,omitempty"`
SubscriptionID string `json:"subscriptionId,omitempty"`
ResourceGroup string `json:"resourceGroup,omitempty"`
Location string `json:"location,omitempty"`
SubnetName string `json:"subnetName,omitempty"`
SecurityGroupName string `json:"securityGroupName,omitempty"`
SecurityGroupResourceGroup string `json:"securityGroupResourceGroup,omitempty"`
LoadBalancerName string `json:"loadBalancerName,omitempty"`
LoadBalancerSku string `json:"loadBalancerSku,omitempty"`
VNetName string `json:"vnetName,omitempty"`
VNetResourceGroup string `json:"vnetResourceGroup,omitempty"`
CloudProviderBackoff bool `json:"cloudProviderBackoff,omitempty"`
UseInstanceMetadata bool `json:"useInstanceMetadata,omitempty"`
VMType string `json:"vmType,omitempty"`
AADClientID string `json:"aadClientId,omitempty"`
AADClientSecret string `json:"aadClientSecret,omitempty"`
}

122
internal/cloud/azure/ccm_test.go
View File

@ -1,122 +0,0 @@
/*
Copyright (c) Edgeless Systems GmbH
SPDX-License-Identifier: AGPL-3.0-only
*/
package azure
import (
"context"
"errors"
"testing"
"github.com/edgelesssys/constellation/v2/internal/kubernetes"
"github.com/edgelesssys/constellation/v2/internal/versions"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
k8s "k8s.io/api/core/v1"
meta "k8s.io/apimachinery/pkg/apis/meta/v1"
)
func TestSecrets(t *testing.T) {
someErr := errors.New("some error")
testCases := map[string]struct {
providerID string
metadata ccmMetadata
cloudServiceAccountURI string
wantSecrets kubernetes.Secrets
wantErr bool
}{
"Secrets works for scale sets": {
providerID: "azure:///subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/virtualMachineScaleSets/scale-set-name/virtualMachines/instance-id",
cloudServiceAccountURI: "serviceaccount://azure?tenant_id=tenant-id&client_id=client-id&client_secret=client-secret&location=location",
metadata: &ccmMetadataStub{loadBalancerName: "load-balancer-name", networkSecurityGroupName: "network-security-group-name"},
wantSecrets: kubernetes.Secrets{
&k8s.Secret{
TypeMeta: meta.TypeMeta{
Kind: "Secret",
APIVersion: "v1",
},
ObjectMeta: meta.ObjectMeta{
Name: "azureconfig",
Namespace: "kube-system",
},
Data: map[string][]byte{
"azure.json": []byte(`{"cloud":"AzurePublicCloud","tenantId":"tenant-id","subscriptionId":"subscription-id","resourceGroup":"resource-group","location":"location","securityGroupName":"network-security-group-name","loadBalancerName":"load-balancer-name","loadBalancerSku":"standard","useInstanceMetadata":true,"vmType":"vmss","aadClientId":"client-id","aadClientSecret":"client-secret"}`),
},
},
},
},
"cannot get load balancer Name": {
providerID: "azure:///subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/virtualMachineScaleSets/scale-set-name/virtualMachines/instance-id",
cloudServiceAccountURI: "serviceaccount://azure?tenant_id=tenant-id&client_id=client-id&client_secret=client-secret&location=location",
metadata: &ccmMetadataStub{getLoadBalancerNameErr: someErr},
wantErr: true,
},
"cannot get network security group name": {
providerID: "azure:///subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/virtualMachineScaleSets/scale-set-name/virtualMachines/instance-id",
cloudServiceAccountURI: "serviceaccount://azure?tenant_id=tenant-id&client_id=client-id&client_secret=client-secret&location=location",
metadata: &ccmMetadataStub{getNetworkSecurityGroupNameErr: someErr},
wantErr: true,
},
"invalid providerID fails": {
providerID: "invalid",
metadata: &ccmMetadataStub{},
wantErr: true,
},
"invalid cloudServiceAccountURI fails": {
providerID: "azure:///subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/virtualMachines/instance-name",
metadata: &ccmMetadataStub{},
cloudServiceAccountURI: "invalid",
wantErr: true,
},
}
for name, tc := range testCases {
t.Run(name, func(t *testing.T) {
assert := assert.New(t)
require := require.New(t)
cloud := NewCloudControllerManager(tc.metadata)
secrets, err := cloud.Secrets(context.Background(), tc.providerID, tc.cloudServiceAccountURI)
if tc.wantErr {
assert.Error(err)
return
}
require.NoError(err)
assert.Equal(tc.wantSecrets, secrets)
})
}
}
func TestTrivialCCMFunctions(t *testing.T) {
assert := assert.New(t)
cloud := CloudControllerManager{}
assert.NotEmpty(cloud.Image(versions.Default))
assert.NotEmpty(cloud.Path())
assert.NotEmpty(cloud.Name())
assert.NotEmpty(cloud.ExtraArgs())
assert.Empty(cloud.ConfigMaps())
assert.NotEmpty(cloud.Volumes())
assert.NotEmpty(cloud.VolumeMounts())
assert.Empty(cloud.Env())
assert.True(cloud.Supported())
}
type ccmMetadataStub struct {
networkSecurityGroupName string
loadBalancerName string
getNetworkSecurityGroupNameErr error
getLoadBalancerNameErr error
}
func (c *ccmMetadataStub) GetNetworkSecurityGroupName(ctx context.Context) (string, error) {
return c.networkSecurityGroupName, c.getNetworkSecurityGroupNameErr
}
func (c *ccmMetadataStub) GetLoadBalancerName(ctx context.Context) (string, error) {
return c.loadBalancerName, c.getLoadBalancerNameErr
}

37
internal/cloud/azure/cloudnodemanager.go
View File

@ -1,37 +0,0 @@
/*
Copyright (c) Edgeless Systems GmbH
SPDX-License-Identifier: AGPL-3.0-only
*/
package azure
import (
"github.com/edgelesssys/constellation/v2/internal/versions"
)
// CloudNodeManager holds the Azure cloud-node-manager configuration.
// reference: https://raw.githubusercontent.com/kubernetes-sigs/cloud-provider-azure/master/examples/out-of-tree/cloud-node-manager.yaml .
type CloudNodeManager struct{}
// Image returns the container image used to provide cloud-node-manager for the cloud-provider.
func (c *CloudNodeManager) Image(k8sVersion versions.ValidK8sVersion) (string, error) {
return versions.VersionConfigs[k8sVersion].CloudNodeManagerImageAzure, nil
}
// Path returns the path used by cloud-node-manager executable within the container image.
func (c *CloudNodeManager) Path() string {
return "cloud-node-manager"
}
// ExtraArgs returns a list of arguments to append to the cloud-node-manager command.
func (c *CloudNodeManager) ExtraArgs() []string {
return []string{
"--wait-routes=true",
}
}
// Supported is used to determine if cloud node manager is implemented for this cloud provider.
func (c *CloudNodeManager) Supported() bool {
return true
}

24
internal/cloud/azure/cloudnodemanager_test.go
View File

@ -1,24 +0,0 @@
/*
Copyright (c) Edgeless Systems GmbH
SPDX-License-Identifier: AGPL-3.0-only
*/
package azure
import (
"testing"
"github.com/edgelesssys/constellation/v2/internal/versions"
"github.com/stretchr/testify/assert"
)
func TestTrivialCNMFunctions(t *testing.T) {
assert := assert.New(t)
cloud := CloudNodeManager{}
assert.NotEmpty(cloud.Image(versions.Default))
assert.NotEmpty(cloud.Path())
assert.NotEmpty(cloud.ExtraArgs())
assert.True(cloud.Supported())
}

72
internal/cloud/azure/metadata.go
View File

@ -8,6 +8,7 @@ package azure
import ( import (
"context" "context"
"encoding/json"
"fmt" "fmt"
"net/http" "net/http"
"regexp" "regexp"
@ -18,6 +19,7 @@ import (
"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork" "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork"
"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources" "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources"
"github.com/edgelesssys/constellation/v2/internal/cloud" "github.com/edgelesssys/constellation/v2/internal/cloud"
"github.com/edgelesssys/constellation/v2/internal/cloud/azureshared"
"github.com/edgelesssys/constellation/v2/internal/cloud/metadata" "github.com/edgelesssys/constellation/v2/internal/cloud/metadata"
) )
@ -201,11 +203,6 @@ func (m *Metadata) getLoadBalancer(ctx context.Context) (*armnetwork.LoadBalance
return nil, fmt.Errorf("could not get any load balancer") return nil, fmt.Errorf("could not get any load balancer")
} }
// SupportsLoadBalancer returns true if the cloud provider supports load balancers.
func (m *Metadata) SupportsLoadBalancer() bool {
return true
}
// GetLoadBalancerName returns the load balancer name of the resource group. // GetLoadBalancerName returns the load balancer name of the resource group.
func (m *Metadata) GetLoadBalancerName(ctx context.Context) (string, error) { func (m *Metadata) GetLoadBalancerName(ctx context.Context) (string, error) {
lb, err := m.getLoadBalancer(ctx) lb, err := m.getLoadBalancer(ctx)
@ -265,9 +262,48 @@ func (m *Metadata) GetLoadBalancerEndpoint(ctx context.Context) (string, error)
return *resp.Properties.IPAddress, nil return *resp.Properties.IPAddress, nil
} }
// Supported is used to determine if metadata API is implemented for this cloud provider. // GetCCMConfig returns the configuration needed for the CCM on Azure.
func (m *Metadata) Supported() bool { func (m *Metadata) GetCCMConfig(ctx context.Context, providerID string, cloudServiceAccountURI string) ([]byte, error) {
return true subscriptionID, resourceGroup, err := azureshared.BasicsFromProviderID(providerID)
if err != nil {
return nil, err
}
creds, err := azureshared.ApplicationCredentialsFromURI(cloudServiceAccountURI)
if err != nil {
return nil, err
}
vmType := "standard"
if _, _, _, _, err := azureshared.ScaleSetInformationFromProviderID(providerID); err == nil {
vmType = "vmss"
}
securityGroupName, err := m.GetNetworkSecurityGroupName(ctx)
if err != nil {
return nil, err
}
loadBalancerName, err := m.GetLoadBalancerName(ctx)
if err != nil {
return nil, err
}
config := cloudConfig{
Cloud: "AzurePublicCloud",
TenantID: creds.TenantID,
SubscriptionID: subscriptionID,
ResourceGroup: resourceGroup,
LoadBalancerSku: "standard",
SecurityGroupName: securityGroupName,
LoadBalancerName: loadBalancerName,
UseInstanceMetadata: true,
VMType: vmType,
Location: creds.Location,
AADClientID: creds.AppClientID,
AADClientSecret: creds.ClientSecretValue,
}
return json.Marshal(config)
} }
// providerID retrieves the current instances providerID. // providerID retrieves the current instances providerID.
@ -343,3 +379,23 @@ func extractSSHKeys(sshConfig armcomputev2.SSHConfiguration) map[string][]string
} }
return sshKeys return sshKeys
} }
type cloudConfig struct {
Cloud string `json:"cloud,omitempty"`
TenantID string `json:"tenantId,omitempty"`
SubscriptionID string `json:"subscriptionId,omitempty"`
ResourceGroup string `json:"resourceGroup,omitempty"`
Location string `json:"location,omitempty"`
SubnetName string `json:"subnetName,omitempty"`
SecurityGroupName string `json:"securityGroupName,omitempty"`
SecurityGroupResourceGroup string `json:"securityGroupResourceGroup,omitempty"`
LoadBalancerName string `json:"loadBalancerName,omitempty"`
LoadBalancerSku string `json:"loadBalancerSku,omitempty"`
VNetName string `json:"vnetName,omitempty"`
VNetResourceGroup string `json:"vnetResourceGroup,omitempty"`
CloudProviderBackoff bool `json:"cloudProviderBackoff,omitempty"`
UseInstanceMetadata bool `json:"useInstanceMetadata,omitempty"`
VMType string `json:"vmType,omitempty"`
AADClientID string `json:"aadClientId,omitempty"`
AADClientSecret string `json:"aadClientSecret,omitempty"`
}

6
internal/cloud/azure/metadata_test.go
View File

@ -463,12 +463,6 @@ func TestGetLoadBalancerEndpoint(t *testing.T) {
} }
} }
func TestMetadataSupported(t *testing.T) {
assert := assert.New(t)
metadata := Metadata{}
assert.True(metadata.Supported())
}
func TestProviderID(t *testing.T) { func TestProviderID(t *testing.T) {
testCases := map[string]struct { testCases := map[string]struct {
imdsAPI imdsAPI imdsAPI imdsAPI

12
internal/cloud/azure/scaleset.go
View File

@ -15,8 +15,8 @@ import (
"github.com/Azure/azure-sdk-for-go/sdk/azcore/to" "github.com/Azure/azure-sdk-for-go/sdk/azcore/to"
armcomputev2 "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v2" armcomputev2 "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v2"
"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork" "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork"
"github.com/edgelesssys/constellation/v2/internal/azureshared"
"github.com/edgelesssys/constellation/v2/internal/cloud" "github.com/edgelesssys/constellation/v2/internal/cloud"
"github.com/edgelesssys/constellation/v2/internal/cloud/azureshared"
"github.com/edgelesssys/constellation/v2/internal/cloud/metadata" "github.com/edgelesssys/constellation/v2/internal/cloud/metadata"
"github.com/edgelesssys/constellation/v2/internal/role" "github.com/edgelesssys/constellation/v2/internal/role"
) )
@ -35,12 +35,8 @@ func (m *Metadata) getScaleSetVM(ctx context.Context, providerID string) (metada
if err != nil { if err != nil {
return metadata.InstanceMetadata{}, err return metadata.InstanceMetadata{}, err
} }
publicIPAddress, err := m.getScaleSetVMPublicIPAddress(ctx, resourceGroup, scaleSet, instanceID, networkInterfaces)
if err != nil {
return metadata.InstanceMetadata{}, err
}
return convertScaleSetVMToCoreInstance(vmResp.VirtualMachineScaleSetVM, networkInterfaces, publicIPAddress) return convertScaleSetVMToCoreInstance(vmResp.VirtualMachineScaleSetVM, networkInterfaces)
} }
// listScaleSetVMs lists all scale set VMs in the current resource group. // listScaleSetVMs lists all scale set VMs in the current resource group.
@ -70,7 +66,7 @@ func (m *Metadata) listScaleSetVMs(ctx context.Context, resourceGroup string) ([
if err != nil { if err != nil {
return nil, err return nil, err
} }
instance, err := convertScaleSetVMToCoreInstance(*vm, interfaces, "") instance, err := convertScaleSetVMToCoreInstance(*vm, interfaces)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -84,7 +80,6 @@ func (m *Metadata) listScaleSetVMs(ctx context.Context, resourceGroup string) ([
// convertScaleSetVMToCoreInstance converts an azure scale set virtual machine with interface configurations into a core.Instance. // convertScaleSetVMToCoreInstance converts an azure scale set virtual machine with interface configurations into a core.Instance.
func convertScaleSetVMToCoreInstance(vm armcomputev2.VirtualMachineScaleSetVM, networkInterfaces []armnetwork.Interface, func convertScaleSetVMToCoreInstance(vm armcomputev2.VirtualMachineScaleSetVM, networkInterfaces []armnetwork.Interface,
publicIPAddress string,
) (metadata.InstanceMetadata, error) { ) (metadata.InstanceMetadata, error) {
if vm.ID == nil { if vm.ID == nil {
return metadata.InstanceMetadata{}, errors.New("retrieving instance from armcompute API client returned no instance ID") return metadata.InstanceMetadata{}, errors.New("retrieving instance from armcompute API client returned no instance ID")
@ -108,7 +103,6 @@ func convertScaleSetVMToCoreInstance(vm armcomputev2.VirtualMachineScaleSetVM, n
ProviderID: "azure://" + *vm.ID, ProviderID: "azure://" + *vm.ID,
Role: extractScaleSetVMRole(vm.Tags), Role: extractScaleSetVMRole(vm.Tags),
VPCIP: extractVPCIP(networkInterfaces), VPCIP: extractVPCIP(networkInterfaces),
PublicIP: publicIPAddress,
SSHKeys: sshKeys, SSHKeys: sshKeys,
}, nil }, nil
} }

5
internal/cloud/azure/scaleset_test.go
View File

@ -155,7 +155,6 @@ func TestConvertScaleSetVMToCoreInstance(t *testing.T) {
testCases := map[string]struct { testCases := map[string]struct {
inVM armcomputev2.VirtualMachineScaleSetVM inVM armcomputev2.VirtualMachineScaleSetVM
inInterface []armnetwork.Interface inInterface []armnetwork.Interface
inPublicIP string
wantErr bool wantErr bool
wantInstance metadata.InstanceMetadata wantInstance metadata.InstanceMetadata
}{ }{
@ -186,12 +185,10 @@ func TestConvertScaleSetVMToCoreInstance(t *testing.T) {
}, },
}, },
}, },
inPublicIP: "192.0.2.100",
wantInstance: metadata.InstanceMetadata{ wantInstance: metadata.InstanceMetadata{
Name: "scale-set-name-instance-id", Name: "scale-set-name-instance-id",
ProviderID: "azure:///subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/virtualMachineScaleSets/scale-set-name/virtualMachines/instance-id", ProviderID: "azure:///subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/virtualMachineScaleSets/scale-set-name/virtualMachines/instance-id",
VPCIP: "192.0.2.0", VPCIP: "192.0.2.0",
PublicIP: "192.0.2.100",
SSHKeys: map[string][]string{}, SSHKeys: map[string][]string{},
}, },
}, },
@ -206,7 +203,7 @@ func TestConvertScaleSetVMToCoreInstance(t *testing.T) {
assert := assert.New(t) assert := assert.New(t)
require := require.New(t) require := require.New(t)
instance, err := convertScaleSetVMToCoreInstance(tc.inVM, tc.inInterface, tc.inPublicIP) instance, err := convertScaleSetVMToCoreInstance(tc.inVM, tc.inInterface)
if tc.wantErr { if tc.wantErr {
assert.Error(err) assert.Error(err)

0
internal/azureshared/appcredentials.go → internal/cloud/azureshared/appcredentials.go
View File

0
internal/azureshared/appcredentials_test.go → internal/cloud/azureshared/appcredentials_test.go
View File

16
internal/cloud/azureshared/doc.go Normal file
View File

@ -0,0 +1,16 @@
/*
Copyright (c) Edgeless Systems GmbH
SPDX-License-Identifier: AGPL-3.0-only
*/
/*
Package gcpshared contains code to parse and define data types
relevant for Microsoft Azure.
This package is intended to have a minimal size and surface. If you
have Azure related code that is not shared by multiple applications,
or if the code interacts with the GCP API, please keep the code in
the application's internal package or add it to the Azure cloud package.
*/
package azureshared

0
internal/azureshared/metadata.go → internal/cloud/azureshared/metadata.go
View File

0
internal/azureshared/metadata_test.go → internal/cloud/azureshared/metadata_test.go
View File

183
internal/cloud/gcp/ccm.go
View File

@ -1,183 +0,0 @@
/*
Copyright (c) Edgeless Systems GmbH
SPDX-License-Identifier: AGPL-3.0-only
*/
package gcp
import (
"context"
"encoding/json"
"fmt"
"strings"
"github.com/edgelesssys/constellation/v2/internal/gcpshared"
"github.com/edgelesssys/constellation/v2/internal/kubernetes"
"github.com/edgelesssys/constellation/v2/internal/versions"
k8s "k8s.io/api/core/v1"
v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)
// CloudControllerManager holds the gcp cloud-controller-manager configuration.
type CloudControllerManager struct {
uid string
projectID string
}
// NewCloudControllerManager returns an initialized cloud controller manager configuration struct for GCP.
func NewCloudControllerManager(ctx context.Context, metadata *Metadata) (*CloudControllerManager, error) {
uid, err := metadata.api.UID(ctx)
if err != nil {
return nil, fmt.Errorf("getting uid from metadata: %w", err)
}
projectID, err := metadata.api.RetrieveProjectID()
if err != nil {
return nil, fmt.Errorf("getting project id from metadata: %w", err)
}
return &CloudControllerManager{
uid: uid,
projectID: projectID,
}, nil
}
// Image returns the container image used to provide cloud-controller-manager for the cloud-provider.
func (c *CloudControllerManager) Image(k8sVersion versions.ValidK8sVersion) (string, error) {
return versions.VersionConfigs[k8sVersion].CloudControllerManagerImageGCP, nil
}
// Path returns the path used by cloud-controller-manager executable within the container image.
func (c *CloudControllerManager) Path() string {
return "/cloud-controller-manager"
}
// Name returns the cloud-provider name as used by k8s cloud-controller-manager (k8s.gcr.io/cloud-controller-manager).
func (c *CloudControllerManager) Name() string {
return "gce"
}
// ExtraArgs returns a list of arguments to append to the cloud-controller-manager command.
func (c *CloudControllerManager) ExtraArgs() []string {
return []string{
"--use-service-account-credentials",
"--controllers=cloud-node,cloud-node-lifecycle,nodeipam,service,route",
"--cloud-config=/etc/gce/gce.conf",
"--cidr-allocator-type=CloudAllocator",
"--allocate-node-cidrs=true",
"--configure-cloud-routes=false",
}
}
// ConfigMaps returns a list of ConfigMaps to deploy together with the k8s cloud-controller-manager
// Reference: https://kubernetes.io/docs/concepts/configuration/configmap/ .
func (c *CloudControllerManager) ConfigMaps() (kubernetes.ConfigMaps, error) {
// GCP CCM expects cloud config to contain the GCP project-id and other configuration.
// reference: https://github.com/kubernetes/cloud-provider-gcp/blob/master/cluster/gce/gci/configure-helper.sh#L791-L892
var config strings.Builder
config.WriteString("[global]\n")
config.WriteString(fmt.Sprintf("project-id = %s\n", c.projectID))
config.WriteString("use-metadata-server = true\n")
config.WriteString(fmt.Sprintf("node-tags = constellation-%s\n", c.uid))
return kubernetes.ConfigMaps{
&k8s.ConfigMap{
TypeMeta: v1.TypeMeta{
Kind: "ConfigMap",
APIVersion: "v1",
},
ObjectMeta: v1.ObjectMeta{
Name: "gceconf",
Namespace: "kube-system",
},
Data: map[string]string{
"gce.conf": config.String(),
},
},
}, nil
}
// Secrets returns a list of secrets to deploy together with the k8s cloud-controller-manager.
// Reference: https://kubernetes.io/docs/concepts/configuration/secret/ .
func (c *CloudControllerManager) Secrets(_ context.Context, _ string, cloudServiceAccountURI string) (kubernetes.Secrets, error) {
serviceAccountKey, err := gcpshared.ServiceAccountKeyFromURI(cloudServiceAccountURI)
if err != nil {
return kubernetes.Secrets{}, err
}
rawKey, err := json.Marshal(serviceAccountKey)
if err != nil {
return kubernetes.Secrets{}, err
}
return kubernetes.Secrets{
&k8s.Secret{
TypeMeta: v1.TypeMeta{
Kind: "Secret",
APIVersion: "v1",
},
ObjectMeta: v1.ObjectMeta{
Name: "gcekey",
Namespace: "kube-system",
},
Data: map[string][]byte{
"key.json": rawKey,
},
},
}, nil
}
// Volumes returns a list of volumes to deploy together with the k8s cloud-controller-manager.
// Reference: https://kubernetes.io/docs/concepts/storage/volumes/ .
func (c *CloudControllerManager) Volumes() []k8s.Volume {
return []k8s.Volume{
{
Name: "gceconf",
VolumeSource: k8s.VolumeSource{
ConfigMap: &k8s.ConfigMapVolumeSource{
LocalObjectReference: k8s.LocalObjectReference{
Name: "gceconf",
},
},
},
},
{
Name: "gcekey",
VolumeSource: k8s.VolumeSource{
Secret: &k8s.SecretVolumeSource{
SecretName: "gcekey",
},
},
},
}
}
// VolumeMounts returns a list of volume mounts to deploy together with the k8s cloud-controller-manager.
func (c *CloudControllerManager) VolumeMounts() []k8s.VolumeMount {
return []k8s.VolumeMount{
{
Name: "gceconf",
ReadOnly: true,
MountPath: "/etc/gce",
},
{
Name: "gcekey",
ReadOnly: true,
MountPath: "/var/secrets/google",
},
}
}
// Env returns a list of k8s environment key-value pairs to deploy together with the k8s cloud-controller-manager.
func (c *CloudControllerManager) Env() []k8s.EnvVar {
return []k8s.EnvVar{
{
Name: "GOOGLE_APPLICATION_CREDENTIALS",
Value: "/var/secrets/google/key.json",
},
}
}
// Supported is used to determine if cloud controller manager is implemented for this cloud provider.
func (c *CloudControllerManager) Supported() bool {
return true
}

148
internal/cloud/gcp/ccm_test.go
View File

@ -1,148 +0,0 @@
/*
Copyright (c) Edgeless Systems GmbH
SPDX-License-Identifier: AGPL-3.0-only
*/
package gcp
import (
"context"
"encoding/json"
"testing"
"github.com/edgelesssys/constellation/v2/internal/cloud/metadata"
"github.com/edgelesssys/constellation/v2/internal/gcpshared"
"github.com/edgelesssys/constellation/v2/internal/kubernetes"
"github.com/edgelesssys/constellation/v2/internal/versions"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
k8s "k8s.io/api/core/v1"
v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)
func TestConfigMaps(t *testing.T) {
testCases := map[string]struct {
instance metadata.InstanceMetadata
wantConfigMaps kubernetes.ConfigMaps
wantErr bool
}{
"ConfigMaps works": {
wantConfigMaps: kubernetes.ConfigMaps{
&k8s.ConfigMap{
TypeMeta: v1.TypeMeta{
Kind: "ConfigMap",
APIVersion: "v1",
},
ObjectMeta: v1.ObjectMeta{
Name: "gceconf",
Namespace: "kube-system",
},
Data: map[string]string{
"gce.conf": `[global]
project-id = project-id
use-metadata-server = true
node-tags = constellation-UID
`,
},
},
},
},
}
for name, tc := range testCases {
t.Run(name, func(t *testing.T) {
assert := assert.New(t)
require := require.New(t)
cloud := CloudControllerManager{
projectID: "project-id",
uid: "UID",
}
configMaps, err := cloud.ConfigMaps()
if tc.wantErr {
assert.Error(err)
return
}
require.NoError(err)
assert.Equal(tc.wantConfigMaps, configMaps)
})
}
}
func TestSecrets(t *testing.T) {
serviceAccountKey := gcpshared.ServiceAccountKey{
Type: "type",
ProjectID: "project-id",
PrivateKeyID: "private-key-id",
PrivateKey: "private-key",
ClientEmail: "client-email",
ClientID: "client-id",
AuthURI: "auth-uri",
TokenURI: "token-uri",
AuthProviderX509CertURL: "auth-provider-x509-cert-url",
ClientX509CertURL: "client-x509-cert-url",
}
rawKey, err := json.Marshal(serviceAccountKey)
require.NoError(t, err)
testCases := map[string]struct {
instance metadata.InstanceMetadata
cloudServiceAccountURI string
wantSecrets kubernetes.Secrets
wantErr bool
}{
"Secrets works": {
cloudServiceAccountURI: "serviceaccount://gcp?type=type&project_id=project-id&private_key_id=private-key-id&private_key=private-key&client_email=client-email&client_id=client-id&auth_uri=auth-uri&token_uri=token-uri&auth_provider_x509_cert_url=auth-provider-x509-cert-url&client_x509_cert_url=client-x509-cert-url",
wantSecrets: kubernetes.Secrets{
&k8s.Secret{
TypeMeta: v1.TypeMeta{
Kind: "Secret",
APIVersion: "v1",
},
ObjectMeta: v1.ObjectMeta{
Name: "gcekey",
Namespace: "kube-system",
},
Data: map[string][]byte{
"key.json": rawKey,
},
},
},
},
"invalid serviceAccountKey fails": {
cloudServiceAccountURI: "invalid",
wantErr: true,
},
}
for name, tc := range testCases {
t.Run(name, func(t *testing.T) {
assert := assert.New(t)
require := require.New(t)
cloud := CloudControllerManager{}
secrets, err := cloud.Secrets(context.Background(), tc.instance.ProviderID, tc.cloudServiceAccountURI)
if tc.wantErr {
assert.Error(err)
return
}
require.NoError(err)
assert.Equal(tc.wantSecrets, secrets)
})
}
}
func TestTrivialCCMFunctions(t *testing.T) {
assert := assert.New(t)
cloud := CloudControllerManager{}
assert.NotEmpty(cloud.Image(versions.Default))
assert.NotEmpty(cloud.Path())
assert.NotEmpty(cloud.Name())
assert.NotEmpty(cloud.ExtraArgs())
assert.NotEmpty(cloud.Volumes())
assert.NotEmpty(cloud.VolumeMounts())
assert.NotEmpty(cloud.Env())
assert.True(cloud.Supported())
}

439
internal/cloud/gcp/client.go
View File

@ -1,439 +0,0 @@
/*
Copyright (c) Edgeless Systems GmbH
SPDX-License-Identifier: AGPL-3.0-only
*/
package gcp
import (
"context"
"errors"
"fmt"
"net"
"regexp"
"strings"
compute "cloud.google.com/go/compute/apiv1"
"github.com/edgelesssys/constellation/v2/internal/cloud"
"github.com/edgelesssys/constellation/v2/internal/cloud/metadata"
"github.com/edgelesssys/constellation/v2/internal/gcpshared"
"github.com/edgelesssys/constellation/v2/internal/role"
"google.golang.org/api/iterator"
computepb "google.golang.org/genproto/googleapis/cloud/compute/v1"
"google.golang.org/protobuf/proto"
)
const (
gcpSSHMetadataKey = "ssh-keys"
)
var zoneFromRegionRegex = regexp.MustCompile("([a-z]*-[a-z]*[0-9])")
// Client implements the gcp.API interface.
type Client struct {
instanceAPI
subnetworkAPI
metadataAPI
forwardingRulesAPI
}
// NewClient creates a new Client.
func NewClient(ctx context.Context) (*Client, error) {
insAPI, err := compute.NewInstancesRESTClient(ctx)
if err != nil {
return nil, err
}
subnetAPI, err := compute.NewSubnetworksRESTClient(ctx)
if err != nil {
return nil, err
}
forwardingRulesAPI, err := compute.NewGlobalForwardingRulesRESTClient(ctx)
if err != nil {
return nil, err
}
return &Client{
instanceAPI: &instanceClient{insAPI},
subnetworkAPI: &subnetworkClient{subnetAPI},
forwardingRulesAPI: &forwardingRulesClient{forwardingRulesAPI},
metadataAPI: &metadataClient{},
}, nil
}
// RetrieveInstances returns list of instances including their ips and metadata.
func (c *Client) RetrieveInstances(ctx context.Context, project, zone string) ([]metadata.InstanceMetadata, error) {
uid, err := c.UID(ctx)
if err != nil {
return nil, err
}
req := &computepb.ListInstancesRequest{
Filter: proto.String(fmt.Sprintf("labels.%s:%s", cloud.TagUID, uid)),
Project: project,
Zone: zone,
}
instanceIterator := c.instanceAPI.List(ctx, req)
instances := []metadata.InstanceMetadata{}
for {
resp, err := instanceIterator.Next()
if err == iterator.Done {
break
}
if err != nil {
return nil, fmt.Errorf("retrieving instance list from compute API client: %w", err)
}
instance, err := convertToCoreInstance(resp, project, zone)
if err != nil {
return nil, err
}
instances = append(instances, instance)
}
return instances, nil
}
// RetrieveInstance returns a an instance including ips and metadata.
func (c *Client) RetrieveInstance(ctx context.Context, project, zone, instanceName string) (metadata.InstanceMetadata, error) {
instance, err := c.getComputeInstance(ctx, project, zone, instanceName)
if err != nil {
return metadata.InstanceMetadata{}, err
}
return convertToCoreInstance(instance, project, zone)
}
// RetrieveProjectID retrieves the GCP projectID containing the current instance.
func (c *Client) RetrieveProjectID() (string, error) {
value, err := c.metadataAPI.ProjectID()
if err != nil {
return "", fmt.Errorf("requesting GCP projectID failed %w", err)
}
return value, nil
}
// RetrieveZone retrieves the GCP zone containing the current instance.
func (c *Client) RetrieveZone() (string, error) {
value, err := c.metadataAPI.Zone()
if err != nil {
return "", fmt.Errorf("requesting GCP zone failed %w", err)
}
return value, nil
}
func (c *Client) RetrieveInstanceName() (string, error) {
value, err := c.metadataAPI.InstanceName()
if err != nil {
return "", fmt.Errorf("requesting GCP instanceName failed %w", err)
}
return value, nil
}
func (c *Client) RetrieveInstanceMetadata(attr string) (string, error) {
value, err := c.metadataAPI.InstanceAttributeValue(attr)
if err != nil {
return "", fmt.Errorf("requesting GCP instance metadata: %w", err)
}
return value, nil
}
// SetInstanceMetadata modifies a key value pair of metadata for the instance specified by project, zone and instanceName.
func (c *Client) SetInstanceMetadata(ctx context.Context, project, zone, instanceName, key, value string) error {
instance, err := c.getComputeInstance(ctx, project, zone, instanceName)
if err != nil {
return fmt.Errorf("retrieving instance metadata: %w", err)
}
if instance == nil || instance.Metadata == nil {
return fmt.Errorf("retrieving instance metadata returned invalid results")
}
// convert instance metadata to map to handle duplicate keys correctly
metadataMap := extractInstanceMetadata(instance.Metadata, key, false)
metadataMap[key] = value
// convert instance metadata back to flat list
metadata := flattenInstanceMetadata(metadataMap, instance.Metadata.Fingerprint, instance.Metadata.Kind)
if err := c.updateInstanceMetadata(ctx, project, zone, instanceName, metadata); err != nil {
return fmt.Errorf("setting instance metadata %v: %v: %w", key, value, err)
}
return nil
}
// UnsetInstanceMetadata modifies a key value pair of metadata for the instance specified by project, zone and instanceName.
func (c *Client) UnsetInstanceMetadata(ctx context.Context, project, zone, instanceName, key string) error {
instance, err := c.getComputeInstance(ctx, project, zone, instanceName)
if err != nil {
return fmt.Errorf("retrieving instance metadata: %w", err)
}
if instance == nil || instance.Metadata == nil {
return fmt.Errorf("retrieving instance metadata returned invalid results")
}
// convert instance metadata to map to handle duplicate keys correctly
// and skip the key to be removed
metadataMap := extractInstanceMetadata(instance.Metadata, key, true)
// convert instance metadata back to flat list
metadata := flattenInstanceMetadata(metadataMap, instance.Metadata.Fingerprint, instance.Metadata.Kind)
if err := c.updateInstanceMetadata(ctx, project, zone, instanceName, metadata); err != nil {
return fmt.Errorf("unsetting instance metadata key %v: %w", key, err)
}
return nil
}
// RetrieveSubnetworkAliasCIDR returns the alias CIDR of the subnetwork specified by project, zone and subnetworkName.
func (c *Client) RetrieveSubnetworkAliasCIDR(ctx context.Context, project, zone, instanceName string) (string, error) {
instance, err := c.getComputeInstance(ctx, project, zone, instanceName)
if err != nil {
return "", err
}
if instance == nil || instance.NetworkInterfaces == nil || len(instance.NetworkInterfaces) == 0 || instance.NetworkInterfaces[0].Subnetwork == nil {
return "", fmt.Errorf("retrieving instance network interfaces failed")
}
subnetworkURL := *instance.NetworkInterfaces[0].Subnetwork
subnetworkURLFragments := strings.Split(subnetworkURL, "/")
subnetworkName := subnetworkURLFragments[len(subnetworkURLFragments)-1]
// convert:
// zone --> region
// europe-west3-b --> europe-west3
region := zoneFromRegionRegex.FindString(zone)
if region == "" {
return "", fmt.Errorf("invalid zone %s", zone)
}
req := &computepb.GetSubnetworkRequest{
Project: project,
Region: region,
Subnetwork: subnetworkName,
}
subnetwork, err := c.subnetworkAPI.Get(ctx, req)
if err != nil {
return "", fmt.Errorf("retrieving subnetwork alias CIDR failed: %w", err)
}
if subnetwork == nil || len(subnetwork.SecondaryIpRanges) == 0 || (subnetwork.SecondaryIpRanges[0]).IpCidrRange == nil {
return "", fmt.Errorf("retrieving subnetwork alias CIDR returned invalid results")
}
return *(subnetwork.SecondaryIpRanges[0]).IpCidrRange, nil
}
// RetrieveLoadBalancerEndpoint returns the endpoint of the load balancer with the constellation-uid tag.
func (c *Client) RetrieveLoadBalancerEndpoint(ctx context.Context, project string) (string, error) {
uid, err := c.UID(ctx)
if err != nil {
return "", err
}
req := &computepb.ListGlobalForwardingRulesRequest{
Project: project,
}
iter := c.forwardingRulesAPI.List(ctx, req)
for {
resp, err := iter.Next()
if err == iterator.Done {
break
}
if err != nil {
return "", fmt.Errorf("retrieving load balancer IP failed: %w", err)
}
if resp.Labels[cloud.TagUID] == uid && resp.Labels["constellation-use"] == "kubernetes" {
if resp.PortRange == nil {
return "", errors.New("load balancer with searched UID has no ports")
}
portRange := strings.Split(*resp.PortRange, "-")
return net.JoinHostPort(*resp.IPAddress, portRange[0]), nil
}
}
return "", fmt.Errorf("retrieving load balancer IP failed: load balancer not found")
}
// Close closes the instanceAPI client.
func (c *Client) Close() error {
if err := c.subnetworkAPI.Close(); err != nil {
return err
}
if err := c.forwardingRulesAPI.Close(); err != nil {
return err
}
return c.instanceAPI.Close()
}
func (c *Client) getComputeInstance(ctx context.Context, project, zone, instanceName string) (*computepb.Instance, error) {
instanceGetReq := &computepb.GetInstanceRequest{
Project: project,
Zone: zone,
Instance: instanceName,
}
instance, err := c.instanceAPI.Get(ctx, instanceGetReq)
if err != nil {
return nil, fmt.Errorf("retrieving compute instance: %w", err)
}
return instance, nil
}
// updateInstanceMetadata updates all instance metadata key-value pairs.
func (c *Client) updateInstanceMetadata(ctx context.Context, project, zone, instanceName string, metadata *computepb.Metadata) error {
setMetadataReq := &computepb.SetMetadataInstanceRequest{
Project: project,
Zone: zone,
Instance: instanceName,
MetadataResource: metadata,
}
if _, err := c.instanceAPI.SetMetadata(ctx, setMetadataReq); err != nil {
return fmt.Errorf("updating instance metadata: %w", err)
}
return nil
}
// UID retrieves the current instances uid.
func (c *Client) UID(ctx context.Context) (string, error) {
// API endpoint: http://metadata.google.internal/computeMetadata/v1/instance/attributes/constellation-uid
instanceID, err := c.InstanceID()
if err != nil {
return "", fmt.Errorf("retrieving instance ID: %w", err)
}
project, err := c.ProjectID()
if err != nil {
return "", fmt.Errorf("retrieving project ID: %w", err)
}
zone, err := c.Zone()
if err != nil {
return "", fmt.Errorf("retrieving zone: %w", err)
}
instance, err := c.instanceAPI.Get(ctx, &computepb.GetInstanceRequest{
Project: project,
Zone: zone,
Instance: instanceID,
})
if err != nil {
return "", fmt.Errorf("retrieving instance labels: %w", err)
}
return instance.Labels[cloud.TagUID], nil
}
// extractVPCIP extracts the primary private IP from a list of interfaces.
func extractVPCIP(interfaces []*computepb.NetworkInterface) string {
for _, interf := range interfaces {
if interf == nil || interf.NetworkIP == nil || interf.Name == nil || *interf.Name != "nic0" {
continue
}
// return private IP from the default interface
return *interf.NetworkIP
}
return ""
}
// extractPublicIP extracts a public IP from a list of interfaces.
func extractPublicIP(interfaces []*computepb.NetworkInterface) string {
for _, interf := range interfaces {
if interf == nil || interf.AccessConfigs == nil || interf.Name == nil || *interf.Name != "nic0" {
continue
}
// return public IP from the default interface
// GCP only supports one type of access config, so returning the first IP should result in a valid public IP
for _, accessConfig := range interf.AccessConfigs {
if accessConfig == nil || accessConfig.NatIP == nil {
continue
}
return *accessConfig.NatIP
}
}
return ""
}
// extractAliasIPRanges extracts alias interface IPs from a list of interfaces.
func extractAliasIPRanges(interfaces []*computepb.NetworkInterface) []string {
ips := []string{}
for _, interf := range interfaces {
if interf == nil || interf.AliasIpRanges == nil {
continue
}
for _, aliasIP := range interf.AliasIpRanges {
if aliasIP == nil || aliasIP.IpCidrRange == nil {
continue
}
ips = append(ips, *aliasIP.IpCidrRange)
}
}
return ips
}
// extractSSHKeys extracts SSH keys from GCP instance metadata.
// reference: https://cloud.google.com/compute/docs/connect/add-ssh-keys .
func extractSSHKeys(metadata map[string]string) map[string][]string {
sshKeysRaw, ok := metadata[gcpSSHMetadataKey]
if !ok {
// ignore missing metadata entry
return map[string][]string{}
}
sshKeyLines := strings.Split(sshKeysRaw, "\n")
keys := map[string][]string{}
for _, sshKeyRaw := range sshKeyLines {
keyParts := strings.SplitN(sshKeyRaw, ":", 2)
if len(keyParts) != 2 {
continue
}
username := keyParts[0]
keyParts = strings.SplitN(keyParts[1], " ", 3)
if len(keyParts) < 2 {
continue
}
keyValue := fmt.Sprintf("%s %s", keyParts[0], keyParts[1])
keys[username] = append(keys[username], keyValue)
}
return keys
}
// convertToCoreInstance converts a *computepb.Instance to a core.Instance.
func convertToCoreInstance(in *computepb.Instance, project string, zone string) (metadata.InstanceMetadata, error) {
if in.Name == nil {
return metadata.InstanceMetadata{}, fmt.Errorf("retrieving instance from compute API client returned invalid instance Name: %v", in.Name)
}
mdata := extractInstanceMetadata(in.Metadata, "", false)
return metadata.InstanceMetadata{
Name: *in.Name,
ProviderID: gcpshared.JoinProviderID(project, zone, *in.Name),
Role: role.FromString(in.Labels[cloud.TagRole]),
VPCIP: extractVPCIP(in.NetworkInterfaces),
PublicIP: extractPublicIP(in.NetworkInterfaces),
AliasIPRanges: extractAliasIPRanges(in.NetworkInterfaces),
SSHKeys: extractSSHKeys(mdata),
}, nil
}
// extractInstanceMetadata will extract the list of instance metadata key-value pairs into a map.
// If "skipKey" is true, "key" will be skipped.
func extractInstanceMetadata(in *computepb.Metadata, key string, skipKey bool) map[string]string {
metadataMap := map[string]string{}
for _, item := range in.Items {
if item == nil || item.Key == nil || item.Value == nil {
continue
}
if skipKey && *item.Key == key {
continue
}
metadataMap[*item.Key] = *item.Value
}
return metadataMap
}
// flattenInstanceMetadata takes a map of metadata key-value pairs and returns a flat list of computepb.Items inside computepb.Metadata.
func flattenInstanceMetadata(metadataMap map[string]string, fingerprint, kind *string) *computepb.Metadata {
metadata := &computepb.Metadata{
Fingerprint: fingerprint,
Kind: kind,
Items: make([]*computepb.Items, len(metadataMap)),
}
i := 0
for mapKey, mapValue := range metadataMap {
metadata.Items[i] = &computepb.Items{Key: proto.String(mapKey), Value: proto.String(mapValue)}
i++
}
return metadata
}

1106
internal/cloud/gcp/client_test.go
View File

File diff suppressed because it is too large Load Diff

319
internal/cloud/gcp/cloud.go Normal file
View File

@ -0,0 +1,319 @@
/*
Copyright (c) Edgeless Systems GmbH
SPDX-License-Identifier: AGPL-3.0-only
*/
package gcp
import (
"context"
"errors"
"fmt"
"net"
"path"
"regexp"
"strings"
compute "cloud.google.com/go/compute/apiv1"
imds "cloud.google.com/go/compute/metadata"
"github.com/edgelesssys/constellation/v2/internal/cloud"
"github.com/edgelesssys/constellation/v2/internal/cloud/gcpshared"
"github.com/edgelesssys/constellation/v2/internal/cloud/metadata"
"github.com/edgelesssys/constellation/v2/internal/role"
"google.golang.org/api/iterator"
computepb "google.golang.org/genproto/googleapis/cloud/compute/v1"
"google.golang.org/protobuf/proto"
)
const (
// tagUsage is a label key used to indicate the use of the resource.
tagUsage = "constellation-use"
)
var zoneFromRegionRegex = regexp.MustCompile("([a-z]*-[a-z]*[0-9])")
// Cloud provides GCP cloud metadata information and API access.
type Cloud struct {
forwardingRulesAPI forwardingRulesAPI
imds imdsAPI
instanceAPI instanceAPI
subnetAPI subnetAPI
closers []func() error
}
// New creates and initializes Cloud.
// The Close method should be called when Cloud is no longer needed.
func New(ctx context.Context) (cloud *Cloud, err error) {
var closers []func() error
insAPI, err := compute.NewInstancesRESTClient(ctx)
if err != nil {
return nil, err
}
closers = append(closers, insAPI.Close)
forwardingRulesAPI, err := compute.NewGlobalForwardingRulesRESTClient(ctx)
if err != nil {
return nil, err
}
closers = append(closers, forwardingRulesAPI.Close)
subnetAPI, err := compute.NewSubnetworksRESTClient(ctx)
if err != nil {
return nil, err
}
closers = append(closers, subnetAPI.Close)
return &Cloud{
imds: imds.NewClient(nil),
instanceAPI: &instanceClient{insAPI},
forwardingRulesAPI: &forwardingRulesClient{forwardingRulesAPI},
subnetAPI: subnetAPI,
closers: closers,
}, nil
}
// Close closes all connections to the GCP API server.
func (c *Cloud) Close() {
for _, close := range c.closers {
_ = close()
}
}
// GetInstance retrieves an instance using its providerID.
func (c *Cloud) GetInstance(ctx context.Context, providerID string) (metadata.InstanceMetadata, error) {
project, zone, instanceName, err := gcpshared.SplitProviderID(providerID)
if err != nil {
return metadata.InstanceMetadata{}, fmt.Errorf("invalid providerID: %w", err)
}
return c.getInstance(ctx, project, zone, instanceName)
}
// GetLoadBalancerEndpoint returns the endpoint of the load balancer.
func (c *Cloud) GetLoadBalancerEndpoint(ctx context.Context) (string, error) {
project, zone, instanceName, err := c.retrieveInstanceInfo()
if err != nil {
return "", err
}
uid, err := c.uid(ctx, project, zone, instanceName)
if err != nil {
return "", err
}
var resp *computepb.ForwardingRule
iter := c.forwardingRulesAPI.List(ctx, &computepb.ListGlobalForwardingRulesRequest{
Project: project,
Filter: proto.String(fmt.Sprintf("(labels.%s:%s) AND (labels.%s:kubernetes)", cloud.TagUID, uid, tagUsage)),
})
for resp, err = iter.Next(); err == nil; resp, err = iter.Next() {
if resp.PortRange == nil {
continue
}
if resp.IPAddress == nil {
continue
}
portRange := strings.Split(*resp.PortRange, "-")
return net.JoinHostPort(*resp.IPAddress, portRange[0]), nil
}
return "", fmt.Errorf("kubernetes load balancer with UID %s not found: %w", uid, err)
}
// List retrieves all instances belonging to the current constellation.
func (c *Cloud) List(ctx context.Context) ([]metadata.InstanceMetadata, error) {
project, zone, instanceName, err := c.retrieveInstanceInfo()
if err != nil {
return nil, err
}
uid, err := c.uid(ctx, project, zone, instanceName)
if err != nil {
return nil, err
}
var instances []metadata.InstanceMetadata
var resp *computepb.Instance
iter := c.instanceAPI.List(ctx, &computepb.ListInstancesRequest{
Filter: proto.String(fmt.Sprintf("labels.%s:%s", cloud.TagUID, uid)),
Project: project,
Zone: zone,
})
for resp, err = iter.Next(); err == nil; resp, err = iter.Next() {
instance, err := convertToInstanceMetadata(resp, project, zone)
if err != nil {
return nil, fmt.Errorf("retrieving instance list from GCP: failed to convert instance: %w", err)
}
// convertToInstanceMetadata already checks for nil resp.NetworkInterfaces
if len(resp.NetworkInterfaces) == 0 || resp.NetworkInterfaces[0] == nil ||
resp.NetworkInterfaces[0].Subnetwork == nil {
return nil, errors.New("retrieving compute instance: received invalid instance")
}
subnetCIDR, err := c.retrieveSubnetworkAliasCIDR(ctx, project, zone, *resp.NetworkInterfaces[0].Subnetwork)
if err != nil {
return nil, fmt.Errorf("retrieving compute instance: failed to retrieve subnet CIDR: %w", err)
}
instance.SecondaryIPRange = subnetCIDR
instances = append(instances, instance)
}
if errors.Is(err, iterator.Done) {
return instances, nil
}
return nil, fmt.Errorf("retrieving instance list from GCP: %w", err)
}
// ProviderID returns the providerID of the current instance.
func (c *Cloud) ProviderID(ctx context.Context) (string, error) {
project, zone, instanceName, err := c.retrieveInstanceInfo()
if err != nil {
return "", err
}
return gcpshared.JoinProviderID(project, zone, instanceName), nil
}
// Self retrieves the current instance.
func (c *Cloud) Self(ctx context.Context) (metadata.InstanceMetadata, error) {
project, zone, instanceName, err := c.retrieveInstanceInfo()
if err != nil {
return metadata.InstanceMetadata{}, err
}
return c.getInstance(ctx, project, zone, instanceName)
}
// UID retrieves the UID of the constellation.
func (c *Cloud) UID(ctx context.Context) (string, error) {
project, zone, instanceName, err := c.retrieveInstanceInfo()
if err != nil {
return "", err
}
return c.uid(ctx, project, zone, instanceName)
}
// getInstance retrieves an instance using its project, zone and name, and parses it to metadata.InstanceMetadata.
func (c *Cloud) getInstance(ctx context.Context, project, zone, instanceName string) (metadata.InstanceMetadata, error) {
gcpInstance, err := c.instanceAPI.Get(ctx, &computepb.GetInstanceRequest{
Project: project,
Zone: zone,
Instance: instanceName,
})
if err != nil {
return metadata.InstanceMetadata{}, fmt.Errorf("retrieving compute instance: %w", err)
}
if gcpInstance == nil || gcpInstance.NetworkInterfaces == nil || len(gcpInstance.NetworkInterfaces) == 0 ||
gcpInstance.NetworkInterfaces[0] == nil || gcpInstance.NetworkInterfaces[0].Subnetwork == nil {
return metadata.InstanceMetadata{}, errors.New("retrieving compute instance: received invalid instance")
}
subnetCIDR, err := c.retrieveSubnetworkAliasCIDR(ctx, project, zone, *gcpInstance.NetworkInterfaces[0].Subnetwork)
if err != nil {
return metadata.InstanceMetadata{}, err
}
instance, err := convertToInstanceMetadata(gcpInstance, project, zone)
if err != nil {
return metadata.InstanceMetadata{}, fmt.Errorf("converting instance: %w", err)
}
instance.SecondaryIPRange = subnetCIDR
return instance, nil
}
// retrieveInstanceInfo retrieves the project, zone and instance name of the current instance using the imds API.
func (c *Cloud) retrieveInstanceInfo() (project, zone, instanceName string, err error) {
project, err = c.imds.ProjectID()
if err != nil {
return "", "", "", fmt.Errorf("retrieving project ID from imds: %w", err)
}
zone, err = c.imds.Zone()
if err != nil {
return "", "", "", fmt.Errorf("retrieving zone from imds: %w", err)
}
instanceName, err = c.imds.InstanceName()
if err != nil {
return "", "", "", fmt.Errorf("retrieving instance name from imds: %w", err)
}
return project, zone, instanceName, nil
}
// retrieveSubnetworkAliasCIDR retrieves the secondary IP range CIDR of the subnetwork,
// identified by project, zone and subnetworkURI.
func (c *Cloud) retrieveSubnetworkAliasCIDR(ctx context.Context, project, zone, subnetworkURI string) (string, error) {
// convert:
// zone --> region
// europe-west3-b --> europe-west3
region := zoneFromRegionRegex.FindString(zone)
if region == "" {
return "", fmt.Errorf("invalid zone %s", zone)
}
req := &computepb.GetSubnetworkRequest{
Project: project,
Region: region,
Subnetwork: path.Base(subnetworkURI),
}
subnetwork, err := c.subnetAPI.Get(ctx, req)
if err != nil {
return "", fmt.Errorf("retrieving subnetwork alias CIDR failed: %w", err)
}
if subnetwork == nil || len(subnetwork.SecondaryIpRanges) == 0 ||
subnetwork.SecondaryIpRanges[0] == nil || subnetwork.SecondaryIpRanges[0].IpCidrRange == nil {
return "", fmt.Errorf("retrieving subnetwork alias CIDR failed: received invalid subnetwork")
}
return *subnetwork.SecondaryIpRanges[0].IpCidrRange, nil
}
// uid retrieves the UID of the instance identified by project, zone and instanceName.
// The UID is retrieved from the instance's labels.
func (c *Cloud) uid(ctx context.Context, project, zone, instanceName string) (string, error) {
instance, err := c.instanceAPI.Get(ctx, &computepb.GetInstanceRequest{
Project: project,
Zone: zone,
Instance: instanceName,
})
if err != nil {
return "", fmt.Errorf("retrieving compute instance: %w", err)
}
if instance == nil || instance.Labels == nil {
return "", errors.New("retrieving compute instance: received instance with invalid labels")
}
return instance.Labels[cloud.TagUID], nil
}
// convertToInstanceMetadata converts a *computepb.Instance to a metadata.InstanceMetadata.
func convertToInstanceMetadata(in *computepb.Instance, project string, zone string) (metadata.InstanceMetadata, error) {
if in.Name == nil {
return metadata.InstanceMetadata{}, fmt.Errorf("missing instance name")
}
var vpcIP string
var ips []string
for _, interf := range in.NetworkInterfaces {
if interf == nil {
continue
}
// use private IP from the default interface
if interf.NetworkIP != nil && interf.Name != nil && *interf.Name == "nic0" {
vpcIP = *interf.NetworkIP
}
if interf.AliasIpRanges == nil {
continue
}
for _, aliasIP := range interf.AliasIpRanges {
if aliasIP != nil && aliasIP.IpCidrRange != nil {
ips = append(ips, *aliasIP.IpCidrRange)
}
}
}
return metadata.InstanceMetadata{
Name: *in.Name,
ProviderID: gcpshared.JoinProviderID(project, zone, *in.Name),
Role: role.FromString(in.Labels[cloud.TagRole]),
VPCIP: vpcIP,
AliasIPRanges: ips,
}, nil
}

898
internal/cloud/gcp/cloud_test.go Normal file
View File

@ -0,0 +1,898 @@
/*
Copyright (c) Edgeless Systems GmbH
SPDX-License-Identifier: AGPL-3.0-only
*/
package gcp
import (
"context"
"errors"
"testing"
"github.com/edgelesssys/constellation/v2/internal/cloud"
"github.com/edgelesssys/constellation/v2/internal/cloud/metadata"
"github.com/edgelesssys/constellation/v2/internal/role"
gax "github.com/googleapis/gax-go/v2"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"go.uber.org/goleak"
"google.golang.org/api/iterator"
computepb "google.golang.org/genproto/googleapis/cloud/compute/v1"
"google.golang.org/protobuf/proto"
)
func TestMain(m *testing.M) {
goleak.VerifyTestMain(m,
// https://github.com/census-instrumentation/opencensus-go/issues/1262
goleak.IgnoreTopFunction("go.opencensus.io/stats/view.(*worker).start"),
)
}
func TestGetInstance(t *testing.T) {
someErr := errors.New("failed")
goodInstance := &computepb.Instance{
Name: proto.String("someInstance"),
Zone: proto.String("someZone-west3-b"),
Labels: map[string]string{
cloud.TagUID: "1234",
cloud.TagRole: role.ControlPlane.String(),
},
NetworkInterfaces: []*computepb.NetworkInterface{
{
Name: proto.String("nic0"),
NetworkIP: proto.String("192.0.2.0"),
AliasIpRanges: []*computepb.AliasIpRange{
{
IpCidrRange: proto.String("192.0.3.0/8"),
},
},
Subnetwork: proto.String("projects/someProject/regions/someRegion/subnetworks/someSubnetwork"),
},
},
}
testCases := map[string]struct {
projectID, instanceName, zone string
instanceAPI stubInstanceAPI
subnetAPI stubSubnetAPI
wantErr bool
wantInstance metadata.InstanceMetadata
}{
"success": {
instanceName: "someInstance",
projectID: "someProject",
zone: "someZone-west3-b",
instanceAPI: stubInstanceAPI{
instance: goodInstance,
},
subnetAPI: stubSubnetAPI{
subnet: &computepb.Subnetwork{
SecondaryIpRanges: []*computepb.SubnetworkSecondaryRange{
{
IpCidrRange: proto.String("198.51.100.0/24"),
},
},
},
},
wantInstance: metadata.InstanceMetadata{
Name: "someInstance",
Role: role.ControlPlane,
ProviderID: "gce://someProject/someZone-west3-b/someInstance",
VPCIP: "192.0.2.0",
AliasIPRanges: []string{"192.0.3.0/8"},
SecondaryIPRange: "198.51.100.0/24",
},
},
"get instance error": {
instanceName: "someInstance",
projectID: "someProject",
zone: "someZone-west3-b",
instanceAPI: stubInstanceAPI{
instanceErr: someErr,
},
subnetAPI: stubSubnetAPI{
subnet: &computepb.Subnetwork{
SecondaryIpRanges: []*computepb.SubnetworkSecondaryRange{
{
IpCidrRange: proto.String("198.51.100.0/24"),
},
},
},
},
wantErr: true,
},
"get subnet error": {
instanceName: "someInstance",
projectID: "someProject",
zone: "someZone-west3-b",
instanceAPI: stubInstanceAPI{
instance: goodInstance,
},
subnetAPI: stubSubnetAPI{
subnetErr: someErr,
},
wantErr: true,
},
"invalid instance": {
instanceName: "someInstance",
projectID: "someProject",
zone: "someZone-west3-b",
instanceAPI: stubInstanceAPI{
instance: nil,
},
subnetAPI: stubSubnetAPI{
subnet: &computepb.Subnetwork{
SecondaryIpRanges: []*computepb.SubnetworkSecondaryRange{
{
IpCidrRange: proto.String("198.51.100.0/24"),
},
},
},
},
wantErr: true,
},
"invalid zone": {
instanceName: "someInstance",
projectID: "someProject",
zone: "invalidZone",
instanceAPI: stubInstanceAPI{
instance: goodInstance,
},
subnetAPI: stubSubnetAPI{
subnet: &computepb.Subnetwork{
SecondaryIpRanges: []*computepb.SubnetworkSecondaryRange{
{
IpCidrRange: proto.String("198.51.100.0/24"),
},
},
},
},
wantErr: true,
},
}
for name, tc := range testCases {
t.Run(name, func(t *testing.T) {
assert := assert.New(t)
require := require.New(t)
cloud := &Cloud{
instanceAPI: &tc.instanceAPI,
subnetAPI: &tc.subnetAPI,
}
instance, err := cloud.getInstance(context.Background(), tc.projectID, tc.zone, tc.instanceName)
if tc.wantErr {
assert.Error(err)
return
}
require.NoError(err)
assert.Equal(tc.wantInstance, instance)
})
}
}
func TestGetLoadbalancerEndpoint(t *testing.T) {
someErr := errors.New("failed")
goodInstance := &computepb.Instance{
Name: proto.String("someInstance"),
Zone: proto.String("someZone-west3-b"),
Labels: map[string]string{
cloud.TagUID: "1234",
cloud.TagRole: role.ControlPlane.String(),
},
NetworkInterfaces: []*computepb.NetworkInterface{
{
Name: proto.String("nic0"),
NetworkIP: proto.String("192.0.2.0"),
AliasIpRanges: []*computepb.AliasIpRange{
{
IpCidrRange: proto.String("192.0.3.0/8"),
},
},
Subnetwork: proto.String("projects/someProject/regions/someRegion/subnetworks/someSubnetwork"),
},
},
}
testCases := map[string]struct {
imds stubIMDS
instanceAPI stubInstanceAPI
forwardingRulesAPI stubForwardingRulesAPI
wantEndpoint string
wantErr bool
}{
"success": {
imds: stubIMDS{
projectID: "someProject",
zone: "someZone-west3-b",
instanceName: "someInstance",
},
instanceAPI: stubInstanceAPI{
instance: goodInstance,
},
forwardingRulesAPI: stubForwardingRulesAPI{
iterator: &stubForwardingRulesIterator{
forwardingRules: []*computepb.ForwardingRule{
{
PortRange: proto.String("6443"),
IPAddress: proto.String("192.0.2.255"),
},
},
},
},
wantEndpoint: "192.0.2.255:6443",
},
"imds error": {
imds: stubIMDS{
projectIDErr: someErr,
zone: "someZone-west3-b",
instanceName: "someInstance",
},
instanceAPI: stubInstanceAPI{
instance: goodInstance,
},
forwardingRulesAPI: stubForwardingRulesAPI{
iterator: &stubForwardingRulesIterator{
forwardingRules: []*computepb.ForwardingRule{
{
PortRange: proto.String("6443"),
IPAddress: proto.String("192.0.2.255"),
},
},
},
},
wantErr: true,
},
"iterator error": {
imds: stubIMDS{
projectID: "someProject",
zone: "someZone-west3-b",
instanceName: "someInstance",
},
instanceAPI: stubInstanceAPI{
instance: goodInstance,
},
forwardingRulesAPI: stubForwardingRulesAPI{
iterator: &stubForwardingRulesIterator{
err: someErr,
},
},
wantErr: true,
},
"no forwarding rules": {
imds: stubIMDS{
projectID: "someProject",
zone: "someZone-west3-b",
instanceName: "someInstance",
},
instanceAPI: stubInstanceAPI{
instance: goodInstance,
},
forwardingRulesAPI: stubForwardingRulesAPI{
iterator: &stubForwardingRulesIterator{},
},
wantErr: true,
},
"missing port range": {
imds: stubIMDS{
projectID: "someProject",
zone: "someZone-west3-b",
instanceName: "someInstance",
},
instanceAPI: stubInstanceAPI{
instance: goodInstance,
},
forwardingRulesAPI: stubForwardingRulesAPI{
iterator: &stubForwardingRulesIterator{
forwardingRules: []*computepb.ForwardingRule{
{
IPAddress: proto.String("192.0.2.255"),
},
},
},
},
wantErr: true,
},
"missing IP address": {
imds: stubIMDS{
projectID: "someProject",
zone: "someZone-west3-b",
instanceName: "someInstance",
},
instanceAPI: stubInstanceAPI{
instance: goodInstance,
},
forwardingRulesAPI: stubForwardingRulesAPI{
iterator: &stubForwardingRulesIterator{
forwardingRules: []*computepb.ForwardingRule{
{
PortRange: proto.String("6443"),
},
},
},
},
wantErr: true,
},
"get instance error": {
imds: stubIMDS{
projectID: "someProject",
zone: "someZone-west3-b",
instanceName: "someInstance",
},
instanceAPI: stubInstanceAPI{
instanceErr: someErr,
},
forwardingRulesAPI: stubForwardingRulesAPI{
iterator: &stubForwardingRulesIterator{
forwardingRules: []*computepb.ForwardingRule{
{
PortRange: proto.String("6443"),
IPAddress: proto.String("192.0.2.255"),
},
},
},
},
wantErr: true,
},
"invalid instance": {
imds: stubIMDS{
projectID: "someProject",
zone: "someZone-west3-b",
instanceName: "someInstance",
},
instanceAPI: stubInstanceAPI{
instance: nil,
},
forwardingRulesAPI: stubForwardingRulesAPI{
iterator: &stubForwardingRulesIterator{
forwardingRules: []*computepb.ForwardingRule{
{
PortRange: proto.String("6443"),
IPAddress: proto.String("192.0.2.255"),
},
},
},
},
wantErr: true,
},
}
for name, tc := range testCases {
t.Run(name, func(t *testing.T) {
assert := assert.New(t)
cloud := &Cloud{
imds: &tc.imds,
instanceAPI: &tc.instanceAPI,
forwardingRulesAPI: &tc.forwardingRulesAPI,
}
endpoint, err := cloud.GetLoadBalancerEndpoint(context.Background())
if tc.wantErr {
assert.Error(err)
return
}
assert.NoError(err)
assert.Equal(tc.wantEndpoint, endpoint)
})
}
}
func TestList(t *testing.T) {
someErr := errors.New("failed")
goodInstance := &computepb.Instance{
Name: proto.String("someInstance"),
Zone: proto.String("someZone-west3-b"),
Labels: map[string]string{
cloud.TagUID: "1234",
cloud.TagRole: role.ControlPlane.String(),
},
NetworkInterfaces: []*computepb.NetworkInterface{
{
Name: proto.String("nic0"),
NetworkIP: proto.String("192.0.2.0"),
AliasIpRanges: []*computepb.AliasIpRange{
{
IpCidrRange: proto.String("198.51.100.0/24"),
},
},
Subnetwork: proto.String("projects/someProject/regions/someRegion/subnetworks/someSubnetwork"),
},
},
}
goodSubnet := &computepb.Subnetwork{
SecondaryIpRanges: []*computepb.SubnetworkSecondaryRange{
{
IpCidrRange: proto.String("198.51.100.0/24"),
},
},
}
testCases := map[string]struct {
imds stubIMDS
instanceAPI stubInstanceAPI
subnetAPI stubSubnetAPI
wantErr bool
wantInstances []metadata.InstanceMetadata
}{
"success": {
imds: stubIMDS{
projectID: "someProject",
zone: "someZone-west3-b",
instanceName: "someInstance",
},
instanceAPI: stubInstanceAPI{
instance: goodInstance,
iterator: &stubInstanceIterator{
instances: []*computepb.Instance{
goodInstance,
},
},
},
subnetAPI: stubSubnetAPI{
subnet: goodSubnet,
},
wantInstances: []metadata.InstanceMetadata{
{
Name: "someInstance",
Role: role.ControlPlane,
ProviderID: "gce://someProject/someZone-west3-b/someInstance",
VPCIP: "192.0.2.0",
AliasIPRanges: []string{"198.51.100.0/24"},
SecondaryIPRange: "198.51.100.0/24",
},
},
},
"list multiple instances": {
imds: stubIMDS{
projectID: "someProject",
zone: "someZone-west3-b",
instanceName: "someInstance",
},
instanceAPI: stubInstanceAPI{
instance: goodInstance,
iterator: &stubInstanceIterator{
instances: []*computepb.Instance{
goodInstance,
{
Name: proto.String("anotherInstance"),
Zone: proto.String("someZone-west3-b"),
Labels: map[string]string{
cloud.TagUID: "1234",
cloud.TagRole: role.Worker.String(),
},
NetworkInterfaces: []*computepb.NetworkInterface{
{
Name: proto.String("nic0"),
NetworkIP: proto.String("192.0.2.1"),
AliasIpRanges: []*computepb.AliasIpRange{
{
IpCidrRange: proto.String("198.51.100.0/24"),
},
},
Subnetwork: proto.String("projects/someProject/regions/someRegion/subnetworks/someSubnetwork"),
},
},
},
},
},
},
subnetAPI: stubSubnetAPI{
subnet: goodSubnet,
},
wantInstances: []metadata.InstanceMetadata{
{
Name: "someInstance",
Role: role.ControlPlane,
ProviderID: "gce://someProject/someZone-west3-b/someInstance",
VPCIP: "192.0.2.0",
AliasIPRanges: []string{"198.51.100.0/24"},
SecondaryIPRange: "198.51.100.0/24",
},
{
Name: "anotherInstance",
Role: role.Worker,
ProviderID: "gce://someProject/someZone-west3-b/anotherInstance",
VPCIP: "192.0.2.1",
AliasIPRanges: []string{"198.51.100.0/24"},
SecondaryIPRange: "198.51.100.0/24",
},
},
},
"imds error": {
imds: stubIMDS{
projectIDErr: someErr,
},
instanceAPI: stubInstanceAPI{
instance: goodInstance,
iterator: &stubInstanceIterator{
instances: []*computepb.Instance{
goodInstance,
},
},
},
subnetAPI: stubSubnetAPI{
subnet: goodSubnet,
},
wantErr: true,
},
"iterator error": {
imds: stubIMDS{
projectID: "someProject",
zone: "someZone-west3-b",
instanceName: "someInstance",
},
instanceAPI: stubInstanceAPI{
instance: goodInstance,
iterator: &stubInstanceIterator{
err: someErr,
},
},
subnetAPI: stubSubnetAPI{
subnet: goodSubnet,
},
wantErr: true,
},
"get instance error": {
imds: stubIMDS{
projectID: "someProject",
zone: "someZone-west3-b",
instanceName: "someInstance",
},
instanceAPI: stubInstanceAPI{
instanceErr: someErr,
iterator: &stubInstanceIterator{
instances: []*computepb.Instance{
goodInstance,
},
},
},
subnetAPI: stubSubnetAPI{
subnet: goodSubnet,
},
wantErr: true,
},
"get subnet error": {
imds: stubIMDS{
projectID: "someProject",
zone: "someZone-west3-b",
instanceName: "someInstance",
},
instanceAPI: stubInstanceAPI{
instance: goodInstance,
iterator: &stubInstanceIterator{
instances: []*computepb.Instance{
goodInstance,
},
},
},
subnetAPI: stubSubnetAPI{
subnetErr: someErr,
},
wantErr: true,
},
}
for name, tc := range testCases {
t.Run(name, func(t *testing.T) {
assert := assert.New(t)
require := require.New(t)
cloud := &Cloud{
imds: &tc.imds,
instanceAPI: &tc.instanceAPI,
subnetAPI: &tc.subnetAPI,
}
instances, err := cloud.List(context.Background())
if tc.wantErr {
assert.Error(err)
return
}
require.NoError(err)
assert.ElementsMatch(tc.wantInstances, instances)
})
}
}
func TestRetrieveInstanceInfo(t *testing.T) {
someErr := errors.New("failed")
testCases := map[string]struct {
imds stubIMDS
wantErr bool
}{
"success": {
imds: stubIMDS{
projectID: "someProject",
zone: "someZone-west3-b",
instanceName: "someInstance",
},
},
"get project id error": {
imds: stubIMDS{
zone: "someZone-west3-b",
instanceName: "someInstance",
projectIDErr: someErr,
},
wantErr: true,
},
"get zone error": {
imds: stubIMDS{
projectID: "someProject",
instanceName: "someInstance",
zoneErr: someErr,
},
wantErr: true,
},
"get instance name error": {
imds: stubIMDS{
projectID: "someProject",
zone: "someZone-west3-b",
instanceNameErr: someErr,
},
wantErr: true,
},
}
for name, tc := range testCases {
t.Run(name, func(t *testing.T) {
assert := assert.New(t)
cloud := &Cloud{
imds: &tc.imds,
}
project, zone, instance, err := cloud.retrieveInstanceInfo()
if tc.wantErr {
assert.Error(err)
return
}
assert.NoError(err)
assert.Equal(tc.imds.projectID, project)
assert.Equal(tc.imds.zone, zone)
assert.Equal(tc.imds.instanceName, instance)
})
}
}
func TestUID(t *testing.T) {
someErr := errors.New("failed")
testCases := map[string]struct {
imds stubIMDS
instanceAPI stubInstanceAPI
wantUID string
wantErr bool
}{
"success": {
imds: stubIMDS{
projectID: "someProject",
zone: "someZone-west3-b",
instanceName: "someInstance",
},
instanceAPI: stubInstanceAPI{
instance: &computepb.Instance{
Name: proto.String("someInstance"),
Zone: proto.String("someZone-west3-b"),
Labels: map[string]string{
cloud.TagUID: "1234",
cloud.TagRole: role.ControlPlane.String(),
},
},
},
wantUID: "1234",
},
"imds error": {
imds: stubIMDS{
projectIDErr: someErr,
zone: "someZone-west3-b",
instanceName: "someInstance",
},
instanceAPI: stubInstanceAPI{
instance: &computepb.Instance{
Name: proto.String("someInstance"),
Zone: proto.String("someZone-west3-b"),
Labels: map[string]string{
cloud.TagUID: "1234",
cloud.TagRole: role.ControlPlane.String(),
},
},
},
wantErr: true,
},
"instance error": {
imds: stubIMDS{
projectID: "someProject",
zone: "someZone-west3-b",
instanceName: "someInstance",
},
instanceAPI: stubInstanceAPI{
instanceErr: someErr,
},
wantErr: true,
},
"invalid instance": {
imds: stubIMDS{
projectID: "someProject",
zone: "someZone-west3-b",
instanceName: "someInstance",
},
instanceAPI: stubInstanceAPI{
instance: nil,
},
wantErr: true,
},
}
for name, tc := range testCases {
t.Run(name, func(t *testing.T) {
assert := assert.New(t)
cloud := &Cloud{
imds: &tc.imds,
instanceAPI: &tc.instanceAPI,
}
uid, err := cloud.UID(context.Background())
if tc.wantErr {
assert.Error(err)
return
}
assert.NoError(err)
assert.Equal(tc.wantUID, uid)
})
}
}
func TestSelfGetInstance(t *testing.T) {
assert := assert.New(t)
require := require.New(t)
cloud := &Cloud{
imds: &stubIMDS{
projectID: "someProject",
zone: "someZone-west3-b",
instanceName: "someInstance",
},
instanceAPI: &stubInstanceAPI{
instance: &computepb.Instance{
Name: proto.String("someInstance"),
Zone: proto.String("someZone-west3-b"),
Labels: map[string]string{
cloud.TagUID: "1234",
cloud.TagRole: role.ControlPlane.String(),
},
NetworkInterfaces: []*computepb.NetworkInterface{
{
Name: proto.String("nic0"),
NetworkIP: proto.String("192.0.2.0"),
AliasIpRanges: []*computepb.AliasIpRange{
{
IpCidrRange: proto.String("192.0.3.0/8"),
},
},
Subnetwork: proto.String("projects/someProject/regions/someRegion/subnetworks/someSubnetwork"),
},
},
},
},
subnetAPI: &stubSubnetAPI{
subnet: &computepb.Subnetwork{
SecondaryIpRanges: []*computepb.SubnetworkSecondaryRange{
{
IpCidrRange: proto.String("198.51.100.0/24"),
},
},
},
},
}
self, err := cloud.Self(context.Background())
require.NoError(err)
instance, err := cloud.GetInstance(context.Background(), self.ProviderID)
require.NoError(err)
assert.Equal(self, instance)
}
type stubForwardingRulesAPI struct {
iterator forwardingRuleIterator
}
func (s *stubForwardingRulesAPI) List(
ctx context.Context, req *computepb.ListGlobalForwardingRulesRequest, opts ...gax.CallOption,
) forwardingRuleIterator {
return s.iterator
}
func (s *stubForwardingRulesAPI) Close() error { return nil }
type stubForwardingRulesIterator struct {
ctr int
forwardingRules []*computepb.ForwardingRule
err error
}
func (s *stubForwardingRulesIterator) Next() (*computepb.ForwardingRule, error) {
if s.err != nil {
return nil, s.err
}
if s.ctr >= len(s.forwardingRules) {
return nil, iterator.Done
}
s.ctr++
return s.forwardingRules[s.ctr-1], nil
}
type stubIMDS struct {
instanceID string
projectID string
zone string
instanceName string
instanceIDErr error
projectIDErr error
zoneErr error
instanceNameErr error
}
func (s *stubIMDS) InstanceID() (string, error) { return s.instanceID, s.instanceIDErr }
func (s *stubIMDS) ProjectID() (string, error) { return s.projectID, s.projectIDErr }
func (s *stubIMDS) Zone() (string, error) { return s.zone, s.zoneErr }
func (s *stubIMDS) InstanceName() (string, error) { return s.instanceName, s.instanceNameErr }
type stubInstanceAPI struct {
instance *computepb.Instance
instanceErr error
iterator *stubInstanceIterator
}
func (s *stubInstanceAPI) Get(
ctx context.Context, req *computepb.GetInstanceRequest, opts ...gax.CallOption,
) (*computepb.Instance, error) {
return s.instance, s.instanceErr
}
func (s *stubInstanceAPI) List(
ctx context.Context, req *computepb.ListInstancesRequest, opts ...gax.CallOption,
) instanceIterator {
return s.iterator
}
func (s *stubInstanceAPI) Close() error { return nil }
type stubInstanceIterator struct {
ctr int
instances []*computepb.Instance
err error
}
func (s *stubInstanceIterator) Next() (*computepb.Instance, error) {
if s.err != nil {
return nil, s.err
}
if s.ctr >= len(s.instances) {
return nil, iterator.Done
}
s.ctr++
return s.instances[s.ctr-1], nil
}
type stubSubnetAPI struct {
subnet *computepb.Subnetwork
subnetErr error
}
func (s *stubSubnetAPI) Get(
ctx context.Context, req *computepb.GetSubnetworkRequest, opts ...gax.CallOption,
) (*computepb.Subnetwork, error) {
return s.subnet, s.subnetErr
}
func (s *stubSubnetAPI) Close() error { return nil }

35
internal/cloud/gcp/cloudnodemanager.go
View File

@ -1,35 +0,0 @@
/*
Copyright (c) Edgeless Systems GmbH
SPDX-License-Identifier: AGPL-3.0-only
*/
package gcp
import "github.com/edgelesssys/constellation/v2/internal/versions"
// CloudNodeManager holds the GCP cloud-node-manager configuration.
type CloudNodeManager struct{}
// Image returns the container image used to provide cloud-node-manager for the cloud-provider.
// Not used on GCP.
func (c *CloudNodeManager) Image(k8sVersion versions.ValidK8sVersion) (string, error) {
return "", nil
}
// Path returns the path used by cloud-node-manager executable within the container image.
// Not used on GCP.
func (c *CloudNodeManager) Path() string {
return ""
}
// ExtraArgs returns a list of arguments to append to the cloud-node-manager command.
// Not used on GCP.
func (c *CloudNodeManager) ExtraArgs() []string {
return []string{}
}
// Supported is used to determine if cloud node manager is implemented for this cloud provider.
func (c *CloudNodeManager) Supported() bool {
return false
}

23
internal/cloud/gcp/cloudnodemanager_test.go
View File

@ -1,23 +0,0 @@
/*
Copyright (c) Edgeless Systems GmbH
SPDX-License-Identifier: AGPL-3.0-only
*/
package gcp
import (
"testing"
"github.com/stretchr/testify/assert"
)
func TestTrivialCNMFunctions(t *testing.T) {
assert := assert.New(t)
cloud := CloudNodeManager{}
assert.Empty(cloud.Image(""))
assert.Empty(cloud.Path())
assert.Empty(cloud.ExtraArgs())
assert.False(cloud.Supported())
}

39
internal/cloud/gcp/api.go → internal/cloud/gcp/interface.go
View File

@ -9,50 +9,29 @@ package gcp
import ( import (
"context" "context"
compute "cloud.google.com/go/compute/apiv1"
"github.com/googleapis/gax-go/v2" "github.com/googleapis/gax-go/v2"
computepb "google.golang.org/genproto/googleapis/cloud/compute/v1" computepb "google.golang.org/genproto/googleapis/cloud/compute/v1"
) )
type instanceAPI interface {
Get(ctx context.Context, req *computepb.GetInstanceRequest, opts ...gax.CallOption) (*computepb.Instance, error)
List(ctx context.Context, req *computepb.ListInstancesRequest, opts ...gax.CallOption) InstanceIterator
SetMetadata(ctx context.Context, req *computepb.SetMetadataInstanceRequest, opts ...gax.CallOption) (*compute.Operation, error)
Close() error
}
type subnetworkAPI interface {
List(ctx context.Context, req *computepb.ListSubnetworksRequest, opts ...gax.CallOption) SubnetworkIterator
Get(ctx context.Context, req *computepb.GetSubnetworkRequest, opts ...gax.CallOption) (*computepb.Subnetwork, error)
Close() error
}
type forwardingRulesAPI interface { type forwardingRulesAPI interface {
List(ctx context.Context, req *computepb.ListGlobalForwardingRulesRequest, opts ...gax.CallOption) ForwardingRuleIterator List(ctx context.Context, req *computepb.ListGlobalForwardingRulesRequest, opts ...gax.CallOption) forwardingRuleIterator
Close() error Close() error
} }
type metadataAPI interface { type imdsAPI interface {
InstanceAttributeValue(attr string) (string, error)
InstanceID() (string, error) InstanceID() (string, error)
ProjectID() (string, error) ProjectID() (string, error)
Zone() (string, error) Zone() (string, error)
InstanceName() (string, error) InstanceName() (string, error)
} }
type Operation interface { type instanceAPI interface {
Proto() *computepb.Operation Get(ctx context.Context, req *computepb.GetInstanceRequest, opts ...gax.CallOption) (*computepb.Instance, error)
List(ctx context.Context, req *computepb.ListInstancesRequest, opts ...gax.CallOption) instanceIterator
Close() error
} }
type InstanceIterator interface { type subnetAPI interface {
Next() (*computepb.Instance, error) Get(ctx context.Context, req *computepb.GetSubnetworkRequest, opts ...gax.CallOption) (*computepb.Subnetwork, error)
} Close() error
type SubnetworkIterator interface {
Next() (*computepb.Subnetwork, error)
}
type ForwardingRuleIterator interface {
Next() (*computepb.ForwardingRule, error)
} }

10
internal/cloud/gcp/logger.go
View File

@ -8,10 +8,11 @@ package gcp
import ( import (
"context" "context"
"fmt"
"log" "log"
"cloud.google.com/go/compute/metadata"
"cloud.google.com/go/logging" "cloud.google.com/go/logging"
"github.com/edgelesssys/constellation/v2/internal/gcpshared"
) )
type Logger struct { type Logger struct {
@ -21,11 +22,12 @@ type Logger struct {
// NewLogger creates a new Cloud Logger for GCP. // NewLogger creates a new Cloud Logger for GCP.
// https://cloud.google.com/logging/docs/setup/go // https://cloud.google.com/logging/docs/setup/go
func NewLogger(ctx context.Context, providerID string, logName string) (*Logger, error) { func NewLogger(ctx context.Context, logName string) (*Logger, error) {
projectID, _, _, err := gcpshared.SplitProviderID(providerID) projectID, err := metadata.NewClient(nil).ProjectID()
if err != nil { if err != nil {
return nil, err return nil, fmt.Errorf("retrieving project ID from imds: %w", err)
} }
client, err := logging.NewClient(ctx, projectID) client, err := logging.NewClient(ctx, projectID)
if err != nil { if err != nil {
return nil, err return nil, err

131
internal/cloud/gcp/metadata.go
View File

@ -1,131 +0,0 @@
/*
Copyright (c) Edgeless Systems GmbH
SPDX-License-Identifier: AGPL-3.0-only
*/
package gcp
import (
"context"
"fmt"
"github.com/edgelesssys/constellation/v2/internal/cloud/metadata"
"github.com/edgelesssys/constellation/v2/internal/gcpshared"
)
// API handles all GCP API requests.
type API interface {
// UID retrieves the current instances uid.
UID(context.Context) (string, error)
// RetrieveInstances retrieves a list of all accessible GCP instances with their metadata.
RetrieveInstances(ctx context.Context, project, zone string) ([]metadata.InstanceMetadata, error)
// RetrieveInstances retrieves a single GCP instances with its metadata.
RetrieveInstance(ctx context.Context, project, zone, instanceName string) (metadata.InstanceMetadata, error)
// RetrieveInstanceMetadata retrieves the GCP instance metadata of the current instance.
RetrieveInstanceMetadata(attr string) (string, error)
// RetrieveProjectID retrieves the GCP projectID containing the current instance.
RetrieveProjectID() (string, error)
// RetrieveZone retrieves the GCP zone containing the current instance.
RetrieveZone() (string, error)
// RetrieveInstanceName retrieves the instance name of the current instance.
RetrieveInstanceName() (string, error)
// RetrieveSubnetworkAliasCIDR retrieves the subnetwork CIDR of the current instance.
RetrieveSubnetworkAliasCIDR(ctx context.Context, project, zone, instanceName string) (string, error)
// RetrieveLoadBalancerEndpoint retrieves the load balancer endpoint of the current instance.
RetrieveLoadBalancerEndpoint(ctx context.Context, project string) (string, error)
// SetInstanceMetadata sets metadata key: value of the instance specified by project, zone and instanceName.
SetInstanceMetadata(ctx context.Context, project, zone, instanceName, key, value string) error
// UnsetInstanceMetadata removes a metadata key-value pair of the instance specified by project, zone and instanceName.
UnsetInstanceMetadata(ctx context.Context, project, zone, instanceName, key string) error
}
// Metadata implements core.ProviderMetadata interface.
type Metadata struct {
api API
}
// New creates a new Provider with real API and FS.
func New(api API) *Metadata {
return &Metadata{
api: api,
}
}
// List retrieves all instances belonging to the current constellation.
func (m *Metadata) List(ctx context.Context) ([]metadata.InstanceMetadata, error) {
project, err := m.api.RetrieveProjectID()
if err != nil {
return nil, err
}
zone, err := m.api.RetrieveZone()
if err != nil {
return nil, err
}
instances, err := m.api.RetrieveInstances(ctx, project, zone)
if err != nil {
return nil, fmt.Errorf("retrieving instances list from GCP api: %w", err)
}
return instances, nil
}
// Self retrieves the current instance.
func (m *Metadata) Self(ctx context.Context) (metadata.InstanceMetadata, error) {
project, err := m.api.RetrieveProjectID()
if err != nil {
return metadata.InstanceMetadata{}, err
}
zone, err := m.api.RetrieveZone()
if err != nil {
return metadata.InstanceMetadata{}, err
}
instanceName, err := m.api.RetrieveInstanceName()
if err != nil {
return metadata.InstanceMetadata{}, err
}
subnetCIDR, err := m.api.RetrieveSubnetworkAliasCIDR(ctx, project, zone, instanceName)
if err != nil {
return metadata.InstanceMetadata{}, err
}
instance, err := m.api.RetrieveInstance(ctx, project, zone, instanceName)
if err != nil {
return metadata.InstanceMetadata{}, err
}
instance.SecondaryIPRange = subnetCIDR
return instance, nil
}
// GetInstance retrieves an instance using its providerID.
func (m *Metadata) GetInstance(ctx context.Context, providerID string) (metadata.InstanceMetadata, error) {
project, zone, instanceName, err := gcpshared.SplitProviderID(providerID)
if err != nil {
return metadata.InstanceMetadata{}, fmt.Errorf("invalid providerID: %w", err)
}
return m.api.RetrieveInstance(ctx, project, zone, instanceName)
}
// SupportsLoadBalancer returns true if the cloud provider supports load balancers.
func (m *Metadata) SupportsLoadBalancer() bool {
return true
}
// GetLoadBalancerEndpoint returns the endpoint of the load balancer.
func (m *Metadata) GetLoadBalancerEndpoint(ctx context.Context) (string, error) {
project, err := m.api.RetrieveProjectID()
if err != nil {
return "", err
}
return m.api.RetrieveLoadBalancerEndpoint(ctx, project)
}
// UID retrieves the UID of the constellation.
func (m *Metadata) UID(ctx context.Context) (string, error) {
return m.api.UID(ctx)
}
// Supported is used to determine if metadata API is implemented for this cloud provider.
func (m *Metadata) Supported() bool {
return true
}

326
internal/cloud/gcp/metadata_test.go
View File

@ -1,326 +0,0 @@
/*
Copyright (c) Edgeless Systems GmbH
SPDX-License-Identifier: AGPL-3.0-only
*/
package gcp
import (
"context"
"errors"
"testing"
"github.com/edgelesssys/constellation/v2/internal/cloud"
"github.com/edgelesssys/constellation/v2/internal/cloud/metadata"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestList(t *testing.T) {
err := errors.New("some err")
uid := "1234"
instancesGenerator := func() *[]metadata.InstanceMetadata {
return &[]metadata.InstanceMetadata{
{
Name: "someInstance",
ProviderID: "gce://someProject/someZone/someInstance",
VPCIP: "192.0.2.0",
},
}
}
testCases := map[string]struct {
client stubGCPClient
instancesGenerator func() *[]metadata.InstanceMetadata
instancesMutator func(*[]metadata.InstanceMetadata)
wantErr bool
wantInstances []metadata.InstanceMetadata
}{
"retrieve works": {
client: stubGCPClient{
projectID: "someProjectID",
zone: "someZone",
retrieveInstanceMetadaValues: map[string]string{
cloud.TagUID: uid,
},
},
instancesGenerator: instancesGenerator,
wantInstances: []metadata.InstanceMetadata{
{
Name: "someInstance",
ProviderID: "gce://someProject/someZone/someInstance",
VPCIP: "192.0.2.0",
},
},
},
"retrieve error is detected": {
client: stubGCPClient{
projectID: "someProjectID",
zone: "someZone",
retrieveInstanceMetadaValues: map[string]string{
cloud.TagUID: uid,
},
retrieveInstancesErr: err,
},
instancesGenerator: instancesGenerator,
wantErr: true,
},
"project metadata retrieval error is detected": {
client: stubGCPClient{
retrieveProjectIDErr: err,
},
instancesGenerator: instancesGenerator,
wantErr: true,
},
"zone retrieval error is detected": {
client: stubGCPClient{
retrieveZoneErr: err,
},
instancesGenerator: instancesGenerator,
wantErr: true,
},
}
for name, tc := range testCases {
t.Run(name, func(t *testing.T) {
assert := assert.New(t)
require := require.New(t)
tc.client.retrieveInstancesValues = *tc.instancesGenerator()
if tc.instancesMutator != nil {
tc.instancesMutator(&tc.client.retrieveInstancesValues)
}
metadata := New(&tc.client)
instances, err := metadata.List(context.Background())
if tc.wantErr {
assert.Error(err)
return
}
require.NoError(err)
assert.ElementsMatch(tc.wantInstances, instances)
})
}
}
func TestSelf(t *testing.T) {
err := errors.New("some err")
uid := "1234"
testCases := map[string]struct {
client stubGCPClient
wantErr bool
wantInstance metadata.InstanceMetadata
}{
"retrieve works": {
client: stubGCPClient{
projectID: "someProjectID",
zone: "someZone",
retrieveInstanceValue: metadata.InstanceMetadata{
Name: "someInstance",
ProviderID: "gce://someProject/someZone/someInstance",
VPCIP: "192.0.2.0",
},
},
wantInstance: metadata.InstanceMetadata{
Name: "someInstance",
ProviderID: "gce://someProject/someZone/someInstance",
VPCIP: "192.0.2.0",
},
},
"retrieve error is detected": {
client: stubGCPClient{
projectID: "someProjectID",
zone: "someZone",
retrieveInstanceMetadaValues: map[string]string{
cloud.TagUID: uid,
},
retrieveInstanceErr: err,
},
wantErr: true,
},
"project id retrieval error is detected": {
client: stubGCPClient{
retrieveProjectIDErr: err,
},
wantErr: true,
},
"zone retrieval error is detected": {
client: stubGCPClient{
retrieveZoneErr: err,
},
wantErr: true,
},
"instance name retrieval error is detected": {
client: stubGCPClient{
retrieveInstanceNameErr: err,
},
wantErr: true,
},
}
for name, tc := range testCases {
t.Run(name, func(t *testing.T) {
assert := assert.New(t)
require := require.New(t)
cloud := New(&tc.client)
instance, err := cloud.Self(context.Background())
if tc.wantErr {
assert.Error(err)
return
}
require.NoError(err)
assert.Equal(tc.wantInstance, instance)
})
}
}
func TestGetInstance(t *testing.T) {
err := errors.New("some err")
testCases := map[string]struct {
providerID string
client stubGCPClient
wantErr bool
wantInstance metadata.InstanceMetadata
}{
"retrieve works": {
providerID: "gce://someProject/someZone/someInstance",
client: stubGCPClient{
retrieveInstanceValue: metadata.InstanceMetadata{
Name: "someInstance",
ProviderID: "gce://someProject/someZone/someInstance",
VPCIP: "192.0.2.0",
},
},
wantInstance: metadata.InstanceMetadata{
Name: "someInstance",
ProviderID: "gce://someProject/someZone/someInstance",
VPCIP: "192.0.2.0",
},
},
"retrieve error is detected": {
providerID: "gce://someProject/someZone/someInstance",
client: stubGCPClient{
retrieveInstanceErr: err,
},
wantErr: true,
},
"malformed providerID with too many fields is detected": {
providerID: "gce://someProject/someZone/someInstance/tooMany/fields",
wantErr: true,
},
"malformed providerID with too few fields is detected": {
providerID: "gce://someProject",
wantErr: true,
},
}
for name, tc := range testCases {
t.Run(name, func(t *testing.T) {
assert := assert.New(t)
require := require.New(t)
cloud := New(&tc.client)
instance, err := cloud.GetInstance(context.Background(), tc.providerID)
if tc.wantErr {
assert.Error(err)
return
}
require.NoError(err)
assert.Equal(tc.wantInstance, instance)
})
}
}
type stubGCPClient struct {
retrieveUIDValue string
retrieveUIDErr error
retrieveInstanceValue metadata.InstanceMetadata
retrieveInstanceErr error
retrieveInstancesValues []metadata.InstanceMetadata
retrieveInstancesErr error
retrieveInstanceMetadaValues map[string]string
retrieveInstanceMetadataErr error
retrieveSubnetworkAliasErr error
projectID string
zone string
instanceName string
loadBalancerIP string
retrieveProjectIDErr error
retrieveZoneErr error
retrieveInstanceNameErr error
setInstanceMetadataErr error
unsetInstanceMetadataErr error
retrieveLoadBalancerErr error
instanceMetadataProjects []string
instanceMetadataZones []string
instanceMetadataInstanceNames []string
instanceMetadataKeys []string
instanceMetadataValues []string
unsetMetadataProjects []string
unsetMetadataZones []string
unsetMetadataInstanceNames []string
unsetMetadataKeys []string
}
func (s *stubGCPClient) RetrieveInstances(ctx context.Context, project, zone string) ([]metadata.InstanceMetadata, error) {
return s.retrieveInstancesValues, s.retrieveInstancesErr
}
func (s *stubGCPClient) RetrieveInstance(ctx context.Context, project, zone string, instanceName string) (metadata.InstanceMetadata, error) {
return s.retrieveInstanceValue, s.retrieveInstanceErr
}
func (s *stubGCPClient) RetrieveInstanceMetadata(attr string) (string, error) {
return s.retrieveInstanceMetadaValues[attr], s.retrieveInstanceMetadataErr
}
func (s *stubGCPClient) RetrieveProjectID() (string, error) {
return s.projectID, s.retrieveProjectIDErr
}
func (s *stubGCPClient) RetrieveZone() (string, error) {
return s.zone, s.retrieveZoneErr
}
func (s *stubGCPClient) RetrieveInstanceName() (string, error) {
return s.instanceName, s.retrieveInstanceNameErr
}
func (s *stubGCPClient) RetrieveLoadBalancerEndpoint(ctx context.Context, project string) (string, error) {
return s.loadBalancerIP, s.retrieveLoadBalancerErr
}
func (s *stubGCPClient) UID(context.Context) (string, error) {
return s.retrieveUIDValue, s.retrieveUIDErr
}
func (s *stubGCPClient) SetInstanceMetadata(ctx context.Context, project, zone, instanceName, key, value string) error {
s.instanceMetadataProjects = append(s.instanceMetadataProjects, project)
s.instanceMetadataZones = append(s.instanceMetadataZones, zone)
s.instanceMetadataInstanceNames = append(s.instanceMetadataInstanceNames, instanceName)
s.instanceMetadataKeys = append(s.instanceMetadataKeys, key)
s.instanceMetadataValues = append(s.instanceMetadataValues, value)
return s.setInstanceMetadataErr
}
func (s *stubGCPClient) UnsetInstanceMetadata(ctx context.Context, project, zone, instanceName, key string) error {
s.unsetMetadataProjects = append(s.unsetMetadataProjects, project)
s.unsetMetadataZones = append(s.unsetMetadataZones, zone)
s.unsetMetadataInstanceNames = append(s.unsetMetadataInstanceNames, instanceName)
s.unsetMetadataKeys = append(s.unsetMetadataKeys, key)
return s.unsetInstanceMetadataErr
}
func (s *stubGCPClient) RetrieveSubnetworkAliasCIDR(ctx context.Context, project, zone, instanceName string) (string, error) {
return "", s.retrieveSubnetworkAliasErr
}

65
internal/cloud/gcp/wrappers.go
View File

@ -10,43 +10,16 @@ import (
"context" "context"
compute "cloud.google.com/go/compute/apiv1" compute "cloud.google.com/go/compute/apiv1"
"cloud.google.com/go/compute/metadata"
"github.com/googleapis/gax-go/v2" "github.com/googleapis/gax-go/v2"
computepb "google.golang.org/genproto/googleapis/cloud/compute/v1" computepb "google.golang.org/genproto/googleapis/cloud/compute/v1"
) )
type instanceClient struct { type forwardingRuleIterator interface {
*compute.InstancesClient Next() (*computepb.ForwardingRule, error)
} }
func (c *instanceClient) Close() error { type instanceIterator interface {
return c.InstancesClient.Close() Next() (*computepb.Instance, error)
}
func (c *instanceClient) List(ctx context.Context, req *computepb.ListInstancesRequest,
opts ...gax.CallOption,
) InstanceIterator {
return c.InstancesClient.List(ctx, req)
}
type subnetworkClient struct {
*compute.SubnetworksClient
}
func (c *subnetworkClient) Close() error {
return c.SubnetworksClient.Close()
}
func (c *subnetworkClient) List(ctx context.Context, req *computepb.ListSubnetworksRequest,
opts ...gax.CallOption,
) SubnetworkIterator {
return c.SubnetworksClient.List(ctx, req)
}
func (c *subnetworkClient) Get(ctx context.Context, req *computepb.GetSubnetworkRequest,
opts ...gax.CallOption,
) (*computepb.Subnetwork, error) {
return c.SubnetworksClient.Get(ctx, req)
} }
type forwardingRulesClient struct { type forwardingRulesClient struct {
@ -59,32 +32,20 @@ func (c *forwardingRulesClient) Close() error {
func (c *forwardingRulesClient) List(ctx context.Context, req *computepb.ListGlobalForwardingRulesRequest, func (c *forwardingRulesClient) List(ctx context.Context, req *computepb.ListGlobalForwardingRulesRequest,
opts ...gax.CallOption, opts ...gax.CallOption,
) ForwardingRuleIterator { ) forwardingRuleIterator {
return c.GlobalForwardingRulesClient.List(ctx, req) return c.GlobalForwardingRulesClient.List(ctx, req)
} }
type metadataClient struct{} type instanceClient struct {
*compute.InstancesClient
func (c *metadataClient) InstanceAttributeValue(attr string) (string, error) {
return metadata.InstanceAttributeValue(attr)
} }
func (c *metadataClient) InstanceID() (string, error) { func (c *instanceClient) Close() error {
return metadata.InstanceID() return c.InstancesClient.Close()
} }
func (c *metadataClient) ProjectID() (string, error) { func (c *instanceClient) List(ctx context.Context, req *computepb.ListInstancesRequest,
return metadata.ProjectID() opts ...gax.CallOption,
} ) instanceIterator {
return c.InstancesClient.List(ctx, req)
func (c *metadataClient) Zone() (string, error) {
return metadata.Zone()
}
func (c *metadataClient) InstanceName() (string, error) {
return metadata.InstanceName()
}
func (c *metadataClient) ProjectAttributeValue(attr string) (string, error) {
return metadata.ProjectAttributeValue(attr)
} }

26
internal/cloud/gcp/writer.go
View File

@ -1,26 +0,0 @@
/*
Copyright (c) Edgeless Systems GmbH
SPDX-License-Identifier: AGPL-3.0-only
*/
package gcp
import (
"fmt"
"github.com/spf13/afero"
)
// Writer implements ConfigWriter.
type Writer struct {
fs afero.Afero
}
// WriteGCEConf persists the GCE config on disk.
func (w *Writer) WriteGCEConf(config string) error {
if err := w.fs.WriteFile("/etc/gce.conf", []byte(config), 0o644); err != nil {
return fmt.Errorf("writing gce config: %w", err)
}
return nil
}

60
internal/cloud/gcp/writer_test.go
View File

@ -1,60 +0,0 @@
/*
Copyright (c) Edgeless Systems GmbH
SPDX-License-Identifier: AGPL-3.0-only
*/
package gcp
import (
"testing"
"github.com/spf13/afero"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestWriteGCEConf(t *testing.T) {
config := "someConfig"
testCases := map[string]struct {
fs afero.Afero
wantValue string
wantErr bool
}{
"write works": {
fs: afero.Afero{
Fs: afero.NewMemMapFs(),
},
wantValue: config,
wantErr: false,
},
"write fails": {
fs: afero.Afero{
Fs: afero.NewReadOnlyFs(afero.NewMemMapFs()),
},
wantErr: true,
},
}
for name, tc := range testCases {
t.Run(name, func(t *testing.T) {
assert := assert.New(t)
require := require.New(t)
writer := Writer{
fs: tc.fs,
}
err := writer.WriteGCEConf(config)
if tc.wantErr {
assert.Error(err)
return
}
require.NoError(err)
value, err := tc.fs.ReadFile("/etc/gce.conf")
assert.NoError(err)
assert.Equal(tc.wantValue, string(value))
})
}
}

15
internal/cloud/gcpshared/doc.go Normal file
View File

@ -0,0 +1,15 @@
/*
Copyright (c) Edgeless Systems GmbH
SPDX-License-Identifier: AGPL-3.0-only
*/
/*
Package gcpshared contains code to parse and define data types
relevant for Google Cloud Platform.
This package is intended to have a minimal size and surface. If you
have GCP related code that is not shared by multiple applications,
or if the code interacts with the GCP API, please keep the code in
the application's internal package or add it to the GCP cloud package.
*/
package gcpshared

4
internal/gcpshared/metadata.go → internal/cloud/gcpshared/providerid.go
View File

@ -13,8 +13,8 @@ import (
var providerIDRegex = regexp.MustCompile(`^gce://([^/]+)/([^/]+)/([^/]+)$`) var providerIDRegex = regexp.MustCompile(`^gce://([^/]+)/([^/]+)/([^/]+)$`)
// SplitProviderID splits a provider's id into core components. // SplitProviderID splits a k8s provider ID for GCP instances into its core components.
// A providerID is build after the schema 'gce://<project-id>/<zone>/<instance-name>' // A provider ID is build after the schema 'gce://<project-id>/<zone>/<instance-name>'
func SplitProviderID(providerID string) (project, zone, instance string, err error) { func SplitProviderID(providerID string) (project, zone, instance string, err error) {
matches := providerIDRegex.FindStringSubmatch(providerID) matches := providerIDRegex.FindStringSubmatch(providerID)
if len(matches) != 4 { if len(matches) != 4 {

5
internal/gcpshared/metadata_test.go → internal/cloud/gcpshared/providerid_test.go
View File

@ -10,13 +10,8 @@ import (
"testing" "testing"
"github.com/stretchr/testify/assert" "github.com/stretchr/testify/assert"
"go.uber.org/goleak"
) )
func TestMain(m *testing.M) {
goleak.VerifyTestMain(m)
}
func TestSplitProviderID(t *testing.T) { func TestSplitProviderID(t *testing.T) {
testCases := map[string]struct { testCases := map[string]struct {
providerID string providerID string

0
internal/gcpshared/serviceaccountkey.go → internal/cloud/gcpshared/serviceaccountkey.go
View File

0
internal/gcpshared/serviceaccountkey_test.go → internal/cloud/gcpshared/serviceaccountkey_test.go
View File

4
internal/cloud/metadata/metadata.go
View File

@ -23,11 +23,9 @@ type InstanceMetadata struct {
Role role.Role Role role.Role
// VPCIP is the primary IP address of the instance in the VPC. // VPCIP is the primary IP address of the instance in the VPC.
VPCIP string VPCIP string
// PublicIP is the primary public IP of the instance, if available, empty string otherwise.
PublicIP string
// SSHKeys maps usernames to ssh public keys. // SSHKeys maps usernames to ssh public keys.
// TODO: remove everywhere.
SSHKeys map[string][]string SSHKeys map[string][]string
// SecondaryIPRange is the VPC wide CIDR from which subnets are attached to VMs as AliasIPRanges. // SecondaryIPRange is the VPC wide CIDR from which subnets are attached to VMs as AliasIPRanges.
// May be empty on certain CSPs. // May be empty on certain CSPs.
SecondaryIPRange string SecondaryIPRange string

79
internal/cloud/qemu/ccm.go
View File

@ -1,79 +0,0 @@
/*
Copyright (c) Edgeless Systems GmbH
SPDX-License-Identifier: AGPL-3.0-only
*/
package qemu
import (
"context"
"github.com/edgelesssys/constellation/v2/internal/cloud/metadata"
"github.com/edgelesssys/constellation/v2/internal/kubernetes"
"github.com/edgelesssys/constellation/v2/internal/versions"
k8s "k8s.io/api/core/v1"
)
// CloudControllerManager holds the QEMU cloud-controller-manager configuration.
type CloudControllerManager struct{}
// Image returns the container image used to provide cloud-controller-manager for the cloud-provider.
func (c CloudControllerManager) Image(k8sVersion versions.ValidK8sVersion) (string, error) {
return "", nil
}
// Path returns the path used by cloud-controller-manager executable within the container image.
func (c CloudControllerManager) Path() string {
return "/qemu-cloud-controller-manager"
}
// Name returns the cloud-provider name as used by k8s cloud-controller-manager (k8s.gcr.io/cloud-controller-manager).
func (c CloudControllerManager) Name() string {
return "qemu"
}
// ExtraArgs returns a list of arguments to append to the cloud-controller-manager command.
func (c CloudControllerManager) ExtraArgs() []string {
return []string{}
}
// ConfigMaps returns a list of ConfigMaps to deploy together with the k8s cloud-controller-manager
// Reference: https://kubernetes.io/docs/concepts/configuration/configmap/ .
func (c CloudControllerManager) ConfigMaps() (kubernetes.ConfigMaps, error) {
return kubernetes.ConfigMaps{}, nil
}
// Secrets returns a list of secrets to deploy together with the k8s cloud-controller-manager.
// Reference: https://kubernetes.io/docs/concepts/configuration/secret/ .
func (c CloudControllerManager) Secrets(ctx context.Context, providerID, cloudServiceAccountURI string) (kubernetes.Secrets, error) {
return kubernetes.Secrets{}, nil
}
// Volumes returns a list of volumes to deploy together with the k8s cloud-controller-manager.
// Reference: https://kubernetes.io/docs/concepts/storage/volumes/ .
func (c CloudControllerManager) Volumes() []k8s.Volume {
return []k8s.Volume{}
}
// VolumeMounts a list of of volume mounts to deploy together with the k8s cloud-controller-manager.
func (c CloudControllerManager) VolumeMounts() []k8s.VolumeMount {
return []k8s.VolumeMount{}
}
// Env returns a list of k8s environment key-value pairs to deploy together with the k8s cloud-controller-manager.
func (c CloudControllerManager) Env() []k8s.EnvVar {
return []k8s.EnvVar{}
}
// PrepareInstance is called on every instance before deploying the cloud-controller-manager.
// Allows for cloud-provider specific hooks.
func (c CloudControllerManager) PrepareInstance(instance metadata.InstanceMetadata, vpnIP string) error {
// no specific hook required.
return nil
}
// Supported is used to determine if cloud controller manager is implemented for this cloud provider.
func (c CloudControllerManager) Supported() bool {
return false
}

35
internal/cloud/qemu/cloudnodemanager.go
View File

@ -1,35 +0,0 @@
/*
Copyright (c) Edgeless Systems GmbH
SPDX-License-Identifier: AGPL-3.0-only
*/
package qemu
import "github.com/edgelesssys/constellation/v2/internal/versions"
// CloudNodeManager holds the QEMU cloud-node-manager configuration.
type CloudNodeManager struct{}
// Image returns the container image used to provide cloud-node-manager for the cloud-provider.
// Not used on QEMU.
func (c *CloudNodeManager) Image(k8sVersion versions.ValidK8sVersion) (string, error) {
return "", nil
}
// Path returns the path used by cloud-node-manager executable within the container image.
// Not used on QEMU.
func (c *CloudNodeManager) Path() string {
return ""
}
// ExtraArgs returns a list of arguments to append to the cloud-node-manager command.
// Not used on QEMU.
func (c *CloudNodeManager) ExtraArgs() []string {
return []string{}
}
// Supported is used to determine if cloud node manager is implemented for this cloud provider.
func (c *CloudNodeManager) Supported() bool {
return false
}

5
internal/cloud/qemu/metadata.go
View File

@ -22,11 +22,6 @@ const qemuMetadataEndpoint = "10.42.0.1:8080"
// Metadata implements core.ProviderMetadata interface for QEMU. // Metadata implements core.ProviderMetadata interface for QEMU.
type Metadata struct{} type Metadata struct{}
// Supported is used to determine if metadata API is implemented for this cloud provider.
func (m *Metadata) Supported() bool {
return true
}
// List retrieves all instances belonging to the current constellation. // List retrieves all instances belonging to the current constellation.
func (m *Metadata) List(ctx context.Context) ([]metadata.InstanceMetadata, error) { func (m *Metadata) List(ctx context.Context) ([]metadata.InstanceMetadata, error) {
instancesRaw, err := m.retrieveMetadata(ctx, "/peers") instancesRaw, err := m.retrieveMetadata(ctx, "/peers")

16
internal/gcpshared/doc.go
View File

@ -1,16 +0,0 @@
/*
Copyright (c) Edgeless Systems GmbH
SPDX-License-Identifier: AGPL-3.0-only
*/
package gcpshared
/*
Package gcpshared contains code that is related to Google Cloud Platform
and is used by multiple microservices.
This package is intended to have a minimal size and surface. If you
have GCP related code that is not shared by multiple microservices,
please keep the code in the microservice's internal package.
*/

5
joinservice/cmd/main.go
View File

@ -130,11 +130,12 @@ func getVPCIP(ctx context.Context, provider string) (string, error) {
return "", err return "", err
} }
case cloudprovider.GCP: case cloudprovider.GCP:
gcpClient, err := gcpcloud.NewClient(ctx) gcpMeta, err := gcpcloud.New(ctx)
if err != nil { if err != nil {
return "", err return "", err
} }
metadata = gcpcloud.New(gcpClient) defer gcpMeta.Close()
metadata = gcpMeta
case cloudprovider.QEMU: case cloudprovider.QEMU:
metadata = &qemucloud.Metadata{} metadata = &qemucloud.Metadata{}
default: default: