Cloud provider GCP: adopt changes to CCM / CNM for GCP

This commit is contained in:
Malte Poll 2022-03-25 10:55:37 +01:00 committed by Malte Poll
parent a59ce30e7b
commit 3c1ddfb94e
9 changed files with 430 additions and 60 deletions

View File

@ -1,35 +1,23 @@
package gcp package gcp
import ( import (
"encoding/json"
"fmt" "fmt"
"strings" "strings"
"github.com/edgelesssys/constellation/coordinator/cloudprovider"
"github.com/edgelesssys/constellation/coordinator/core" "github.com/edgelesssys/constellation/coordinator/core"
"github.com/spf13/afero" "github.com/edgelesssys/constellation/coordinator/kubernetes/k8sapi/resources"
k8s "k8s.io/api/core/v1"
v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
) )
// ConfigWriter abstracts writing of the /etc/gce.conf config file.
type ConfigWriter interface {
// WriteGCEConf writes the config to the filesystem of the current instance.
WriteGCEConf(config string) error
}
// CloudControllerManager holds the gcp cloud-controller-manager configuration. // CloudControllerManager holds the gcp cloud-controller-manager configuration.
type CloudControllerManager struct { type CloudControllerManager struct{}
writer ConfigWriter
}
// NewCCM creates a new CloudControllerManager.
func NewCCM() *CloudControllerManager {
return &CloudControllerManager{
writer: &Writer{fs: afero.Afero{Fs: afero.NewOsFs()}},
}
}
// Image returns the container image used to provide cloud-controller-manager for the cloud-provider. // Image returns the container image used to provide cloud-controller-manager for the cloud-provider.
func (c *CloudControllerManager) Image() string { func (c *CloudControllerManager) Image() string {
// TODO: use newer "cloud-provider-gcp" from https://github.com/kubernetes/cloud-provider-gcp when newer releases are available return cloudprovider.CloudControllerManagerImageGCP
return "ghcr.io/malt3/cloud-provider-gcp:latest"
} }
// Path returns the path used by cloud-controller-manager executable within the container image. // Path returns the path used by cloud-controller-manager executable within the container image.
@ -42,21 +30,130 @@ func (c *CloudControllerManager) Name() string {
return "gce" return "gce"
} }
// PrepareInstance is called on every instance before deploying the cloud-controller-manager. // ExtraArgs returns a list of arguments to append to the cloud-controller-manager command.
// Allows for cloud-provider specific hooks. func (c *CloudControllerManager) ExtraArgs() []string {
func (c *CloudControllerManager) PrepareInstance(instance core.Instance, vpnIP string) error { return []string{
// GCP CCM expects "/etc/gce.conf" to contain the GCP project-id and other configuration. "--use-service-account-credentials",
"--controllers=cloud-node,cloud-node-lifecycle",
"--cloud-config=/etc/gce/gce.conf",
}
}
// ConfigMaps returns a list of ConfigMaps to deploy together with the k8s cloud-controller-manager
// Reference: https://kubernetes.io/docs/concepts/configuration/configmap/ .
func (c *CloudControllerManager) ConfigMaps(instance core.Instance) (resources.ConfigMaps, error) {
// GCP CCM expects cloud config to contain the GCP project-id and other configuration.
// reference: https://github.com/kubernetes/cloud-provider-gcp/blob/master/cluster/gce/gci/configure-helper.sh#L791-L892 // reference: https://github.com/kubernetes/cloud-provider-gcp/blob/master/cluster/gce/gci/configure-helper.sh#L791-L892
var config strings.Builder var config strings.Builder
config.WriteString("[global]\n") config.WriteString("[global]\n")
projectID, _, _, err := splitProviderID(instance.ProviderID) projectID, _, _, err := splitProviderID(instance.ProviderID)
if err != nil { if err != nil {
return fmt.Errorf("retrieving GCP project-id failed: %w", err) return resources.ConfigMaps{}, err
} }
config.WriteString(fmt.Sprintf("project-id = %s\n", projectID)) config.WriteString(fmt.Sprintf("project-id = %s\n", projectID))
config.WriteString("use-metadata-server = false\n") config.WriteString("use-metadata-server = false\n")
return c.writer.WriteGCEConf(config.String()) return resources.ConfigMaps{
&k8s.ConfigMap{
TypeMeta: v1.TypeMeta{
Kind: "ConfigMap",
APIVersion: "v1",
},
ObjectMeta: v1.ObjectMeta{
Name: "gceconf",
Namespace: "kube-system",
},
Data: map[string]string{
"gce.conf": config.String(),
},
},
}, nil
}
// Secrets returns a list of secrets to deploy together with the k8s cloud-controller-manager.
// Reference: https://kubernetes.io/docs/concepts/configuration/secret/ .
func (c *CloudControllerManager) Secrets(instance core.Instance, cloudServiceAccountURI string) (resources.Secrets, error) {
serviceAccountKey, err := getServiceAccountKey(cloudServiceAccountURI)
if err != nil {
return resources.Secrets{}, err
}
rawKey, err := json.Marshal(serviceAccountKey)
if err != nil {
return resources.Secrets{}, err
}
return resources.Secrets{
&k8s.Secret{
TypeMeta: v1.TypeMeta{
Kind: "Secret",
APIVersion: "v1",
},
ObjectMeta: v1.ObjectMeta{
Name: "gcekey",
Namespace: "kube-system",
},
Data: map[string][]byte{
"key.json": rawKey,
},
},
}, nil
}
// Volumes returns a list of volumes to deploy together with the k8s cloud-controller-manager.
// Reference: https://kubernetes.io/docs/concepts/storage/volumes/ .
func (c *CloudControllerManager) Volumes() []k8s.Volume {
return []k8s.Volume{
{
Name: "gceconf",
VolumeSource: k8s.VolumeSource{
ConfigMap: &k8s.ConfigMapVolumeSource{
LocalObjectReference: k8s.LocalObjectReference{
Name: "gceconf",
},
},
},
},
{
Name: "gcekey",
VolumeSource: k8s.VolumeSource{
Secret: &k8s.SecretVolumeSource{
SecretName: "gcekey",
},
},
},
}
}
// VolumeMounts returns a list of volume mounts to deploy together with the k8s cloud-controller-manager.
func (c *CloudControllerManager) VolumeMounts() []k8s.VolumeMount {
return []k8s.VolumeMount{
{
Name: "gceconf",
ReadOnly: true,
MountPath: "/etc/gce",
},
{
Name: "gcekey",
ReadOnly: true,
MountPath: "/var/secrets/google",
},
}
}
// Env returns a list of k8s environment key-value pairs to deploy together with the k8s cloud-controller-manager.
func (c *CloudControllerManager) Env() []k8s.EnvVar {
return []k8s.EnvVar{
{
Name: "GOOGLE_APPLICATION_CREDENTIALS",
Value: "/var/secrets/google/key.json",
},
}
}
// PrepareInstance is called on every instance before deploying the cloud-controller-manager.
// Allows for cloud-provider specific hooks.
func (c *CloudControllerManager) PrepareInstance(instance core.Instance, vpnIP string) error {
return nil
} }
// Supported is used to determine if cloud controller manager is implemented for this cloud provider. // Supported is used to determine if cloud controller manager is implemented for this cloud provider.

View File

@ -1,36 +1,47 @@
package gcp package gcp
import ( import (
"errors" "encoding/json"
"testing" "testing"
"github.com/edgelesssys/constellation/cli/gcp/client"
"github.com/edgelesssys/constellation/coordinator/core" "github.com/edgelesssys/constellation/coordinator/core"
"github.com/edgelesssys/constellation/coordinator/kubernetes/k8sapi/resources"
"github.com/stretchr/testify/assert" "github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require" "github.com/stretchr/testify/require"
k8s "k8s.io/api/core/v1"
v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
) )
func TestPrepareInstance(t *testing.T) { func TestConfigMaps(t *testing.T) {
err := errors.New("some err")
vpnIP := "192.0.2.0"
instance := core.Instance{
Name: "someInstance",
ProviderID: "gce://someProjectID/someZone/someInstance",
IPs: []string{"192.0.2.0"},
}
testCases := map[string]struct { testCases := map[string]struct {
writer stubWriter instance core.Instance
expectedConfigMaps resources.ConfigMaps
expectErr bool expectErr bool
expectedConfig string
}{ }{
"prepare works": { "ConfigMaps works": {
expectedConfig: `[global] instance: core.Instance{ProviderID: "gce://project-id/zone/instance-name"},
project-id = someProjectID expectedConfigMaps: resources.ConfigMaps{
&k8s.ConfigMap{
TypeMeta: v1.TypeMeta{
Kind: "ConfigMap",
APIVersion: "v1",
},
ObjectMeta: v1.ObjectMeta{
Name: "gceconf",
Namespace: "kube-system",
},
Data: map[string]string{
"gce.conf": `[global]
project-id = project-id
use-metadata-server = false use-metadata-server = false
`, `,
}, },
"GCE conf write error is detected": { },
writer: stubWriter{writeErr: err}, },
},
"invalid providerID fails": {
instance: core.Instance{ProviderID: "invalid"},
expectErr: true, expectErr: true,
}, },
} }
@ -40,15 +51,77 @@ use-metadata-server = false
assert := assert.New(t) assert := assert.New(t)
require := require.New(t) require := require.New(t)
ccm := CloudControllerManager{writer: &tc.writer} cloud := CloudControllerManager{}
err := ccm.PrepareInstance(instance, vpnIP) configMaps, err := cloud.ConfigMaps(tc.instance)
if tc.expectErr { if tc.expectErr {
assert.Error(err) assert.Error(err)
return return
} }
require.NoError(err) require.NoError(err)
assert.ElementsMatch([]string{tc.expectedConfig}, tc.writer.configs) assert.Equal(tc.expectedConfigMaps, configMaps)
})
}
}
func TestSecrets(t *testing.T) {
serviceAccountKey := client.ServiceAccountKey{
Type: "type",
ProjectID: "project-id",
PrivateKeyID: "private-key-id",
PrivateKey: "private-key",
ClientEmail: "client-email",
ClientID: "client-id",
AuthURI: "auth-uri",
TokenURI: "token-uri",
AuthProviderX509CertURL: "auth-provider-x509-cert-url",
ClientX509CertURL: "client-x509-cert-url",
}
rawKey, err := json.Marshal(serviceAccountKey)
require.NoError(t, err)
testCases := map[string]struct {
instance core.Instance
cloudServiceAccountURI string
expectedSecrets resources.Secrets
expectErr bool
}{
"Secrets works": {
cloudServiceAccountURI: "serviceaccount://gcp?type=type&project_id=project-id&private_key_id=private-key-id&private_key=private-key&client_email=client-email&client_id=client-id&auth_uri=auth-uri&token_uri=token-uri&auth_provider_x509_cert_url=auth-provider-x509-cert-url&client_x509_cert_url=client-x509-cert-url",
expectedSecrets: resources.Secrets{
&k8s.Secret{
TypeMeta: v1.TypeMeta{
Kind: "Secret",
APIVersion: "v1",
},
ObjectMeta: v1.ObjectMeta{
Name: "gcekey",
Namespace: "kube-system",
},
Data: map[string][]byte{
"key.json": rawKey,
},
},
},
},
"invalid serviceAccountKey fails": {
cloudServiceAccountURI: "invalid",
expectErr: true,
},
}
for name, tc := range testCases {
t.Run(name, func(t *testing.T) {
assert := assert.New(t)
require := require.New(t)
cloud := CloudControllerManager{}
secrets, err := cloud.Secrets(tc.instance, tc.cloudServiceAccountURI)
if tc.expectErr {
assert.Error(err)
return
}
require.NoError(err)
assert.Equal(tc.expectedSecrets, secrets)
}) })
} }
} }
@ -60,16 +133,10 @@ func TestTrivialCCMFunctions(t *testing.T) {
assert.NotEmpty(cloud.Image()) assert.NotEmpty(cloud.Image())
assert.NotEmpty(cloud.Path()) assert.NotEmpty(cloud.Path())
assert.NotEmpty(cloud.Name()) assert.NotEmpty(cloud.Name())
assert.NotEmpty(cloud.ExtraArgs())
assert.NotEmpty(cloud.Volumes())
assert.NotEmpty(cloud.VolumeMounts())
assert.NotEmpty(cloud.Env())
assert.NoError(cloud.PrepareInstance(core.Instance{}, "192.0.2.0"))
assert.True(cloud.Supported()) assert.True(cloud.Supported())
} }
type stubWriter struct {
writeErr error
configs []string
}
func (s *stubWriter) WriteGCEConf(config string) error {
s.configs = append(s.configs, config)
return s.writeErr
}

View File

@ -2,7 +2,6 @@ package gcp
import ( import (
"context" "context"
"errors"
"fmt" "fmt"
"regexp" "regexp"
"strings" "strings"
@ -267,7 +266,7 @@ func splitProviderID(providerID string) (project, zone, instance string, err err
providerIDregex := regexp.MustCompile(`^gce://([^/]+)/([^/]+)/([^/]+)$`) providerIDregex := regexp.MustCompile(`^gce://([^/]+)/([^/]+)/([^/]+)$`)
matches := providerIDregex.FindStringSubmatch(providerID) matches := providerIDregex.FindStringSubmatch(providerID)
if len(matches) != 4 { if len(matches) != 4 {
return "", "", "", errors.New("error splitting providerID") return "", "", "", fmt.Errorf("error splitting providerID: %v", providerID)
} }
return matches[1], matches[2], matches[3], nil return matches[1], matches[2], matches[3], nil
} }

View File

@ -0,0 +1,27 @@
package gcp
// CloudNodeManager holds the GCP cloud-node-manager configuration.
type CloudNodeManager struct{}
// Image returns the container image used to provide cloud-node-manager for the cloud-provider.
// Not used on GCP.
func (c *CloudNodeManager) Image() string {
return ""
}
// Path returns the path used by cloud-node-manager executable within the container image.
// Not used on GCP.
func (c *CloudNodeManager) Path() string {
return ""
}
// ExtraArgs returns a list of arguments to append to the cloud-node-manager command.
// Not used on GCP.
func (c *CloudNodeManager) ExtraArgs() []string {
return []string{}
}
// Supported is used to determine if cloud node manager is implemented for this cloud provider.
func (c *CloudNodeManager) Supported() bool {
return false
}

View File

@ -0,0 +1,17 @@
package gcp
import (
"testing"
"github.com/stretchr/testify/assert"
)
func TestTrivialCNMFunctions(t *testing.T) {
assert := assert.New(t)
cloud := CloudNodeManager{}
assert.Empty(cloud.Image())
assert.Empty(cloud.Path())
assert.Empty(cloud.ExtraArgs())
assert.False(cloud.Supported())
}

View File

@ -0,0 +1,62 @@
package gcp
import (
"fmt"
"net/url"
"github.com/edgelesssys/constellation/cli/gcp/client"
)
// getServiceAccountKey converts a cloudServiceAccountURI into a GCP ServiceAccountKey.
func getServiceAccountKey(cloudServiceAccountURI string) (client.ServiceAccountKey, error) {
uri, err := url.Parse(cloudServiceAccountURI)
if err != nil {
return client.ServiceAccountKey{}, err
}
if uri.Scheme != "serviceaccount" {
return client.ServiceAccountKey{}, fmt.Errorf("invalid service account URI: invalid scheme: %s", uri.Scheme)
}
if uri.Host != "gcp" {
return client.ServiceAccountKey{}, fmt.Errorf("invalid service account URI: invalid host: %s", uri.Host)
}
query := uri.Query()
if query.Get("type") == "" {
return client.ServiceAccountKey{}, fmt.Errorf("invalid service account URI: missing parameter \"type\": %s", uri)
}
if query.Get("project_id") == "" {
return client.ServiceAccountKey{}, fmt.Errorf("invalid service account URI: missing parameter \"project_id\": %s", uri)
}
if query.Get("private_key_id") == "" {
return client.ServiceAccountKey{}, fmt.Errorf("invalid service account URI: missing parameter \"private_key_id\": %s", uri)
}
if query.Get("private_key") == "" {
return client.ServiceAccountKey{}, fmt.Errorf("invalid service account URI: missing parameter \"private_key\": %s", uri)
}
if query.Get("client_email") == "" {
return client.ServiceAccountKey{}, fmt.Errorf("invalid service account URI: missing parameter \"client_email\": %s", uri)
}
if query.Get("client_id") == "" {
return client.ServiceAccountKey{}, fmt.Errorf("invalid service account URI: missing parameter \"client_id\": %s", uri)
}
if query.Get("token_uri") == "" {
return client.ServiceAccountKey{}, fmt.Errorf("invalid service account URI: missing parameter \"token_uri\": %s", uri)
}
if query.Get("auth_provider_x509_cert_url") == "" {
return client.ServiceAccountKey{}, fmt.Errorf("invalid service account URI: missing parameter \"auth_provider_x509_cert_url\": %s", uri)
}
if query.Get("client_x509_cert_url") == "" {
return client.ServiceAccountKey{}, fmt.Errorf("invalid service account URI: missing parameter \"client_x509_cert_url\": %s", uri)
}
return client.ServiceAccountKey{
Type: query.Get("type"),
ProjectID: query.Get("project_id"),
PrivateKeyID: query.Get("private_key_id"),
PrivateKey: query.Get("private_key"),
ClientEmail: query.Get("client_email"),
ClientID: query.Get("client_id"),
AuthURI: query.Get("auth_uri"),
TokenURI: query.Get("token_uri"),
AuthProviderX509CertURL: query.Get("auth_provider_x509_cert_url"),
ClientX509CertURL: query.Get("client_x509_cert_url"),
}, nil
}

View File

@ -0,0 +1,97 @@
package gcp
import (
"testing"
"github.com/edgelesssys/constellation/cli/gcp/client"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestGetServiceAccountKey(t *testing.T) {
serviceAccountKey := client.ServiceAccountKey{
Type: "type",
ProjectID: "project-id",
PrivateKeyID: "private-key-id",
PrivateKey: "private-key",
ClientEmail: "client-email",
ClientID: "client-id",
AuthURI: "auth-uri",
TokenURI: "token-uri",
AuthProviderX509CertURL: "auth-provider-x509-cert-url",
ClientX509CertURL: "client-x509-cert-url",
}
testCases := map[string]struct {
cloudServiceAccountURI string
expectedKey client.ServiceAccountKey
expectErr bool
}{
"getServiceAccountKey works": {
cloudServiceAccountURI: "serviceaccount://gcp?type=type&project_id=project-id&private_key_id=private-key-id&private_key=private-key&client_email=client-email&client_id=client-id&auth_uri=auth-uri&token_uri=token-uri&auth_provider_x509_cert_url=auth-provider-x509-cert-url&client_x509_cert_url=client-x509-cert-url",
expectedKey: serviceAccountKey,
},
"missing type": {
cloudServiceAccountURI: "serviceaccount://gcp?project_id=project-id&private_key_id=private-key-id&private_key=private-key&client_email=client-email&client_id=client-id&auth_uri=auth-uri&token_uri=token-uri&auth_provider_x509_cert_url=auth-provider-x509-cert-url&client_x509_cert_url=client-x509-cert-url",
expectErr: true,
},
"missing project_id": {
cloudServiceAccountURI: "serviceaccount://gcp?type=type&private_key_id=private-key-id&private_key=private-key&client_email=client-email&client_id=client-id&auth_uri=auth-uri&token_uri=token-uri&auth_provider_x509_cert_url=auth-provider-x509-cert-url&client_x509_cert_url=client-x509-cert-url",
expectErr: true,
},
"missing private_key_id": {
cloudServiceAccountURI: "serviceaccount://gcp?type=type&project_id=project-id&private_key=private-key&client_email=client-email&client_id=client-id&auth_uri=auth-uri&token_uri=token-uri&auth_provider_x509_cert_url=auth-provider-x509-cert-url&client_x509_cert_url=client-x509-cert-url",
expectErr: true,
},
"missing private_key": {
cloudServiceAccountURI: "serviceaccount://gcp?type=type&project_id=project-id&private_key_id=private-key-id&client_email=client-email&client_id=client-id&auth_uri=auth-uri&token_uri=token-uri&auth_provider_x509_cert_url=auth-provider-x509-cert-url&client_x509_cert_url=client-x509-cert-url",
expectErr: true,
},
"missing client_email": {
cloudServiceAccountURI: "serviceaccount://gcp?type=type&project_id=project-id&private_key_id=private-key-id&private_key=private-key&client_id=client-id&auth_uri=auth-uri&token_uri=token-uri&auth_provider_x509_cert_url=auth-provider-x509-cert-url&client_x509_cert_url=client-x509-cert-url",
expectErr: true,
},
"missing client_id": {
cloudServiceAccountURI: "serviceaccount://gcp?type=type&project_id=project-id&private_key_id=private-key-id&private_key=private-key&client_email=client-email&auth_uri=auth-uri&token_uri=token-uri&auth_provider_x509_cert_url=auth-provider-x509-cert-url&client_x509_cert_url=client-x509-cert-url",
expectErr: true,
},
"missing token_uri": {
cloudServiceAccountURI: "serviceaccount://gcp?type=type&project_id=project-id&private_key_id=private-key-id&private_key=private-key&client_email=client-email&client_id=client-id&auth_uri=auth-uri&auth_provider_x509_cert_url=auth-provider-x509-cert-url&client_x509_cert_url=client-x509-cert-url",
expectErr: true,
},
"missing auth_provider_x509_cert_url": {
cloudServiceAccountURI: "serviceaccount://gcp?type=type&project_id=project-id&private_key_id=private-key-id&private_key=private-key&client_email=client-email&client_id=client-id&auth_uri=auth-uri&token_uri=token-uri&client_x509_cert_url=client-x509-cert-url",
expectErr: true,
},
"missing client_x509_cert_url": {
cloudServiceAccountURI: "serviceaccount://gcp?type=type&project_id=project-id&private_key_id=private-key-id&private_key=private-key&client_email=client-email&client_id=client-id&auth_uri=auth-uri&token_uri=token-uri&auth_provider_x509_cert_url=auth-provider-x509-cert-url",
expectErr: true,
},
"invalid URI fails": {
cloudServiceAccountURI: "\x00",
expectErr: true,
},
"incorrect URI scheme fails": {
cloudServiceAccountURI: "invalid",
expectErr: true,
},
"incorrect URI host fails": {
cloudServiceAccountURI: "serviceaccount://incorrect",
expectErr: true,
},
}
for name, tc := range testCases {
t.Run(name, func(t *testing.T) {
assert := assert.New(t)
require := require.New(t)
key, err := getServiceAccountKey(tc.cloudServiceAccountURI)
if tc.expectErr {
assert.Error(err)
return
}
require.NoError(err)
assert.Equal(tc.expectedKey, key)
})
}
}

View File

@ -3,4 +3,7 @@ package cloudprovider
const ( const (
// CloudControllerManagerImageAWS is the CCM image used on AWS. // CloudControllerManagerImageAWS is the CCM image used on AWS.
CloudControllerManagerImageAWS = "us.gcr.io/k8s-artifacts-prod/provider-aws/cloud-controller-manager:v1.22.0-alpha.0" CloudControllerManagerImageAWS = "us.gcr.io/k8s-artifacts-prod/provider-aws/cloud-controller-manager:v1.22.0-alpha.0"
// CloudControllerManagerImageGCP is the CCM image used on GCP.
// TODO: use newer "cloud-provider-gcp" from https://github.com/kubernetes/cloud-provider-gcp when newer releases are available.
CloudControllerManagerImageGCP = "ghcr.io/malt3/cloud-provider-gcp:latest"
) )

View File

@ -106,7 +106,8 @@ func main() {
log.Fatalf("creating GCP client failed: %v\n", err) log.Fatalf("creating GCP client failed: %v\n", err)
} }
metadata = gcpcloud.New(gcpClient) metadata = gcpcloud.New(gcpClient)
cloudControllerManager = gcpcloud.NewCCM() cloudControllerManager = &gcpcloud.CloudControllerManager{}
cloudNodeManager = &gcpcloud.CloudNodeManager{}
autoscaler = &gcpcloud.Autoscaler{} autoscaler = &gcpcloud.Autoscaler{}
bindIP = defaultIP bindIP = defaultIP
bindPort = defaultPort bindPort = defaultPort