Git-Mirrors/constellation
1
0
Fork 0
mirror of https://github.com/edgelesssys/constellation.git synced 2024-12-24 23:19:39 -05:00
Code Issues Packages Projects Releases Wiki Activity

Deploy Konnectivity

Browse Source
This commit is contained in:
Leonard Cohnen 2022-09-01 03:40:29 +02:00 committed by 3u13r
parent 15592e8f3f
commit 7163c161b6
32 changed files with 1243 additions and 496 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
CHANGELOG.md
View File

@ -34,6 +34,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
- GCP: Support for higher end N2D standard (128 & 224 vCPUs), *high-mem* and *high-cpu* VMs
- Add `constellation upgrade` to update node images in Constellation.
- Add cilium v1.12.1 with strict mode v2
- Konnectivity is now deployed for secure API server to node/pod/service communication.
### Changed
<!-- For changes in existing functionality. -->

25
bootstrapper/cmd/bootstrapper/main.go
View File

@ -11,7 +11,6 @@ import (
"encoding/json"
"flag"
"io"
"net"
"os"
"strconv"
@ -34,10 +33,8 @@ import (
"github.com/edgelesssys/constellation/internal/cloud/vmtype"
"github.com/edgelesssys/constellation/internal/constants"
"github.com/edgelesssys/constellation/internal/file"
"github.com/edgelesssys/constellation/internal/iproute"
"github.com/edgelesssys/constellation/internal/logger"
"github.com/edgelesssys/constellation/internal/oid"
"github.com/edgelesssys/constellation/internal/role"
"github.com/spf13/afero"
"go.uber.org/zap"
)
@ -107,9 +104,6 @@ func main() {
)
openTPM = vtpm.OpenVTPM
fs = afero.NewOsFs()
if err := setLoadbalancerRoute(ctx, metadata); err != nil {
log.With(zap.Error(err)).Fatalf("Failed to set loadbalancer route")
}
log.Infof("Added load balancer IP to routing table")
case cloudprovider.Azure:
pcrs, err := vtpm.GetSelectedPCRs(vtpm.OpenVTPM, vtpm.AzurePCRSelection)
@ -181,22 +175,3 @@ func main() {
run(issuer, openTPM, fileHandler, clusterInitJoiner, metadataAPI, bindIP, bindPort, log, cloudLogger)
}
func setLoadbalancerRoute(ctx context.Context, meta metadataAPI) error {
self, err := meta.Self(ctx)
if err != nil {
return err
}
if self.Role != role.ControlPlane {
return nil
}
endpoint, err := meta.GetLoadBalancerEndpoint(ctx)
if err != nil {
return err
}
ip, _, err := net.SplitHostPort(endpoint)
if err != nil {
return err
}
return iproute.AddToLocalRoutingTable(ctx, ip)
}

37
bootstrapper/internal/certificate/certificate.go Normal file
View File

@ -0,0 +1,37 @@
/*
Copyright (c) Edgeless Systems GmbH
SPDX-License-Identifier: AGPL-3.0-only
*/
package certificate
import (
"crypto/ecdsa"
"crypto/elliptic"
"crypto/rand"
"crypto/x509"
"encoding/pem"
)
// GetCertificateRequest returns a certificate request and matching private key.
func GetCertificateRequest(csrTemplate *x509.CertificateRequest) (certificateRequest []byte, privateKey []byte, err error) {
privK, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
if err != nil {
return nil, nil, err
}
keyBytes, err := x509.MarshalECPrivateKey(privK)
if err != nil {
return nil, nil, err
}
keyPem := pem.EncodeToMemory(&pem.Block{
Type: "EC PRIVATE KEY",
Bytes: keyBytes,
})
certificateRequest, err = x509.CreateCertificateRequest(rand.Reader, csrTemplate, privK)
if err != nil {
return nil, nil, err
}
return certificateRequest, keyPem, nil
}

24
bootstrapper/internal/kubelet/kubelet.go
View File

@ -7,14 +7,11 @@ SPDX-License-Identifier: AGPL-3.0-only
package kubelet
import (
"crypto/ecdsa"
"crypto/elliptic"
"crypto/rand"
"crypto/x509"
"crypto/x509/pkix"
"encoding/pem"
"net"
"github.com/edgelesssys/constellation/bootstrapper/internal/certificate"
"k8s.io/kubernetes/cmd/kubeadm/app/constants"
)
@ -27,18 +24,6 @@ const (
// GetCertificateRequest returns a certificate request and macthing private key for the kubelet.
func GetCertificateRequest(nodeName string, ips []net.IP) (certificateRequest []byte, privateKey []byte, err error) {
privK, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
if err != nil {
return nil, nil, err
}
keyBytes, err := x509.MarshalECPrivateKey(privK)
if err != nil {
return nil, nil, err
}
kubeletKey := pem.EncodeToMemory(&pem.Block{
Type: "EC PRIVATE KEY",
Bytes: keyBytes,
})
csrTemplate := &x509.CertificateRequest{
Subject: pkix.Name{
Organization: []string{constants.NodesGroup},
@ -46,10 +31,5 @@ func GetCertificateRequest(nodeName string, ips []net.IP) (certificateRequest []
},
IPAddresses: ips,
}
certificateRequest, err = x509.CreateCertificateRequest(rand.Reader, csrTemplate, privK)
if err != nil {
return nil, nil, err
}
return certificateRequest, kubeletKey, nil
return certificate.GetCertificateRequest(csrTemplate)
}

16
bootstrapper/internal/kubernetes/k8sapi/kubeadm_config.go
View File

@ -72,6 +72,7 @@ func (c *CoreOSConfiguration) InitConfiguration(externalCloudProvider bool, k8sV
"audit-log-maxbackup": "10", // CIS benchmark - Default value of Rancher
"audit-log-maxsize": "100", // CIS benchmark - Default value of Rancher
"profiling": "false", // CIS benchmark
"egress-selector-config-file": "/etc/kubernetes/egress-selector-configuration.yaml",
"kubelet-certificate-authority": filepath.Join(
kubeconstants.KubernetesDir,
kubeconstants.DefaultCertificateDir,
@ -101,6 +102,20 @@ func (c *CoreOSConfiguration) InitConfiguration(externalCloudProvider bool, k8sV
ReadOnly: true,
PathType: corev1.HostPathFile,
},
{
Name: "egress-config",
HostPath: "/etc/kubernetes/egress-selector-configuration.yaml",
MountPath: "/etc/kubernetes/egress-selector-configuration.yaml",
ReadOnly: true,
PathType: corev1.HostPathFile,
},
{
Name: "konnectivity-uds",
HostPath: "/etc/kubernetes/konnectivity-server",
MountPath: "/etc/kubernetes/konnectivity-server",
ReadOnly: false,
PathType: corev1.HostPathDirectoryOrCreate,
},
},
},
CertSANs: []string{"127.0.0.1"},
@ -133,6 +148,7 @@ func (c *CoreOSConfiguration) InitConfiguration(externalCloudProvider bool, k8sV
"TLS_RSA_WITH_AES_256_GCM_SHA384",
"TLS_RSA_WITH_AES_128_GCM_SHA256",
}, // CIS benchmark
StaticPodPath: "/etc/kubernetes/manifests",
TypeMeta: metav1.TypeMeta{
APIVersion: kubeletconf.SchemeGroupVersion.String(),
Kind: "KubeletConfiguration",

380
bootstrapper/internal/kubernetes/k8sapi/resources/konnectivity.go Normal file
View File

@ -0,0 +1,380 @@
/*
Copyright (c) Edgeless Systems GmbH
SPDX-License-Identifier: AGPL-3.0-only
*/
package resources
import (
"crypto/x509"
"crypto/x509/pkix"
"github.com/edgelesssys/constellation/bootstrapper/internal/certificate"
"github.com/edgelesssys/constellation/internal/kubernetes"
"github.com/edgelesssys/constellation/internal/versions"
appsv1 "k8s.io/api/apps/v1"
corev1 "k8s.io/api/core/v1"
rbacv1 "k8s.io/api/rbac/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/intstr"
"k8s.io/apiserver/pkg/apis/apiserver"
)
const (
// KonnectivityCertificateFilename is the path to the kubelets certificate.
KonnectivityCertificateFilename = "/etc/kubernetes/konnectivity.crt"
// KonnectivityKeyFilename is the path to the kubelets private key.
KonnectivityKeyFilename = "/etc/kubernetes/konnectivity.key"
)
type konnectivityAgents struct {
DaemonSet appsv1.DaemonSet
ClusterRoleBinding rbacv1.ClusterRoleBinding
ServiceAccount corev1.ServiceAccount
}
type konnectivityServerStaticPod struct {
StaticPod corev1.Pod
}
type egressSelectorConfiguration struct {
EgressSelectorConfiguration apiserver.EgressSelectorConfiguration
}
func NewKonnectivityAgents(konnectivityServerAddress string) *konnectivityAgents {
return &konnectivityAgents{
DaemonSet: appsv1.DaemonSet{
TypeMeta: metav1.TypeMeta{
APIVersion: "apps/v1",
Kind: "DaemonSet",
},
ObjectMeta: metav1.ObjectMeta{
Name: "konnectivity-agent",
Namespace: "kube-system",
Labels: map[string]string{
"k8s-app": "konnectivity-agent",
"addonmanager.kubernetes.io/mode": "Reconcile",
},
},
Spec: appsv1.DaemonSetSpec{
Selector: &metav1.LabelSelector{
MatchLabels: map[string]string{
"k8s-app": "konnectivity-agent",
},
},
Template: corev1.PodTemplateSpec{
ObjectMeta: metav1.ObjectMeta{
Labels: map[string]string{
"k8s-app": "konnectivity-agent",
},
},
Spec: corev1.PodSpec{
PriorityClassName: "system-cluster-critical",
Tolerations: []corev1.Toleration{
{
Key: "node-role.kubernetes.io/master",
Operator: corev1.TolerationOpExists,
Effect: corev1.TaintEffectNoSchedule,
},
{
Key: "node-role.kubernetes.io/control-plane",
Operator: corev1.TolerationOpExists,
Effect: corev1.TaintEffectNoSchedule,
},
{
Key: "CriticalAddonsOnly",
Operator: corev1.TolerationOpExists,
},
{
Key: "node.kubernetes.io/not-ready",
Operator: corev1.TolerationOpExists,
Effect: corev1.TaintEffectNoExecute,
},
},
Containers: []corev1.Container{
{
Name: "konnectivity-agent",
Image: versions.KonnectivityAgentImage,
Command: []string{
"/proxy-agent",
},
Args: []string{
"--logtostderr=true",
"--proxy-server-host=" + konnectivityServerAddress,
"--ca-cert=/var/run/secrets/kubernetes.io/serviceaccount/ca.crt",
"--proxy-server-port=8132",
"--admin-server-port=8133",
"--health-server-port=8134",
"--service-account-token-path=/var/run/secrets/tokens/konnectivity-agent-token",
"--agent-identifiers=host=$(HOST_IP)",
// we will be able to avoid constant polling when either one is done:
// https://github.com/kubernetes-sigs/apiserver-network-proxy/issues/358
// https://github.com/kubernetes-sigs/apiserver-network-proxy/issues/273
"--sync-forever=true",
// Ensure stable connection to the konnectivity server.
"--keepalive-time=60s",
"--sync-interval=1s",
"--sync-interval-cap=3s",
"--probe-interval=1s",
"--v=3",
},
Env: []corev1.EnvVar{
{
Name: "HOST_IP",
ValueFrom: &corev1.EnvVarSource{
FieldRef: &corev1.ObjectFieldSelector{
APIVersion: "v1",
FieldPath: "status.hostIP",
},
},
},
},
VolumeMounts: []corev1.VolumeMount{
{
Name: "konnectivity-agent-token",
MountPath: "/var/run/secrets/tokens",
ReadOnly: true,
},
},
LivenessProbe: &corev1.Probe{
ProbeHandler: corev1.ProbeHandler{
HTTPGet: &corev1.HTTPGetAction{
Path: "/healthz",
Port: intstr.FromInt(8134),
},
},
InitialDelaySeconds: 15,
TimeoutSeconds: 15,
},
},
},
ServiceAccountName: "konnectivity-agent",
Volumes: []corev1.Volume{
{
Name: "konnectivity-agent-token",
VolumeSource: corev1.VolumeSource{
Projected: &corev1.ProjectedVolumeSource{
Sources: []corev1.VolumeProjection{
{
ServiceAccountToken: &corev1.ServiceAccountTokenProjection{
Audience: "system:konnectivity-server",
Path: "konnectivity-agent-token",
},
},
},
},
},
},
},
},
},
},
},
ClusterRoleBinding: rbacv1.ClusterRoleBinding{
TypeMeta: metav1.TypeMeta{
APIVersion: "rbac.authorization.k8s.io/v1",
Kind: "ClusterRoleBinding",
},
ObjectMeta: metav1.ObjectMeta{
Name: "system:konnectivity-server",
Labels: map[string]string{
"kubernetes.io/cluster-service": "true",
"addonmanager.kubernetes.io/mode": "Reconcile",
},
},
RoleRef: rbacv1.RoleRef{
APIGroup: "rbac.authorization.k8s.io",
Kind: "ClusterRole",
Name: "system:auth-delegator",
},
Subjects: []rbacv1.Subject{
{
APIGroup: "rbac.authorization.k8s.io",
Kind: "User",
Name: "system:konnectivity-server",
},
},
},
ServiceAccount: corev1.ServiceAccount{
TypeMeta: metav1.TypeMeta{
APIVersion: "v1",
Kind: "ServiceAccount",
},
ObjectMeta: metav1.ObjectMeta{
Name: "konnectivity-agent",
Namespace: "kube-system",
Labels: map[string]string{
"kubernetes.io/cluster-service": "true",
"addonmanager.kubernetes.io/mode": "Reconcile",
},
},
},
}
}
func NewKonnectivityServerStaticPod() *konnectivityServerStaticPod {
udsHostPathType := corev1.HostPathDirectoryOrCreate
return &konnectivityServerStaticPod{
StaticPod: corev1.Pod{
TypeMeta: metav1.TypeMeta{
APIVersion: "v1",
Kind: "Pod",
},
ObjectMeta: metav1.ObjectMeta{
Name: "konnectivity-server",
Namespace: "kube-system",
},
Spec: corev1.PodSpec{
PriorityClassName: "system-cluster-critical",
HostNetwork: true,
Containers: []corev1.Container{
{
Name: "konnectivity-server-container",
Image: versions.KonnectivityServerImage,
Command: []string{"/proxy-server"},
Args: []string{
"--logtostderr=true",
// This needs to be consistent with the value set in egressSelectorConfiguration.
"--uds-name=/etc/kubernetes/konnectivity-server/konnectivity-server.socket",
// The following two lines assume the Konnectivity server is
// deployed on the same machine as the apiserver, and the certs and
// key of the API Server are at the specified location.
"--cluster-cert=/etc/kubernetes/pki/apiserver.crt",
"--cluster-key=/etc/kubernetes/pki/apiserver.key",
// This needs to be consistent with the value set in egressSelectorConfiguration.
"--mode=grpc",
"--server-port=0",
"--agent-port=8132",
"--admin-port=8133",
"--health-port=8134",
"--v=5",
"--agent-namespace=kube-system",
"--agent-service-account=konnectivity-agent",
"--kubeconfig=/etc/kubernetes/konnectivity-server.conf",
"--authentication-audience=system:konnectivity-server",
"--proxy-strategies=destHost",
},
LivenessProbe: &corev1.Probe{
ProbeHandler: corev1.ProbeHandler{
HTTPGet: &corev1.HTTPGetAction{
Path: "/healthz",
Port: intstr.FromInt(8134),
},
},
InitialDelaySeconds: 30,
TimeoutSeconds: 60,
},
Ports: []corev1.ContainerPort{
{
Name: "agent-port",
ContainerPort: 8132,
HostPort: 8132,
},
{
Name: "admin-port",
ContainerPort: 8133,
HostPort: 8133,
},
{
Name: "health-port",
ContainerPort: 8134,
HostPort: 8134,
},
},
VolumeMounts: []corev1.VolumeMount{
{
Name: "k8s-certs",
MountPath: "/etc/kubernetes/pki",
ReadOnly: true,
},
{
Name: "kubeconfig",
MountPath: "/etc/kubernetes/konnectivity-server.conf",
ReadOnly: true,
},
{
Name: "konnectivity-uds",
MountPath: "/etc/kubernetes/konnectivity-server",
ReadOnly: false,
},
},
},
},
Volumes: []corev1.Volume{
{
Name: "k8s-certs",
VolumeSource: corev1.VolumeSource{
HostPath: &corev1.HostPathVolumeSource{
Path: "/etc/kubernetes/pki",
},
},
},
{
Name: "kubeconfig",
VolumeSource: corev1.VolumeSource{
HostPath: &corev1.HostPathVolumeSource{
Path: "/etc/kubernetes/konnectivity-server.conf",
},
},
},
{
Name: "konnectivity-uds",
VolumeSource: corev1.VolumeSource{
HostPath: &corev1.HostPathVolumeSource{
Path: "/etc/kubernetes/konnectivity-server",
Type: &udsHostPathType,
},
},
},
},
},
},
}
}
func NewEgressSelectorConfiguration() *egressSelectorConfiguration {
return &egressSelectorConfiguration{
EgressSelectorConfiguration: apiserver.EgressSelectorConfiguration{
TypeMeta: metav1.TypeMeta{
APIVersion: "apiserver.k8s.io/v1beta1",
Kind: "EgressSelectorConfiguration",
},
EgressSelections: []apiserver.EgressSelection{
{
Name: "cluster",
Connection: apiserver.Connection{
ProxyProtocol: "GRPC",
Transport: &apiserver.Transport{
UDS: &apiserver.UDSTransport{
UDSName: "/etc/kubernetes/konnectivity-server/konnectivity-server.socket",
},
},
},
},
},
},
}
}
func (v *konnectivityAgents) Marshal() ([]byte, error) {
return kubernetes.MarshalK8SResources(v)
}
func (v *konnectivityServerStaticPod) Marshal() ([]byte, error) {
return kubernetes.MarshalK8SResources(v)
}
func (v *egressSelectorConfiguration) Marshal() ([]byte, error) {
return kubernetes.MarshalK8SResources(v)
}
// GetCertificateRequest returns a certificate request and matching private key for the konnectivity server.
func GetKonnectivityCertificateRequest() (certificateRequest []byte, privateKey []byte, err error) {
csrTemplate := &x509.CertificateRequest{
Subject: pkix.Name{
CommonName: "system:konnectivity-server",
},
}
return certificate.GetCertificateRequest(csrTemplate)
}

28
bootstrapper/internal/kubernetes/k8sapi/resources/konnectivity_test.go Normal file
View File

@ -0,0 +1,28 @@
/*
Copyright (c) Edgeless Systems GmbH
SPDX-License-Identifier: AGPL-3.0-only
*/
package resources
import (
"testing"
"github.com/edgelesssys/constellation/internal/kubernetes"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestKonnectivityMarshalUnmarshal(t *testing.T) {
require := require.New(t)
assert := assert.New(t)
kmsDepl := NewKonnectivityAgents("192.168.2.1")
data, err := kmsDepl.Marshal()
require.NoError(err)
var recreated konnectivityAgents
require.NoError(kubernetes.UnmarshalK8SResources(data, &recreated))
assert.Equal(kmsDepl, &recreated)
}

216
bootstrapper/internal/kubernetes/k8sapi/util.go
View File

@ -28,6 +28,7 @@ import (
"github.com/edgelesssys/constellation/bootstrapper/internal/kubernetes/k8sapi/resources"
"github.com/edgelesssys/constellation/internal/constants"
"github.com/edgelesssys/constellation/internal/kubernetes"
"github.com/edgelesssys/constellation/internal/role"
kubeconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
"github.com/edgelesssys/constellation/internal/crypto"
@ -130,7 +131,7 @@ func (k *KubernetesUtil) InstallComponents(ctx context.Context, version versions
}
func (k *KubernetesUtil) InitCluster(
ctx context.Context, initConfig []byte, nodeName string, ips []net.IP, log *logger.Logger,
ctx context.Context, initConfig []byte, nodeName string, ips []net.IP, controlPlaneEndpoint string, log *logger.Logger,
) error {
// TODO: audit policy should be user input
auditPolicy, err := resources.NewDefaultAuditPolicy().Marshal()
@ -180,6 +181,11 @@ func (k *KubernetesUtil) InitCluster(
return err
}
log.Infof("Preparing node for Konnectivity")
if err := k.prepareControlPlaneForKonnectivity(ctx, controlPlaneEndpoint); err != nil {
return fmt.Errorf("setup konnectivity: %w", err)
}
// initialize the cluster
log.Infof("Initializing the cluster using kubeadm init")
cmd = exec.CommandContext(ctx, kubeadmPath, "init", "-v=5", "--skip-phases=preflight,certs,addon/kube-proxy", "--config", initConfigFile.Name())
@ -195,6 +201,64 @@ func (k *KubernetesUtil) InitCluster(
return nil
}
func (k *KubernetesUtil) prepareControlPlaneForKonnectivity(ctx context.Context, loadBalancerEndpoint string) error {
if !strings.Contains(loadBalancerEndpoint, ":") {
loadBalancerEndpoint = net.JoinHostPort(loadBalancerEndpoint, strconv.Itoa(constants.KubernetesPort))
}
if err := os.MkdirAll("/etc/kubernetes/manifests", os.ModePerm); err != nil {
return fmt.Errorf("creating static pods directory: %w", err)
}
konnectivityServerYaml, err := resources.NewKonnectivityServerStaticPod().Marshal()
if err != nil {
return fmt.Errorf("generating konnectivity server static pod: %w", err)
}
if err := os.WriteFile("/etc/kubernetes/manifests/konnectivity-server.yaml", konnectivityServerYaml, 0o644); err != nil {
return fmt.Errorf("writing konnectivity server pod: %w", err)
}
egressConfigYaml, err := resources.NewEgressSelectorConfiguration().Marshal()
if err != nil {
return fmt.Errorf("generating egress selector configuration: %w", err)
}
if err := os.WriteFile("/etc/kubernetes/egress-selector-configuration.yaml", egressConfigYaml, 0o644); err != nil {
return fmt.Errorf("writing egress selector config: %w", err)
}
if err := k.createSignedKonnectivityCert(); err != nil {
return fmt.Errorf("generating konnectivity server certificate: %w", err)
}
if out, err := exec.CommandContext(ctx, kubectlPath, "config", "set-credentials", "--kubeconfig", "/etc/kubernetes/konnectivity-server.conf", "system:konnectivity-server",
"--client-certificate", "/etc/kubernetes/konnectivity.crt", "--client-key", "/etc/kubernetes/konnectivity.key", "--embed-certs=true").CombinedOutput(); err != nil {
return fmt.Errorf("konnectivity kubeconfig set-credentials: %w, %s", err, string(out))
}
if out, err := exec.CommandContext(ctx, kubectlPath, "--kubeconfig", "/etc/kubernetes/konnectivity-server.conf", "config", "set-cluster", "kubernetes", "--server", "https://"+loadBalancerEndpoint,
"--certificate-authority", "/etc/kubernetes/pki/ca.crt", "--embed-certs=true").CombinedOutput(); err != nil {
return fmt.Errorf("konnectivity kubeconfig set-cluster: %w, %s", err, string(out))
}
if out, err := exec.CommandContext(ctx, kubectlPath, "--kubeconfig", "/etc/kubernetes/konnectivity-server.conf", "config", "set-context", "system:konnectivity-server@kubernetes",
"--cluster", "kubernetes", "--user", "system:konnectivity-server").CombinedOutput(); err != nil {
return fmt.Errorf("konnectivity kubeconfig set-context: %w, %s", err, string(out))
}
if out, err := exec.CommandContext(ctx, kubectlPath, "--kubeconfig", "/etc/kubernetes/konnectivity-server.conf", "config", "use-context", "system:konnectivity-server@kubernetes").CombinedOutput(); err != nil {
return fmt.Errorf("konnectivity kubeconfig use-context: %w, %s", err, string(out))
}
// cleanup
if err := os.Remove("/etc/kubernetes/konnectivity.crt"); err != nil {
return fmt.Errorf("removing konnectivity certificate: %w", err)
}
if err := os.Remove("/etc/kubernetes/konnectivity.key"); err != nil {
return fmt.Errorf("removing konnectivity key: %w", err)
}
return nil
}
func (k *KubernetesUtil) SetupKonnectivity(kubectl Client, konnectivityAgentsDaemonSet kubernetes.Marshaler) error {
return kubectl.Apply(konnectivityAgentsDaemonSet, true)
}
func (k *KubernetesUtil) SetupHelmDeployments(ctx context.Context, kubectl Client, helmDeployments []byte, in SetupPodNetworkInput, log *logger.Logger) error {
var helmDeploy helm.Deployments
if err := json.Unmarshal(helmDeployments, &helmDeploy); err != nil {
@ -450,7 +514,7 @@ func (k *KubernetesUtil) SetupNodeOperator(ctx context.Context, kubectl Client,
}
// JoinCluster joins existing Kubernetes cluster using kubeadm join.
func (k *KubernetesUtil) JoinCluster(ctx context.Context, joinConfig []byte, log *logger.Logger) error {
func (k *KubernetesUtil) JoinCluster(ctx context.Context, joinConfig []byte, peerRole role.Role, controlPlaneEndpoint string, log *logger.Logger) error {
// TODO: audit policy should be user input
auditPolicy, err := resources.NewDefaultAuditPolicy().Marshal()
if err != nil {
@ -469,6 +533,13 @@ func (k *KubernetesUtil) JoinCluster(ctx context.Context, joinConfig []byte, log
return fmt.Errorf("writing kubeadm init yaml config %v: %w", joinConfigFile.Name(), err)
}
if peerRole == role.ControlPlane {
log.Infof("Prep Init Kubernetes cluster")
if err := k.prepareControlPlaneForKonnectivity(ctx, controlPlaneEndpoint); err != nil {
return fmt.Errorf("setup konnectivity: %w", err)
}
}
// run `kubeadm join` to join a worker node to an existing Kubernetes cluster
cmd := exec.CommandContext(ctx, kubeadmPath, "join", "-v=5", "--config", joinConfigFile.Name())
out, err := cmd.CombinedOutput()
@ -497,6 +568,7 @@ func (k *KubernetesUtil) StartKubelet() error {
// createSignedKubeletCert manually creates a Kubernetes CA signed kubelet certificate for the bootstrapper node.
// This is necessary because this node does not request a certificate from the join service.
func (k *KubernetesUtil) createSignedKubeletCert(nodeName string, ips []net.IP) error {
// Create CSR
certRequestRaw, kubeletKey, err := kubelet.GetCertificateRequest(nodeName, ips)
if err != nil {
return err
@ -505,48 +577,12 @@ func (k *KubernetesUtil) createSignedKubeletCert(nodeName string, ips []net.IP)
return err
}
parentCertRaw, err := k.file.Read(filepath.Join(
kubeconstants.KubernetesDir,
kubeconstants.DefaultCertificateDir,
kubeconstants.CACertName,
))
if err != nil {
return err
}
parentCert, err := crypto.PemToX509Cert(parentCertRaw)
if err != nil {
return err
}
parentKeyRaw, err := k.file.Read(filepath.Join(
kubeconstants.KubernetesDir,
kubeconstants.DefaultCertificateDir,
kubeconstants.CAKeyName,
))
if err != nil {
return err
}
parentKeyPEM, _ := pem.Decode(parentKeyRaw)
var parentKey any
switch parentKeyPEM.Type {
case "EC PRIVATE KEY":
parentKey, err = x509.ParseECPrivateKey(parentKeyPEM.Bytes)
case "RSA PRIVATE KEY":
parentKey, err = x509.ParsePKCS1PrivateKey(parentKeyPEM.Bytes)
case "PRIVATE KEY":
parentKey, err = x509.ParsePKCS8PrivateKey(parentKeyPEM.Bytes)
default:
err = fmt.Errorf("unsupported key type %q", parentKeyPEM.Type)
}
if err != nil {
return err
}
certRequest, err := x509.ParseCertificateRequest(certRequestRaw)
if err != nil {
return err
}
// Prepare certificate signing
serialNumber, err := crypto.GenerateCertificateSerialNumber()
if err != nil {
return err
@ -570,10 +606,18 @@ func (k *KubernetesUtil) createSignedKubeletCert(nodeName string, ips []net.IP)
IPAddresses: certRequest.IPAddresses,
}
parentCert, parentKey, err := k.getKubernetesCACertAndKey()
if err != nil {
return err
}
// Sign the certificate
certRaw, err := x509.CreateCertificate(rand.Reader, certTmpl, parentCert, certRequest.PublicKey, parentKey)
if err != nil {
return err
}
// Write the certificate
kubeletCert := pem.EncodeToMemory(&pem.Block{
Type: "CERTIFICATE",
Bytes: certRaw,
@ -581,3 +625,97 @@ func (k *KubernetesUtil) createSignedKubeletCert(nodeName string, ips []net.IP)
return k.file.Write(kubelet.CertificateFilename, kubeletCert, file.OptMkdirAll)
}
// createSignedKonnectivityCert manually creates a Kubernetes CA signed certificate for the Konnectivity server.
func (k *KubernetesUtil) createSignedKonnectivityCert() error {
// Create CSR
certRequestRaw, keyPem, err := resources.GetKonnectivityCertificateRequest()
if err != nil {
return err
}
if err := k.file.Write(resources.KonnectivityKeyFilename, keyPem, file.OptMkdirAll); err != nil {
return err
}
certRequest, err := x509.ParseCertificateRequest(certRequestRaw)
if err != nil {
return err
}
// Prepare certificate signing
serialNumber, err := crypto.GenerateCertificateSerialNumber()
if err != nil {
return err
}
now := time.Now()
// Create the kubelet certificate
// For a reference on the certificate fields, see: https://kubernetes.io/docs/setup/best-practices/certificates/
certTmpl := &x509.Certificate{
SerialNumber: serialNumber,
NotBefore: now.Add(-2 * time.Hour),
NotAfter: now.Add(24 * 365 * time.Hour),
Subject: certRequest.Subject,
}
parentCert, parentKey, err := k.getKubernetesCACertAndKey()
if err != nil {
return err
}
// Sign the certificate
certRaw, err := x509.CreateCertificate(rand.Reader, certTmpl, parentCert, certRequest.PublicKey, parentKey)
if err != nil {
return err
}
// Write the certificate
konnectivityCert := pem.EncodeToMemory(&pem.Block{
Type: "CERTIFICATE",
Bytes: certRaw,
})
return k.file.Write(resources.KonnectivityCertificateFilename, konnectivityCert, file.OptMkdirAll)
}
// getKubernetesCACertAndKey returns the Kubernetes CA certificate and key.
// The key of type `any` can be consumed by `x509.CreateCertificate()`.
func (k *KubernetesUtil) getKubernetesCACertAndKey() (*x509.Certificate, any, error) {
parentCertRaw, err := k.file.Read(filepath.Join(
kubeconstants.KubernetesDir,
kubeconstants.DefaultCertificateDir,
kubeconstants.CACertName,
))
if err != nil {
return nil, nil, err
}
parentCert, err := crypto.PemToX509Cert(parentCertRaw)
if err != nil {
return nil, nil, err
}
parentKeyRaw, err := k.file.Read(filepath.Join(
kubeconstants.KubernetesDir,
kubeconstants.DefaultCertificateDir,
kubeconstants.CAKeyName,
))
if err != nil {
return nil, nil, err
}
parentKeyPEM, _ := pem.Decode(parentKeyRaw)
var parentKey any
switch parentKeyPEM.Type {
case "EC PRIVATE KEY":
parentKey, err = x509.ParseECPrivateKey(parentKeyPEM.Bytes)
case "RSA PRIVATE KEY":
parentKey, err = x509.ParsePKCS1PrivateKey(parentKeyPEM.Bytes)
case "PRIVATE KEY":
parentKey, err = x509.ParsePKCS8PrivateKey(parentKeyPEM.Bytes)
default:
err = fmt.Errorf("unsupported key type %q", parentKeyPEM.Type)
}
if err != nil {
return nil, nil, err
}
return parentCert, parentKey, nil
}

6
bootstrapper/internal/kubernetes/k8sutil.go
View File

@ -13,19 +13,21 @@ import (
"github.com/edgelesssys/constellation/bootstrapper/internal/kubernetes/k8sapi"
"github.com/edgelesssys/constellation/internal/kubernetes"
"github.com/edgelesssys/constellation/internal/logger"
"github.com/edgelesssys/constellation/internal/role"
"github.com/edgelesssys/constellation/internal/versions"
)
type clusterUtil interface {
InstallComponents(ctx context.Context, version versions.ValidK8sVersion) error
InitCluster(ctx context.Context, initConfig []byte, nodeName string, ips []net.IP, log *logger.Logger) error
JoinCluster(ctx context.Context, joinConfig []byte, log *logger.Logger) error
InitCluster(ctx context.Context, initConfig []byte, nodeName string, ips []net.IP, controlPlaneEndpoint string, log *logger.Logger) error
JoinCluster(ctx context.Context, joinConfig []byte, peerRole role.Role, controlPlaneEndpoint string, log *logger.Logger) error
SetupHelmDeployments(ctx context.Context, client k8sapi.Client, helmDeployments []byte, in k8sapi.SetupPodNetworkInput, log *logger.Logger) error
SetupAccessManager(kubectl k8sapi.Client, sshUsers kubernetes.Marshaler) error
SetupAutoscaling(kubectl k8sapi.Client, clusterAutoscalerConfiguration kubernetes.Marshaler, secrets kubernetes.Marshaler) error
SetupJoinService(kubectl k8sapi.Client, joinServiceConfiguration kubernetes.Marshaler) error
SetupCloudControllerManager(kubectl k8sapi.Client, cloudControllerManagerConfiguration kubernetes.Marshaler, configMaps kubernetes.Marshaler, secrets kubernetes.Marshaler) error
SetupCloudNodeManager(kubectl k8sapi.Client, cloudNodeManagerConfiguration kubernetes.Marshaler) error
SetupKonnectivity(kubectl k8sapi.Client, konnectivityAgentsDaemonSet kubernetes.Marshaler) error
SetupKMS(kubectl k8sapi.Client, kmsConfiguration kubernetes.Marshaler) error
SetupVerificationService(kubectl k8sapi.Client, verificationServiceConfiguration kubernetes.Marshaler) error
SetupGCPGuestAgent(kubectl k8sapi.Client, gcpGuestAgentConfiguration kubernetes.Marshaler) error

32
bootstrapper/internal/kubernetes/kubernetes.go
View File

@ -19,7 +19,6 @@ import (
"github.com/edgelesssys/constellation/bootstrapper/internal/kubernetes/k8sapi/resources"
"github.com/edgelesssys/constellation/internal/cloud/metadata"
"github.com/edgelesssys/constellation/internal/constants"
"github.com/edgelesssys/constellation/internal/iproute"
"github.com/edgelesssys/constellation/internal/logger"
"github.com/edgelesssys/constellation/internal/role"
"github.com/edgelesssys/constellation/internal/versions"
@ -156,7 +155,7 @@ func (k *KubeWrapper) InitCluster(
return nil, fmt.Errorf("encoding kubeadm init configuration as YAML: %w", err)
}
log.Infof("Initializing Kubernetes cluster")
if err := k.clusterUtil.InitCluster(ctx, initConfigYAML, nodeName, validIPs, log); err != nil {
if err := k.clusterUtil.InitCluster(ctx, initConfigYAML, nodeName, validIPs, controlPlaneEndpoint, log); err != nil {
return nil, fmt.Errorf("kubeadm init: %w", err)
}
kubeConfig, err := k.GetKubeconfig()
@ -178,6 +177,19 @@ func (k *KubeWrapper) InitCluster(
return nil, fmt.Errorf("setting up pod network: %w", err)
}
var controlPlaneIP string
if strings.Contains(controlPlaneEndpoint, ":") {
controlPlaneIP, _, err = net.SplitHostPort(controlPlaneEndpoint)
if err != nil {
return nil, fmt.Errorf("parsing control plane endpoint: %w", err)
}
} else {
controlPlaneIP = controlPlaneEndpoint
}
if err = k.clusterUtil.SetupKonnectivity(k.client, resources.NewKonnectivityAgents(controlPlaneIP)); err != nil {
return nil, fmt.Errorf("setting up konnectivity: %w", err)
}
kms := resources.NewKMSDeployment(k.cloudProvider, kmsConfig)
if err = k.clusterUtil.SetupKMS(k.client, kms); err != nil {
return nil, fmt.Errorf("setting up kms: %w", err)
@ -277,19 +289,7 @@ func (k *KubeWrapper) JoinCluster(ctx context.Context, args *kubeadm.BootstrapTo
zap.String("nodeIP", nodeInternalIP),
).Infof("Setting information for node")
// Step 2: Remove load balancer from local routing table on GCP.
if k.cloudProvider == "gcp" {
ip, _, err := net.SplitHostPort(loadbalancerEndpoint)
if err != nil {
return fmt.Errorf("parsing load balancer IP: %w", err)
}
if err := iproute.RemoveFromLocalRoutingTable(ctx, ip); err != nil {
return fmt.Errorf("removing load balancer IP from routing table: %w", err)
}
log.Infof("Removed load balancer IP from routing table")
}
// Step 3: configure kubeadm join config
// Step 2: configure kubeadm join config
joinConfig := k.configProvider.JoinConfiguration(k.cloudControllerManager.Supported())
joinConfig.SetAPIServerEndpoint(args.APIServerEndpoint)
joinConfig.SetToken(args.Token)
@ -305,7 +305,7 @@ func (k *KubeWrapper) JoinCluster(ctx context.Context, args *kubeadm.BootstrapTo
return fmt.Errorf("encoding kubeadm join configuration as YAML: %w", err)
}
log.With(zap.String("apiServerEndpoint", args.APIServerEndpoint)).Infof("Joining Kubernetes cluster")
if err := k.clusterUtil.JoinCluster(ctx, joinConfigYAML, log); err != nil {
if err := k.clusterUtil.JoinCluster(ctx, joinConfigYAML, peerRole, loadbalancerEndpoint, log); err != nil {
return fmt.Errorf("joining cluster: %v; %w ", string(joinConfigYAML), err)
}

21
bootstrapper/internal/kubernetes/kubernetes_test.go
View File

@ -263,6 +263,18 @@ func TestInitCluster(t *testing.T) {
wantErr: true,
k8sVersion: versions.Default,
},
"kubeadm init fails when setting up konnectivity": {
clusterUtil: stubClusterUtil{setupKonnectivityError: someErr},
kubeconfigReader: &stubKubeconfigReader{
Kubeconfig: []byte("someKubeconfig"),
},
providerMetadata: &stubProviderMetadata{SupportedResp: false},
CloudControllerManager: &stubCloudControllerManager{},
CloudNodeManager: &stubCloudNodeManager{SupportedResp: false},
ClusterAutoscaler: &stubClusterAutoscaler{},
wantErr: true,
k8sVersion: versions.Default,
},
"kubeadm init fails when setting up verification service": {
clusterUtil: stubClusterUtil{setupVerificationServiceErr: someErr},
kubeconfigReader: &stubKubeconfigReader{
@ -522,6 +534,7 @@ type stubClusterUtil struct {
setupJoinServiceError error
setupCloudControllerManagerError error
setupCloudNodeManagerError error
setupKonnectivityError error
setupKMSError error
setupAccessManagerError error
setupVerificationServiceErr error
@ -536,11 +549,15 @@ type stubClusterUtil struct {
joinConfigs [][]byte
}
func (s *stubClusterUtil) SetupKonnectivity(kubectl k8sapi.Client, konnectivityAgentsDaemonSet kubernetes.Marshaler) error {
return s.setupKonnectivityError
}
func (s *stubClusterUtil) InstallComponents(ctx context.Context, version versions.ValidK8sVersion) error {
return s.installComponentsErr
}
func (s *stubClusterUtil) InitCluster(ctx context.Context, initConfig []byte, nodeName string, ips []net.IP, log *logger.Logger) error {
func (s *stubClusterUtil) InitCluster(ctx context.Context, initConfig []byte, nodeName string, ips []net.IP, controlPlaneEndpoint string, log *logger.Logger) error {
s.initConfigs = append(s.initConfigs, initConfig)
return s.initClusterErr
}
@ -593,7 +610,7 @@ func (s *stubClusterUtil) SetupNodeOperator(ctx context.Context, kubectl k8sapi.
return s.setupNodeOperatorErr
}
func (s *stubClusterUtil) JoinCluster(ctx context.Context, joinConfig []byte, log *logger.Logger) error {
func (s *stubClusterUtil) JoinCluster(ctx context.Context, joinConfig []byte, peerRole role.Role, controlPlaneEndpoint string, log *logger.Logger) error {
s.joinConfigs = append(s.joinConfigs, joinConfig)
return s.joinClusterErr
}

37
cli/internal/azure/loadbalancer.go
View File

@ -30,6 +30,7 @@ const (
verifyHealthProbeName = "verifyHealthProbe"
coordHealthProbeName = "coordHealthProbe"
debugdHealthProbeName = "debugdHealthProbe"
konnectivityHealthProbeName = "konnectivityHealthProbe"
)
// Azure returns a Azure representation of LoadBalancer.
@ -86,6 +87,13 @@ func (l LoadBalancer) Azure() armnetwork.LoadBalancer {
Port: to.Ptr[int32](constants.DebugdPort),
},
},
{
Name: to.Ptr(konnectivityHealthProbeName),
Properties: &armnetwork.ProbePropertiesFormat{
Protocol: to.Ptr(armnetwork.ProbeProtocolTCP),
Port: to.Ptr[int32](constants.KonnectivityPort),
},
},
},
LoadBalancingRules: []*armnetwork.LoadBalancingRule{
{
@ -175,6 +183,35 @@ func (l LoadBalancer) Azure() armnetwork.LoadBalancer {
},
},
},
{
Name: to.Ptr("konnectivityLoadBalancerRule"),
Properties: &armnetwork.LoadBalancingRulePropertiesFormat{
FrontendIPConfiguration: &armnetwork.SubResource{
ID: to.Ptr("/subscriptions/" + l.Subscription +
"/resourceGroups/" + l.ResourceGroup +
"/providers/Microsoft.Network/loadBalancers/" + l.Name +
"/frontendIPConfigurations/" + frontEndIPConfigName),
},
FrontendPort: to.Ptr[int32](constants.KonnectivityPort),
BackendPort: to.Ptr[int32](constants.KonnectivityPort),
Protocol: to.Ptr(armnetwork.TransportProtocolTCP),
Probe: &armnetwork.SubResource{
ID: to.Ptr("/subscriptions/" + l.Subscription +
"/resourceGroups/" + l.ResourceGroup +
"/providers/Microsoft.Network/loadBalancers/" + l.Name +
"/probes/" + konnectivityHealthProbeName),
},
DisableOutboundSnat: to.Ptr(true),
BackendAddressPools: []*armnetwork.SubResource{
{
ID: to.Ptr("/subscriptions/" + l.Subscription +
"/resourceGroups/" + l.ResourceGroup +
"/providers/Microsoft.Network/loadBalancers/" + l.Name +
"/backendAddressPools/" + backEndAddressPoolControlPlaneName),
},
},
},
},
},
OutboundRules: []*armnetwork.OutboundRule{
{

2
cli/internal/cloudcmd/create_test.go
View File

@ -40,7 +40,7 @@ func TestCreator(t *testing.T) {
GCPSubnetwork: "subnetwork",
GCPLoadbalancers: []string{"kube-lb", "boot-lb", "verify-lb"},
GCPFirewalls: []string{
"bootstrapper", "ssh", "nodeport", "kubernetes",
"bootstrapper", "ssh", "nodeport", "kubernetes", "konnectivity",
"allow-cluster-internal-tcp", "allow-cluster-internal-udp", "allow-cluster-internal-icmp",
"allow-node-internal-tcp", "allow-node-internal-udp", "allow-node-internal-icmp",
},

30
cli/internal/gcp/client/api.go
View File

@ -49,29 +49,37 @@ type firewallsAPI interface {
type forwardingRulesAPI interface {
Close() error
Delete(ctx context.Context, req *computepb.DeleteForwardingRuleRequest,
Delete(ctx context.Context, req *computepb.DeleteGlobalForwardingRuleRequest,
opts ...gax.CallOption) (Operation, error)
Insert(ctx context.Context, req *computepb.InsertForwardingRuleRequest,
Insert(ctx context.Context, req *computepb.InsertGlobalForwardingRuleRequest,
opts ...gax.CallOption) (Operation, error)
Get(ctx context.Context, req *computepb.GetForwardingRuleRequest,
Get(ctx context.Context, req *computepb.GetGlobalForwardingRuleRequest,
opts ...gax.CallOption) (*computepb.ForwardingRule, error)
SetLabels(ctx context.Context, req *computepb.SetLabelsForwardingRuleRequest,
SetLabels(ctx context.Context, req *computepb.SetLabelsGlobalForwardingRuleRequest,
opts ...gax.CallOption) (Operation, error)
}
type backendServicesAPI interface {
Close() error
Delete(ctx context.Context, req *computepb.DeleteRegionBackendServiceRequest,
Delete(ctx context.Context, req *computepb.DeleteBackendServiceRequest,
opts ...gax.CallOption) (Operation, error)
Insert(ctx context.Context, req *computepb.InsertRegionBackendServiceRequest,
Insert(ctx context.Context, req *computepb.InsertBackendServiceRequest,
opts ...gax.CallOption) (Operation, error)
}
type healthChecksAPI interface {
Close() error
Delete(ctx context.Context, req *computepb.DeleteRegionHealthCheckRequest,
Delete(ctx context.Context, req *computepb.DeleteHealthCheckRequest,
opts ...gax.CallOption) (Operation, error)
Insert(ctx context.Context, req *computepb.InsertRegionHealthCheckRequest,
Insert(ctx context.Context, req *computepb.InsertHealthCheckRequest,
opts ...gax.CallOption) (Operation, error)
}
type targetTCPProxiesAPI interface {
Close() error
Delete(ctx context.Context, req *computepb.DeleteTargetTcpProxyRequest,
opts ...gax.CallOption) (Operation, error)
Insert(ctx context.Context, req *computepb.InsertTargetTcpProxyRequest,
opts ...gax.CallOption) (Operation, error)
}
@ -129,11 +137,11 @@ type projectsAPI interface {
type addressesAPI interface {
Close() error
Insert(ctx context.Context, req *computepb.InsertAddressRequest,
Insert(ctx context.Context, req *computepb.InsertGlobalAddressRequest,
opts ...gax.CallOption) (Operation, error)
Get(ctx context.Context, req *computepb.GetAddressRequest,
Get(ctx context.Context, req *computepb.GetGlobalAddressRequest,
opts ...gax.CallOption) (*computepb.Address, error)
Delete(ctx context.Context, req *computepb.DeleteAddressRequest,
Delete(ctx context.Context, req *computepb.DeleteGlobalAddressRequest,
opts ...gax.CallOption) (Operation, error)
}

66
cli/internal/gcp/client/api_test.go
View File

@ -231,7 +231,7 @@ func (a stubBackendServicesAPI) Close() error {
return nil
}
func (a stubBackendServicesAPI) Insert(ctx context.Context, req *computepb.InsertRegionBackendServiceRequest,
func (a stubBackendServicesAPI) Insert(ctx context.Context, req *computepb.InsertBackendServiceRequest,
opts ...gax.CallOption,
) (Operation, error) {
if a.insertErr != nil {
@ -240,12 +240,46 @@ func (a stubBackendServicesAPI) Insert(ctx context.Context, req *computepb.Inser
return &stubOperation{
&computepb.Operation{
Name: proto.String("name"),
Region: proto.String("region"),
},
}, nil
}
func (a stubBackendServicesAPI) Delete(ctx context.Context, req *computepb.DeleteRegionBackendServiceRequest,
func (a stubBackendServicesAPI) Delete(ctx context.Context, req *computepb.DeleteBackendServiceRequest,
opts ...gax.CallOption,
) (Operation, error) {
if a.deleteErr != nil {
return nil, a.deleteErr
}
return &stubOperation{
&computepb.Operation{
Name: proto.String("name"),
},
}, nil
}
type stubTargetTCPProxiesAPI struct {
insertErr error
deleteErr error
}
func (a stubTargetTCPProxiesAPI) Close() error {
return nil
}
func (a stubTargetTCPProxiesAPI) Insert(ctx context.Context, req *computepb.InsertTargetTcpProxyRequest,
opts ...gax.CallOption,
) (Operation, error) {
if a.insertErr != nil {
return nil, a.insertErr
}
return &stubOperation{
&computepb.Operation{
Name: proto.String("name"),
},
}, nil
}
func (a stubTargetTCPProxiesAPI) Delete(ctx context.Context, req *computepb.DeleteTargetTcpProxyRequest,
opts ...gax.CallOption,
) (Operation, error) {
if a.deleteErr != nil {
@ -254,7 +288,6 @@ func (a stubBackendServicesAPI) Delete(ctx context.Context, req *computepb.Delet
return &stubOperation{
&computepb.Operation{
Name: proto.String("name"),
Region: proto.String("region"),
},
}, nil
}
@ -271,7 +304,7 @@ func (a stubForwardingRulesAPI) Close() error {
return nil
}
func (a stubForwardingRulesAPI) Insert(ctx context.Context, req *computepb.InsertForwardingRuleRequest,
func (a stubForwardingRulesAPI) Insert(ctx context.Context, req *computepb.InsertGlobalForwardingRuleRequest,
opts ...gax.CallOption,
) (Operation, error) {
if a.insertErr != nil {
@ -280,12 +313,11 @@ func (a stubForwardingRulesAPI) Insert(ctx context.Context, req *computepb.Inser
return &stubOperation{
&computepb.Operation{
Name: proto.String("name"),
Region: proto.String("region"),
},
}, nil
}
func (a stubForwardingRulesAPI) Delete(ctx context.Context, req *computepb.DeleteForwardingRuleRequest,
func (a stubForwardingRulesAPI) Delete(ctx context.Context, req *computepb.DeleteGlobalForwardingRuleRequest,
opts ...gax.CallOption,
) (Operation, error) {
if a.deleteErr != nil {
@ -294,12 +326,11 @@ func (a stubForwardingRulesAPI) Delete(ctx context.Context, req *computepb.Delet
return &stubOperation{
&computepb.Operation{
Name: proto.String("name"),
Region: proto.String("region"),
},
}, nil
}
func (a stubForwardingRulesAPI) Get(ctx context.Context, req *computepb.GetForwardingRuleRequest,
func (a stubForwardingRulesAPI) Get(ctx context.Context, req *computepb.GetGlobalForwardingRuleRequest,
opts ...gax.CallOption,
) (*computepb.ForwardingRule, error) {
if a.getErr != nil {
@ -308,7 +339,7 @@ func (a stubForwardingRulesAPI) Get(ctx context.Context, req *computepb.GetForwa
return a.forwardingRule, nil
}
func (a stubForwardingRulesAPI) SetLabels(ctx context.Context, req *computepb.SetLabelsForwardingRuleRequest,
func (a stubForwardingRulesAPI) SetLabels(ctx context.Context, req *computepb.SetLabelsGlobalForwardingRuleRequest,
opts ...gax.CallOption,
) (Operation, error) {
if a.deleteErr != nil {
@ -317,7 +348,6 @@ func (a stubForwardingRulesAPI) SetLabels(ctx context.Context, req *computepb.Se
return &stubOperation{
&computepb.Operation{
Name: proto.String("name"),
Region: proto.String("region"),
},
}, nil
}
@ -331,7 +361,7 @@ func (a stubHealthChecksAPI) Close() error {
return nil
}
func (a stubHealthChecksAPI) Insert(ctx context.Context, req *computepb.InsertRegionHealthCheckRequest,
func (a stubHealthChecksAPI) Insert(ctx context.Context, req *computepb.InsertHealthCheckRequest,
opts ...gax.CallOption,
) (Operation, error) {
if a.insertErr != nil {
@ -340,12 +370,11 @@ func (a stubHealthChecksAPI) Insert(ctx context.Context, req *computepb.InsertRe
return &stubOperation{
&computepb.Operation{
Name: proto.String("name"),
Region: proto.String("region"),
},
}, nil
}
func (a stubHealthChecksAPI) Delete(ctx context.Context, req *computepb.DeleteRegionHealthCheckRequest,
func (a stubHealthChecksAPI) Delete(ctx context.Context, req *computepb.DeleteHealthCheckRequest,
opts ...gax.CallOption,
) (Operation, error) {
if a.deleteErr != nil {
@ -354,7 +383,6 @@ func (a stubHealthChecksAPI) Delete(ctx context.Context, req *computepb.DeleteRe
return &stubOperation{
&computepb.Operation{
Name: proto.String("name"),
Region: proto.String("region"),
},
}, nil
}
@ -511,7 +539,7 @@ type stubAddressesAPI struct {
deleteErr error
}
func (a stubAddressesAPI) Insert(context.Context, *computepb.InsertAddressRequest,
func (a stubAddressesAPI) Insert(context.Context, *computepb.InsertGlobalAddressRequest,
...gax.CallOption,
) (Operation, error) {
if a.insertErr != nil {
@ -520,18 +548,17 @@ func (a stubAddressesAPI) Insert(context.Context, *computepb.InsertAddressReques
return &stubOperation{
&computepb.Operation{
Name: proto.String("name"),
Region: proto.String("region"),
},
}, nil
}
func (a stubAddressesAPI) Get(ctx context.Context, req *computepb.GetAddressRequest,
func (a stubAddressesAPI) Get(ctx context.Context, req *computepb.GetGlobalAddressRequest,
opts ...gax.CallOption,
) (*computepb.Address, error) {
return &computepb.Address{Address: a.getAddr}, a.getErr
}
func (a stubAddressesAPI) Delete(context.Context, *computepb.DeleteAddressRequest,
func (a stubAddressesAPI) Delete(context.Context, *computepb.DeleteGlobalAddressRequest,
...gax.CallOption,
) (Operation, error) {
if a.deleteErr != nil {
@ -540,7 +567,6 @@ func (a stubAddressesAPI) Delete(context.Context, *computepb.DeleteAddressReques
return &stubOperation{
&computepb.Operation{
Name: proto.String("name"),
Region: proto.String("region"),
},
}, nil
}

17
cli/internal/gcp/client/client.go
View File

@ -36,6 +36,7 @@ type Client struct {
forwardingRulesAPI
backendServicesAPI
healthChecksAPI
targetTCPProxiesAPI
instanceTemplateAPI
instanceGroupManagersAPI
iamAPI
@ -109,25 +110,31 @@ func NewFromDefault(ctx context.Context) (*Client, error) {
return nil, err
}
closers = append(closers, subnetAPI)
forwardingRulesAPI, err := compute.NewForwardingRulesRESTClient(ctx)
forwardingRulesAPI, err := compute.NewGlobalForwardingRulesRESTClient(ctx)
if err != nil {
_ = closeAll(closers)
return nil, err
}
closers = append(closers, forwardingRulesAPI)
backendServicesAPI, err := compute.NewRegionBackendServicesRESTClient(ctx)
backendServicesAPI, err := compute.NewBackendServicesRESTClient(ctx)
if err != nil {
_ = closeAll(closers)
return nil, err
}
closers = append(closers, backendServicesAPI)
targetTCPProxiesAPI, err := compute.NewTargetTcpProxiesRESTClient(ctx)
if err != nil {
_ = closeAll(closers)
return nil, err
}
closers = append(closers, targetTCPProxiesAPI)
targetPoolsAPI, err := compute.NewTargetPoolsRESTClient(ctx)
if err != nil {
_ = closeAll(closers)
return nil, err
}
closers = append(closers, targetPoolsAPI)
healthChecksAPI, err := compute.NewRegionHealthChecksRESTClient(ctx)
healthChecksAPI, err := compute.NewHealthChecksRESTClient(ctx)
if err != nil {
_ = closeAll(closers)
return nil, err
@ -157,7 +164,7 @@ func NewFromDefault(ctx context.Context) (*Client, error) {
return nil, err
}
closers = append(closers, projectsAPI)
addressesAPI, err := compute.NewAddressesRESTClient(ctx)
addressesAPI, err := compute.NewGlobalAddressesRESTClient(ctx)
if err != nil {
_ = closeAll(closers)
return nil, err
@ -173,6 +180,7 @@ func NewFromDefault(ctx context.Context) (*Client, error) {
firewallsAPI: &firewallsClient{fwAPI},
forwardingRulesAPI: &forwardingRulesClient{forwardingRulesAPI},
backendServicesAPI: &backendServicesClient{backendServicesAPI},
targetTCPProxiesAPI: &targetTCPProxiesClient{targetTCPProxiesAPI},
healthChecksAPI: &healthChecksClient{healthChecksAPI},
instanceTemplateAPI: &instanceTemplateClient{templAPI},
instanceGroupManagersAPI: &instanceGroupManagersClient{groupAPI},
@ -282,6 +290,7 @@ func (c *Client) SetState(stat state.ConstellationState) {
hasForwardingRules: true,
hasBackendService: true,
hasHealthCheck: true,
hasTargetTCPProxy: true,
}
c.loadbalancers = append(c.loadbalancers, lb)
}

76
cli/internal/gcp/client/gcpwrappers.go
View File

@ -53,75 +53,95 @@ func (c *firewallsClient) Insert(ctx context.Context, req *computepb.InsertFirew
}
type forwardingRulesClient struct {
*compute.ForwardingRulesClient
*compute.GlobalForwardingRulesClient
}
func (c *forwardingRulesClient) Close() error {
return c.ForwardingRulesClient.Close()
return c.GlobalForwardingRulesClient.Close()
}
func (c *forwardingRulesClient) Delete(ctx context.Context, req *computepb.DeleteForwardingRuleRequest,
func (c *forwardingRulesClient) Delete(ctx context.Context, req *computepb.DeleteGlobalForwardingRuleRequest,
opts ...gax.CallOption,
) (Operation, error) {
return c.ForwardingRulesClient.Delete(ctx, req)
return c.GlobalForwardingRulesClient.Delete(ctx, req)
}
func (c *forwardingRulesClient) Insert(ctx context.Context, req *computepb.InsertForwardingRuleRequest,
func (c *forwardingRulesClient) Insert(ctx context.Context, req *computepb.InsertGlobalForwardingRuleRequest,
opts ...gax.CallOption,
) (Operation, error) {
return c.ForwardingRulesClient.Insert(ctx, req)
return c.GlobalForwardingRulesClient.Insert(ctx, req)
}
func (c *forwardingRulesClient) Get(ctx context.Context, req *computepb.GetForwardingRuleRequest,
func (c *forwardingRulesClient) Get(ctx context.Context, req *computepb.GetGlobalForwardingRuleRequest,
opts ...gax.CallOption,
) (*computepb.ForwardingRule, error) {
return c.ForwardingRulesClient.Get(ctx, req)
return c.GlobalForwardingRulesClient.Get(ctx, req)
}
func (c *forwardingRulesClient) SetLabels(ctx context.Context, req *computepb.SetLabelsForwardingRuleRequest,
func (c *forwardingRulesClient) SetLabels(ctx context.Context, req *computepb.SetLabelsGlobalForwardingRuleRequest,
opts ...gax.CallOption,
) (Operation, error) {
return c.ForwardingRulesClient.SetLabels(ctx, req)
return c.GlobalForwardingRulesClient.SetLabels(ctx, req)
}
type backendServicesClient struct {
*compute.RegionBackendServicesClient
*compute.BackendServicesClient
}
func (c *backendServicesClient) Close() error {
return c.RegionBackendServicesClient.Close()
return c.BackendServicesClient.Close()
}
func (c *backendServicesClient) Insert(ctx context.Context, req *computepb.InsertRegionBackendServiceRequest,
func (c *backendServicesClient) Insert(ctx context.Context, req *computepb.InsertBackendServiceRequest,
opts ...gax.CallOption,
) (Operation, error) {
return c.RegionBackendServicesClient.Insert(ctx, req)
return c.BackendServicesClient.Insert(ctx, req)
}
func (c *backendServicesClient) Delete(ctx context.Context, req *computepb.DeleteRegionBackendServiceRequest,
func (c *backendServicesClient) Delete(ctx context.Context, req *computepb.DeleteBackendServiceRequest,
opts ...gax.CallOption,
) (Operation, error) {
return c.RegionBackendServicesClient.Delete(ctx, req)
return c.BackendServicesClient.Delete(ctx, req)
}
type targetTCPProxiesClient struct {
*compute.TargetTcpProxiesClient
}
func (c *targetTCPProxiesClient) Close() error {
return c.TargetTcpProxiesClient.Close()
}
func (c *targetTCPProxiesClient) Delete(ctx context.Context, req *computepb.DeleteTargetTcpProxyRequest,
opts ...gax.CallOption,
) (Operation, error) {
return c.TargetTcpProxiesClient.Delete(ctx, req)
}
func (c *targetTCPProxiesClient) Insert(ctx context.Context, req *computepb.InsertTargetTcpProxyRequest,
opts ...gax.CallOption,
) (Operation, error) {
return c.TargetTcpProxiesClient.Insert(ctx, req)
}
type healthChecksClient struct {
*compute.RegionHealthChecksClient
*compute.HealthChecksClient
}
func (c *healthChecksClient) Close() error {
return c.RegionHealthChecksClient.Close()
return c.HealthChecksClient.Close()
}
func (c *healthChecksClient) Delete(ctx context.Context, req *computepb.DeleteRegionHealthCheckRequest,
func (c *healthChecksClient) Delete(ctx context.Context, req *computepb.DeleteHealthCheckRequest,
opts ...gax.CallOption,
) (Operation, error) {
return c.RegionHealthChecksClient.Delete(ctx, req)
return c.HealthChecksClient.Delete(ctx, req)
}
func (c *healthChecksClient) Insert(ctx context.Context, req *computepb.InsertRegionHealthCheckRequest,
func (c *healthChecksClient) Insert(ctx context.Context, req *computepb.InsertHealthCheckRequest,
opts ...gax.CallOption,
) (Operation, error) {
return c.RegionHealthChecksClient.Insert(ctx, req)
return c.HealthChecksClient.Insert(ctx, req)
}
type networksClient struct {
@ -257,21 +277,21 @@ func (c *projectsClient) SetIamPolicy(ctx context.Context, req *iampb.SetIamPoli
}
type addressesClient struct {
*compute.AddressesClient
*compute.GlobalAddressesClient
}
func (c *addressesClient) Insert(ctx context.Context, req *computepb.InsertAddressRequest,
func (c *addressesClient) Insert(ctx context.Context, req *computepb.InsertGlobalAddressRequest,
opts ...gax.CallOption,
) (Operation, error) {
return c.AddressesClient.Insert(ctx, req)
return c.GlobalAddressesClient.Insert(ctx, req)
}
func (c *addressesClient) Delete(ctx context.Context, req *computepb.DeleteAddressRequest,
func (c *addressesClient) Delete(ctx context.Context, req *computepb.DeleteGlobalAddressRequest,
opts ...gax.CallOption,
) (Operation, error) {
return c.AddressesClient.Delete(ctx, req)
return c.GlobalAddressesClient.Delete(ctx, req)
}
func (c *addressesClient) Close() error {
return c.AddressesClient.Close()
return c.GlobalAddressesClient.Close()
}

1
cli/internal/gcp/client/instances.go
View File

@ -90,6 +90,7 @@ func (c *Client) CreateInstances(ctx context.Context, input CreateInstancesInput
{Name: proto.String("debugd"), Port: proto.Int32(constants.DebugdPort)},
{Name: proto.String("bootstrapper"), Port: proto.Int32(constants.BootstrapperPort)},
{Name: proto.String("verify"), Port: proto.Int32(constants.VerifyServiceNodePortGRPC)},
{Name: proto.String("konnectivity"), Port: proto.Int32(constants.KonnectivityPort)},
},
Template: c.controlPlaneTemplate,
UID: c.uid,

107
cli/internal/gcp/client/loadbalancer.go
View File

@ -32,6 +32,7 @@ type loadBalancer struct {
hasHealthCheck bool
hasBackendService bool
hasForwardingRules bool
hasTargetTCPProxy bool
}
// CreateLoadBalancers creates all necessary load balancers.
@ -69,6 +70,14 @@ func (c *Client) CreateLoadBalancers(ctx context.Context, isDebugCluster bool) e
healthCheck: computepb.HealthCheck_TCP,
})
c.loadbalancers = append(c.loadbalancers, &loadBalancer{
name: c.buildResourceName("konnectivity"),
ip: c.loadbalancerIPname,
frontendPort: constants.KonnectivityPort,
backendPortName: "konnectivity",
healthCheck: computepb.HealthCheck_TCP,
})
// Only create when the debug cluster flag is set in the Constellation config
if isDebugCluster {
c.loadbalancers = append(c.loadbalancers, &loadBalancer{
@ -107,6 +116,9 @@ func (c *Client) createLoadBalancer(ctx context.Context, lb *loadBalancer) error
if err := c.createBackendService(ctx, lb); err != nil {
return fmt.Errorf("creating backend services: %w", err)
}
if err := c.createTargetTCPProxy(ctx, lb); err != nil {
return fmt.Errorf("creating target TCP proxies: %w", err)
}
if err := c.createForwardingRules(ctx, lb); err != nil {
return fmt.Errorf("creating forwarding rules: %w", err)
}
@ -114,9 +126,8 @@ func (c *Client) createLoadBalancer(ctx context.Context, lb *loadBalancer) error
}
func (c *Client) createHealthCheck(ctx context.Context, lb *loadBalancer) error {
req := &computepb.InsertRegionHealthCheckRequest{
req := &computepb.InsertHealthCheckRequest{
Project: c.project,
Region: c.region,
HealthCheckResource: &computepb.HealthCheck{
Name: proto.String(lb.name),
Type: proto.String(computepb.HealthCheck_Type_name[int32(lb.healthCheck)]),
@ -144,18 +155,17 @@ func (c *Client) createHealthCheck(ctx context.Context, lb *loadBalancer) error
}
func (c *Client) createBackendService(ctx context.Context, lb *loadBalancer) error {
req := &computepb.InsertRegionBackendServiceRequest{
req := &computepb.InsertBackendServiceRequest{
Project: c.project,
Region: c.region,
BackendServiceResource: &computepb.BackendService{
Name: proto.String(lb.name),
Protocol: proto.String(computepb.BackendService_Protocol_name[int32(computepb.BackendService_TCP)]),
LoadBalancingScheme: proto.String(computepb.BackendService_LoadBalancingScheme_name[int32(computepb.BackendService_EXTERNAL)]),
HealthChecks: []string{c.resourceURI(scopeRegion, "healthChecks", lb.name)},
HealthChecks: []string{c.resourceURI(scopeGlobal, "healthChecks", lb.name)},
PortName: proto.String(lb.backendPortName),
Backends: []*computepb.Backend{
{
BalancingMode: proto.String(computepb.Backend_BalancingMode_name[int32(computepb.Backend_CONNECTION)]),
BalancingMode: proto.String(computepb.Backend_BalancingMode_name[int32(computepb.Backend_UTILIZATION)]),
Group: proto.String(c.resourceURI(scopeZone, "instanceGroups", c.controlPlaneInstanceGroup)),
},
},
@ -175,16 +185,16 @@ func (c *Client) createBackendService(ctx context.Context, lb *loadBalancer) err
}
func (c *Client) createForwardingRules(ctx context.Context, lb *loadBalancer) error {
req := &computepb.InsertForwardingRuleRequest{
req := &computepb.InsertGlobalForwardingRuleRequest{
Project: c.project,
Region: c.region,
ForwardingRuleResource: &computepb.ForwardingRule{
Name: proto.String(lb.name),
IPAddress: proto.String(c.resourceURI(scopeRegion, "addresses", c.loadbalancerIPname)),
IPAddress: proto.String(c.resourceURI(scopeGlobal, "addresses", c.loadbalancerIPname)),
IPProtocol: proto.String(computepb.ForwardingRule_IPProtocolEnum_name[int32(computepb.ForwardingRule_TCP)]),
LoadBalancingScheme: proto.String(computepb.ForwardingRule_LoadBalancingScheme_name[int32(computepb.ForwardingRule_EXTERNAL)]),
Ports: []string{strconv.Itoa(lb.frontendPort)},
BackendService: proto.String(c.resourceURI(scopeRegion, "backendServices", lb.name)),
PortRange: proto.String(strconv.Itoa(lb.frontendPort)),
Target: proto.String(c.resourceURI(scopeGlobal, "targetTcpProxies", lb.name)),
},
}
resp, err := c.forwardingRulesAPI.Insert(ctx, req)
@ -205,9 +215,8 @@ func (c *Client) createForwardingRules(ctx context.Context, lb *loadBalancer) er
// labelLoadBalancer labels a load balancer (its forwarding rules) so that it can be found by applications in the cluster.
func (c *Client) labelLoadBalancer(ctx context.Context, name string) error {
forwardingRule, err := c.forwardingRulesAPI.Get(ctx, &computepb.GetForwardingRuleRequest{
forwardingRule, err := c.forwardingRulesAPI.Get(ctx, &computepb.GetGlobalForwardingRuleRequest{
Project: c.project,
Region: c.region,
ForwardingRule: name,
})
if err != nil {
@ -217,11 +226,10 @@ func (c *Client) labelLoadBalancer(ctx context.Context, name string) error {
return fmt.Errorf("forwarding rule %s has no label fingerprint", name)
}
resp, err := c.forwardingRulesAPI.SetLabels(ctx, &computepb.SetLabelsForwardingRuleRequest{
resp, err := c.forwardingRulesAPI.SetLabels(ctx, &computepb.SetLabelsGlobalForwardingRuleRequest{
Project: c.project,
Region: c.region,
Resource: name,
RegionSetLabelsRequestResource: &computepb.RegionSetLabelsRequest{
GlobalSetLabelsRequestResource: &computepb.GlobalSetLabelsRequest{
Labels: map[string]string{"constellation-uid": c.uid},
LabelFingerprint: forwardingRule.LabelFingerprint,
},
@ -233,6 +241,26 @@ func (c *Client) labelLoadBalancer(ctx context.Context, name string) error {
return c.waitForOperations(ctx, []Operation{resp})
}
func (c *Client) createTargetTCPProxy(ctx context.Context, lb *loadBalancer) error {
req := &computepb.InsertTargetTcpProxyRequest{
Project: c.project,
TargetTcpProxyResource: &computepb.TargetTcpProxy{
Name: proto.String(lb.name),
Service: proto.String(c.resourceURI(scopeGlobal, "backendServices", lb.name)),
},
}
resp, err := c.targetTCPProxiesAPI.Insert(ctx, req)
if err != nil {
return fmt.Errorf("inserting target tcp proxy: %w", err)
}
if err := c.waitForOperations(ctx, []Operation{resp}); err != nil {
return err
}
lb.hasTargetTCPProxy = true
return nil
}
// TerminateLoadBalancers terminates all load balancers.
func (c *Client) TerminateLoadBalancers(ctx context.Context) error {
errC := make(chan error)
@ -276,6 +304,12 @@ func (c *Client) terminateLoadBalancer(ctx context.Context, lb *loadBalancer) er
}
}
if lb.hasTargetTCPProxy {
if err := c.terminateTargetTCPProxy(ctx, lb); err != nil {
return fmt.Errorf("terminating target tcp proxy: %w", err)
}
}
if lb.hasBackendService {
if err := c.terminateBackendService(ctx, lb); err != nil {
return fmt.Errorf("terminating backend services: %w", err)
@ -293,9 +327,8 @@ func (c *Client) terminateLoadBalancer(ctx context.Context, lb *loadBalancer) er
}
func (c *Client) terminateForwadingRules(ctx context.Context, lb *loadBalancer) error {
resp, err := c.forwardingRulesAPI.Delete(ctx, &computepb.DeleteForwardingRuleRequest{
resp, err := c.forwardingRulesAPI.Delete(ctx, &computepb.DeleteGlobalForwardingRuleRequest{
Project: c.project,
Region: c.region,
ForwardingRule: lb.name,
})
if isNotFoundError(err) {
@ -314,10 +347,30 @@ func (c *Client) terminateForwadingRules(ctx context.Context, lb *loadBalancer)
return nil
}
func (c *Client) terminateBackendService(ctx context.Context, lb *loadBalancer) error {
resp, err := c.backendServicesAPI.Delete(ctx, &computepb.DeleteRegionBackendServiceRequest{
func (c *Client) terminateTargetTCPProxy(ctx context.Context, lb *loadBalancer) error {
resp, err := c.targetTCPProxiesAPI.Delete(ctx, &computepb.DeleteTargetTcpProxyRequest{
Project: c.project,
TargetTcpProxy: lb.name,
})
if isNotFoundError(err) {
lb.hasTargetTCPProxy = false
return nil
}
if err != nil {
return fmt.Errorf("deleting target tcp proxy: %w", err)
}
if err := c.waitForOperations(ctx, []Operation{resp}); err != nil {
return err
}
lb.hasTargetTCPProxy = false
return nil
}
func (c *Client) terminateBackendService(ctx context.Context, lb *loadBalancer) error {
resp, err := c.backendServicesAPI.Delete(ctx, &computepb.DeleteBackendServiceRequest{
Project: c.project,
Region: c.region,
BackendService: lb.name,
})
if isNotFoundError(err) {
@ -337,9 +390,8 @@ func (c *Client) terminateBackendService(ctx context.Context, lb *loadBalancer)
}
func (c *Client) terminateHealthCheck(ctx context.Context, lb *loadBalancer) error {
resp, err := c.healthChecksAPI.Delete(ctx, &computepb.DeleteRegionHealthCheckRequest{
resp, err := c.healthChecksAPI.Delete(ctx, &computepb.DeleteHealthCheckRequest{
Project: c.project,
Region: c.region,
HealthCheck: lb.name,
})
if isNotFoundError(err) {
@ -360,9 +412,8 @@ func (c *Client) terminateHealthCheck(ctx context.Context, lb *loadBalancer) err
func (c *Client) createIPAddr(ctx context.Context) error {
ipName := c.buildResourceName()
insertReq := &computepb.InsertAddressRequest{
insertReq := &computepb.InsertGlobalAddressRequest{
Project: c.project,
Region: c.region,
AddressResource: &computepb.Address{
Name: proto.String(ipName),
},
@ -376,9 +427,8 @@ func (c *Client) createIPAddr(ctx context.Context) error {
}
c.loadbalancerIPname = ipName
getReq := &computepb.GetAddressRequest{
getReq := &computepb.GetGlobalAddressRequest{
Project: c.project,
Region: c.region,
Address: c.loadbalancerIPname,
}
addr, err := c.addressesAPI.Get(ctx, getReq)
@ -398,9 +448,8 @@ func (c *Client) deleteIPAddr(ctx context.Context) error {
return nil
}
req := &computepb.DeleteAddressRequest{
req := &computepb.DeleteGlobalAddressRequest{
Project: c.project,
Region: c.region,
Address: c.loadbalancerIPname,
}
op, err := c.addressesAPI.Delete(ctx, req)

185
cli/internal/gcp/client/loadbalancer_test.go
View File

@ -27,8 +27,9 @@ func TestCreateLoadBalancers(t *testing.T) {
addrAPI addressesAPI
healthAPI healthChecksAPI
backendAPI backendServicesAPI
proxyAPI targetTCPProxiesAPI
forwardAPI forwardingRulesAPI
opRegAPI operationRegionAPI
operationAPI operationGlobalAPI
isDebugCluster bool
wantErr bool
}{
@ -36,31 +37,44 @@ func TestCreateLoadBalancers(t *testing.T) {
addrAPI: &stubAddressesAPI{getAddr: proto.String("192.0.2.1")},
healthAPI: &stubHealthChecksAPI{},
backendAPI: &stubBackendServicesAPI{},
proxyAPI: &stubTargetTCPProxiesAPI{},
forwardAPI: &stubForwardingRulesAPI{forwardingRule: forwardingRule},
opRegAPI: stubOperationRegionAPI{},
operationAPI: stubOperationGlobalAPI{},
},
"successful create (debug cluster)": {
addrAPI: &stubAddressesAPI{getAddr: proto.String("192.0.2.1")},
healthAPI: &stubHealthChecksAPI{},
backendAPI: &stubBackendServicesAPI{},
proxyAPI: &stubTargetTCPProxiesAPI{},
forwardAPI: &stubForwardingRulesAPI{forwardingRule: forwardingRule},
opRegAPI: stubOperationRegionAPI{},
operationAPI: stubOperationGlobalAPI{},
isDebugCluster: true,
},
"createIPAddr fails": {
addrAPI: &stubAddressesAPI{insertErr: someErr},
healthAPI: &stubHealthChecksAPI{},
backendAPI: &stubBackendServicesAPI{},
proxyAPI: &stubTargetTCPProxiesAPI{},
forwardAPI: &stubForwardingRulesAPI{forwardingRule: forwardingRule},
opRegAPI: stubOperationRegionAPI{},
operationAPI: stubOperationGlobalAPI{},
wantErr: true,
},
"createLB fails": {
addrAPI: &stubAddressesAPI{},
healthAPI: &stubHealthChecksAPI{},
backendAPI: &stubBackendServicesAPI{insertErr: someErr},
proxyAPI: &stubTargetTCPProxiesAPI{},
forwardAPI: &stubForwardingRulesAPI{forwardingRule: forwardingRule},
opRegAPI: stubOperationRegionAPI{},
operationAPI: stubOperationGlobalAPI{},
wantErr: true,
},
"createTcpProxy fails": {
addrAPI: &stubAddressesAPI{getAddr: proto.String("192.0.2.1")},
healthAPI: &stubHealthChecksAPI{},
backendAPI: &stubBackendServicesAPI{},
proxyAPI: &stubTargetTCPProxiesAPI{insertErr: someErr},
forwardAPI: &stubForwardingRulesAPI{forwardingRule: forwardingRule},
operationAPI: stubOperationGlobalAPI{},
wantErr: true,
},
}
@ -76,10 +90,11 @@ func TestCreateLoadBalancers(t *testing.T) {
name: "name",
uid: "uid",
addressesAPI: tc.addrAPI,
targetTCPProxiesAPI: tc.proxyAPI,
healthChecksAPI: tc.healthAPI,
backendServicesAPI: tc.backendAPI,
forwardingRulesAPI: tc.forwardAPI,
operationRegionAPI: tc.opRegAPI,
operationGlobalAPI: tc.operationAPI,
}
err := client.CreateLoadBalancers(ctx, tc.isDebugCluster)
@ -104,10 +119,10 @@ func TestCreateLoadBalancers(t *testing.T) {
}
if tc.isDebugCluster {
assert.Equal(4, len(client.loadbalancers))
assert.Equal(5, len(client.loadbalancers))
assert.True(foundDebugdLB, "debugd loadbalancer not found in debug-mode")
} else {
assert.Equal(3, len(client.loadbalancers))
assert.Equal(4, len(client.loadbalancers))
assert.False(foundDebugdLB, "debugd loadbalancer found in non-debug mode")
}
})
@ -117,23 +132,26 @@ func TestCreateLoadBalancers(t *testing.T) {
func TestCreateLoadBalancer(t *testing.T) {
someErr := errors.New("failed")
testCases := map[string]struct {
operationRegionAPI operationRegionAPI
operationGlobalAPI operationGlobalAPI
healthChecksAPI healthChecksAPI
backendServicesAPI backendServicesAPI
forwardingRulesAPI forwardingRulesAPI
targetTCPProxiesAPI targetTCPProxiesAPI
wantErr bool
wantLB *loadBalancer
}{
"successful create": {
healthChecksAPI: stubHealthChecksAPI{},
backendServicesAPI: stubBackendServicesAPI{},
targetTCPProxiesAPI: stubTargetTCPProxiesAPI{},
forwardingRulesAPI: stubForwardingRulesAPI{forwardingRule: &compute.ForwardingRule{LabelFingerprint: proto.String("fingerprint")}},
operationRegionAPI: stubOperationRegionAPI{},
operationGlobalAPI: stubOperationGlobalAPI{},
wantLB: &loadBalancer{
name: "name",
frontendPort: 1234,
backendPortName: "testport",
hasHealthCheck: true,
hasTargetTCPProxy: true,
hasBackendService: true,
hasForwardingRules: true,
},
@ -141,14 +159,16 @@ func TestCreateLoadBalancer(t *testing.T) {
"successful create with label": {
healthChecksAPI: stubHealthChecksAPI{},
backendServicesAPI: stubBackendServicesAPI{},
targetTCPProxiesAPI: stubTargetTCPProxiesAPI{},
forwardingRulesAPI: stubForwardingRulesAPI{forwardingRule: &compute.ForwardingRule{LabelFingerprint: proto.String("fingerprint")}},
operationRegionAPI: stubOperationRegionAPI{},
operationGlobalAPI: stubOperationGlobalAPI{},
wantLB: &loadBalancer{
name: "name",
frontendPort: 1234,
backendPortName: "testport",
label: true,
hasHealthCheck: true,
hasTargetTCPProxy: true,
hasBackendService: true,
hasForwardingRules: true,
},
@ -156,8 +176,9 @@ func TestCreateLoadBalancer(t *testing.T) {
"CreateLoadBalancer fails when getting forwarding rule": {
healthChecksAPI: stubHealthChecksAPI{},
backendServicesAPI: stubBackendServicesAPI{},
targetTCPProxiesAPI: stubTargetTCPProxiesAPI{},
forwardingRulesAPI: stubForwardingRulesAPI{getErr: someErr},
operationRegionAPI: stubOperationRegionAPI{},
operationGlobalAPI: stubOperationGlobalAPI{},
wantErr: true,
wantLB: &loadBalancer{
name: "name",
@ -165,6 +186,7 @@ func TestCreateLoadBalancer(t *testing.T) {
backendPortName: "testport",
label: true,
hasHealthCheck: true,
hasTargetTCPProxy: true,
hasBackendService: true,
hasForwardingRules: true,
},
@ -172,8 +194,9 @@ func TestCreateLoadBalancer(t *testing.T) {
"CreateLoadBalancer fails when label fingerprint is missing": {
healthChecksAPI: stubHealthChecksAPI{},
backendServicesAPI: stubBackendServicesAPI{},
targetTCPProxiesAPI: stubTargetTCPProxiesAPI{},
forwardingRulesAPI: stubForwardingRulesAPI{forwardingRule: &compute.ForwardingRule{}},
operationRegionAPI: stubOperationRegionAPI{},
operationGlobalAPI: stubOperationGlobalAPI{},
wantErr: true,
wantLB: &loadBalancer{
name: "name",
@ -181,6 +204,7 @@ func TestCreateLoadBalancer(t *testing.T) {
backendPortName: "testport",
label: true,
hasHealthCheck: true,
hasTargetTCPProxy: true,
hasBackendService: true,
hasForwardingRules: true,
},
@ -188,23 +212,26 @@ func TestCreateLoadBalancer(t *testing.T) {
"CreateLoadBalancer fails when creating health check": {
healthChecksAPI: stubHealthChecksAPI{insertErr: someErr},
backendServicesAPI: stubBackendServicesAPI{},
targetTCPProxiesAPI: stubTargetTCPProxiesAPI{},
forwardingRulesAPI: stubForwardingRulesAPI{forwardingRule: &compute.ForwardingRule{LabelFingerprint: proto.String("fingerprint")}},
operationRegionAPI: stubOperationRegionAPI{},
operationGlobalAPI: stubOperationGlobalAPI{},
wantErr: true,
wantLB: &loadBalancer{
name: "name",
frontendPort: 1234,
backendPortName: "testport",
hasHealthCheck: false,
hasTargetTCPProxy: false,
hasBackendService: false,
hasForwardingRules: false,
},
},
"CreateLoadBalancer fails when creating backend service": {
healthChecksAPI: stubHealthChecksAPI{},
targetTCPProxiesAPI: stubTargetTCPProxiesAPI{},
backendServicesAPI: stubBackendServicesAPI{insertErr: someErr},
forwardingRulesAPI: stubForwardingRulesAPI{},
operationRegionAPI: stubOperationRegionAPI{},
operationGlobalAPI: stubOperationGlobalAPI{},
wantErr: true,
wantLB: &loadBalancer{
name: "name",
@ -212,14 +239,33 @@ func TestCreateLoadBalancer(t *testing.T) {
backendPortName: "testport",
hasHealthCheck: true,
hasBackendService: false,
hasTargetTCPProxy: false,
hasForwardingRules: false,
},
},
"CreateLoadBalancer fails when creating forwarding rule": {
healthChecksAPI: stubHealthChecksAPI{},
backendServicesAPI: stubBackendServicesAPI{},
targetTCPProxiesAPI: stubTargetTCPProxiesAPI{},
forwardingRulesAPI: stubForwardingRulesAPI{insertErr: someErr},
operationRegionAPI: stubOperationRegionAPI{},
operationGlobalAPI: stubOperationGlobalAPI{},
wantErr: true,
wantLB: &loadBalancer{
name: "name",
frontendPort: 1234,
backendPortName: "testport",
hasHealthCheck: true,
hasBackendService: true,
hasTargetTCPProxy: true,
hasForwardingRules: false,
},
},
"CreateLoadBalancer fails when creating target proxy rule": {
healthChecksAPI: stubHealthChecksAPI{},
backendServicesAPI: stubBackendServicesAPI{},
targetTCPProxiesAPI: stubTargetTCPProxiesAPI{insertErr: someErr},
forwardingRulesAPI: stubForwardingRulesAPI{},
operationGlobalAPI: stubOperationGlobalAPI{},
wantErr: true,
wantLB: &loadBalancer{
name: "name",
@ -233,8 +279,9 @@ func TestCreateLoadBalancer(t *testing.T) {
"CreateLoadBalancer fails when waiting on operation": {
healthChecksAPI: stubHealthChecksAPI{},
backendServicesAPI: stubBackendServicesAPI{},
targetTCPProxiesAPI: stubTargetTCPProxiesAPI{},
forwardingRulesAPI: stubForwardingRulesAPI{forwardingRule: &compute.ForwardingRule{LabelFingerprint: proto.String("fingerprint")}},
operationRegionAPI: stubOperationRegionAPI{waitErr: someErr},
operationGlobalAPI: stubOperationGlobalAPI{waitErr: someErr},
wantErr: true,
wantLB: &loadBalancer{
name: "name",
@ -259,8 +306,9 @@ func TestCreateLoadBalancer(t *testing.T) {
uid: "uid",
backendServicesAPI: tc.backendServicesAPI,
forwardingRulesAPI: tc.forwardingRulesAPI,
targetTCPProxiesAPI: tc.targetTCPProxiesAPI,
healthChecksAPI: tc.healthChecksAPI,
operationRegionAPI: tc.operationRegionAPI,
operationGlobalAPI: tc.operationGlobalAPI,
}
lb := &loadBalancer{
name: tc.wantLB.name,
@ -289,6 +337,7 @@ func TestTerminateLoadbalancers(t *testing.T) {
name: "name",
hasHealthCheck: true,
hasBackendService: true,
hasTargetTCPProxy: true,
hasForwardingRules: true,
}
}
@ -297,31 +346,35 @@ func TestTerminateLoadbalancers(t *testing.T) {
addrAPI addressesAPI
healthAPI healthChecksAPI
backendAPI backendServicesAPI
targetAPI targetTCPProxiesAPI
forwardAPI forwardingRulesAPI
opRegionAPI operationRegionAPI
opGlobalAPI operationGlobalAPI
wantErr bool
}{
"successful terminate": {
addrAPI: &stubAddressesAPI{},
healthAPI: &stubHealthChecksAPI{},
backendAPI: &stubBackendServicesAPI{},
targetAPI: &stubTargetTCPProxiesAPI{},
forwardAPI: &stubForwardingRulesAPI{},
opRegionAPI: stubOperationRegionAPI{},
opGlobalAPI: stubOperationGlobalAPI{},
},
"deleteIPAddr fails": {
addrAPI: &stubAddressesAPI{deleteErr: someErr},
healthAPI: &stubHealthChecksAPI{},
backendAPI: &stubBackendServicesAPI{},
targetAPI: &stubTargetTCPProxiesAPI{},
forwardAPI: &stubForwardingRulesAPI{},
opRegionAPI: stubOperationRegionAPI{},
opGlobalAPI: stubOperationGlobalAPI{},
wantErr: true,
},
"deleteLB fails": {
addrAPI: &stubAddressesAPI{},
healthAPI: &stubHealthChecksAPI{},
backendAPI: &stubBackendServicesAPI{deleteErr: someErr},
targetAPI: &stubTargetTCPProxiesAPI{},
forwardAPI: &stubForwardingRulesAPI{},
opRegionAPI: stubOperationRegionAPI{},
opGlobalAPI: stubOperationGlobalAPI{},
wantErr: true,
},
}
@ -339,8 +392,9 @@ func TestTerminateLoadbalancers(t *testing.T) {
addressesAPI: tc.addrAPI,
healthChecksAPI: tc.healthAPI,
backendServicesAPI: tc.backendAPI,
targetTCPProxiesAPI: tc.targetAPI,
forwardingRulesAPI: tc.forwardAPI,
operationRegionAPI: tc.opRegionAPI,
operationGlobalAPI: tc.opGlobalAPI,
loadbalancerIPname: "loadbalancerIPid",
loadbalancers: []*loadBalancer{
newRunningLB(),
@ -369,6 +423,7 @@ func TestTerminateLoadBalancer(t *testing.T) {
return &loadBalancer{
name: "name",
hasHealthCheck: true,
hasTargetTCPProxy: true,
hasBackendService: true,
hasForwardingRules: true,
}
@ -376,9 +431,10 @@ func TestTerminateLoadBalancer(t *testing.T) {
testCases := map[string]struct {
lb *loadBalancer
opRegionAPI operationRegionAPI
opGlobalAPI operationGlobalAPI
healthChecksAPI healthChecksAPI
backendServicesAPI backendServicesAPI
targetTCPProxiesAPI targetTCPProxiesAPI
forwardingRulesAPI forwardingRulesAPI
wantErr bool
wantLB *loadBalancer
@ -388,7 +444,8 @@ func TestTerminateLoadBalancer(t *testing.T) {
healthChecksAPI: stubHealthChecksAPI{},
backendServicesAPI: stubBackendServicesAPI{},
forwardingRulesAPI: stubForwardingRulesAPI{},
opRegionAPI: stubOperationRegionAPI{},
targetTCPProxiesAPI: stubTargetTCPProxiesAPI{},
opGlobalAPI: stubOperationGlobalAPI{},
wantLB: &loadBalancer{},
},
"terminate partially created loadbalancer": {
@ -401,7 +458,8 @@ func TestTerminateLoadBalancer(t *testing.T) {
healthChecksAPI: stubHealthChecksAPI{},
backendServicesAPI: stubBackendServicesAPI{},
forwardingRulesAPI: stubForwardingRulesAPI{deleteErr: someErr},
opRegionAPI: stubOperationRegionAPI{},
targetTCPProxiesAPI: stubTargetTCPProxiesAPI{},
opGlobalAPI: stubOperationGlobalAPI{},
wantLB: &loadBalancer{},
},
"terminate partially created loadbalancer 2": {
@ -414,7 +472,8 @@ func TestTerminateLoadBalancer(t *testing.T) {
healthChecksAPI: stubHealthChecksAPI{},
backendServicesAPI: stubBackendServicesAPI{deleteErr: someErr},
forwardingRulesAPI: stubForwardingRulesAPI{deleteErr: someErr},
opRegionAPI: stubOperationRegionAPI{},
targetTCPProxiesAPI: stubTargetTCPProxiesAPI{},
opGlobalAPI: stubOperationGlobalAPI{},
wantLB: &loadBalancer{},
},
"no-op for nil loadbalancer": {
@ -425,7 +484,8 @@ func TestTerminateLoadBalancer(t *testing.T) {
healthChecksAPI: stubHealthChecksAPI{deleteErr: notFoundErr},
backendServicesAPI: stubBackendServicesAPI{},
forwardingRulesAPI: stubForwardingRulesAPI{},
opRegionAPI: stubOperationRegionAPI{},
targetTCPProxiesAPI: stubTargetTCPProxiesAPI{},
opGlobalAPI: stubOperationGlobalAPI{},
wantLB: &loadBalancer{},
},
"backend service not found": {
@ -433,7 +493,8 @@ func TestTerminateLoadBalancer(t *testing.T) {
healthChecksAPI: stubHealthChecksAPI{},
backendServicesAPI: stubBackendServicesAPI{deleteErr: notFoundErr},
forwardingRulesAPI: stubForwardingRulesAPI{},
opRegionAPI: stubOperationRegionAPI{},
targetTCPProxiesAPI: stubTargetTCPProxiesAPI{},
opGlobalAPI: stubOperationGlobalAPI{},
wantLB: &loadBalancer{},
},
"forwarding rules not found": {
@ -441,7 +502,8 @@ func TestTerminateLoadBalancer(t *testing.T) {
healthChecksAPI: stubHealthChecksAPI{},
backendServicesAPI: stubBackendServicesAPI{},
forwardingRulesAPI: stubForwardingRulesAPI{deleteErr: notFoundErr},
opRegionAPI: stubOperationRegionAPI{},
targetTCPProxiesAPI: stubTargetTCPProxiesAPI{},
opGlobalAPI: stubOperationGlobalAPI{},
wantLB: &loadBalancer{},
},
"fails for loadbalancer without name": {
@ -454,13 +516,15 @@ func TestTerminateLoadBalancer(t *testing.T) {
healthChecksAPI: stubHealthChecksAPI{deleteErr: someErr},
backendServicesAPI: stubBackendServicesAPI{},
forwardingRulesAPI: stubForwardingRulesAPI{},
opRegionAPI: stubOperationRegionAPI{},
targetTCPProxiesAPI: stubTargetTCPProxiesAPI{},
opGlobalAPI: stubOperationGlobalAPI{},
wantErr: true,
wantLB: &loadBalancer{
name: "name",
hasHealthCheck: true,
hasBackendService: false,
hasForwardingRules: false,
hasTargetTCPProxy: false,
},
},
"fails when deleting backend service": {
@ -468,13 +532,15 @@ func TestTerminateLoadBalancer(t *testing.T) {
healthChecksAPI: stubHealthChecksAPI{},
backendServicesAPI: stubBackendServicesAPI{deleteErr: someErr},
forwardingRulesAPI: stubForwardingRulesAPI{},
opRegionAPI: stubOperationRegionAPI{},
targetTCPProxiesAPI: stubTargetTCPProxiesAPI{},
opGlobalAPI: stubOperationGlobalAPI{},
wantErr: true,
wantLB: &loadBalancer{
name: "name",
hasHealthCheck: true,
hasBackendService: true,
hasForwardingRules: false,
hasTargetTCPProxy: false,
},
},
"fails when deleting forwarding rule": {
@ -482,13 +548,31 @@ func TestTerminateLoadBalancer(t *testing.T) {
healthChecksAPI: stubHealthChecksAPI{},
backendServicesAPI: stubBackendServicesAPI{},
forwardingRulesAPI: stubForwardingRulesAPI{deleteErr: someErr},
opRegionAPI: stubOperationRegionAPI{},
targetTCPProxiesAPI: stubTargetTCPProxiesAPI{},
opGlobalAPI: stubOperationGlobalAPI{},
wantErr: true,
wantLB: &loadBalancer{
name: "name",
hasHealthCheck: true,
hasBackendService: true,
hasForwardingRules: true,
hasTargetTCPProxy: true,
},
},
"fails when deleting tcp proxy rule": {
lb: newRunningLB(),
healthChecksAPI: stubHealthChecksAPI{},
backendServicesAPI: stubBackendServicesAPI{},
forwardingRulesAPI: stubForwardingRulesAPI{},
targetTCPProxiesAPI: stubTargetTCPProxiesAPI{deleteErr: someErr},
opGlobalAPI: stubOperationGlobalAPI{},
wantErr: true,
wantLB: &loadBalancer{
name: "name",
hasHealthCheck: true,
hasBackendService: true,
hasForwardingRules: false,
hasTargetTCPProxy: true,
},
},
"fails when waiting on operation": {
@ -496,13 +580,15 @@ func TestTerminateLoadBalancer(t *testing.T) {
healthChecksAPI: stubHealthChecksAPI{},
backendServicesAPI: stubBackendServicesAPI{},
forwardingRulesAPI: stubForwardingRulesAPI{},
opRegionAPI: stubOperationRegionAPI{waitErr: someErr},
targetTCPProxiesAPI: stubTargetTCPProxiesAPI{},
opGlobalAPI: stubOperationGlobalAPI{waitErr: someErr},
wantErr: true,
wantLB: &loadBalancer{
name: "name",
hasHealthCheck: true,
hasBackendService: true,
hasForwardingRules: true,
hasTargetTCPProxy: true,
},
},
}
@ -520,7 +606,8 @@ func TestTerminateLoadBalancer(t *testing.T) {
backendServicesAPI: tc.backendServicesAPI,
forwardingRulesAPI: tc.forwardingRulesAPI,
healthChecksAPI: tc.healthChecksAPI,
operationRegionAPI: tc.opRegionAPI,
targetTCPProxiesAPI: tc.targetTCPProxiesAPI,
operationGlobalAPI: tc.opGlobalAPI,
}
err := client.terminateLoadBalancer(ctx, tc.lb)
@ -541,31 +628,31 @@ func TestCreateIPAddr(t *testing.T) {
testCases := map[string]struct {
addrAPI addressesAPI
opAPI operationRegionAPI
opAPI operationGlobalAPI
wantErr bool
}{
"successful create": {
addrAPI: stubAddressesAPI{getAddr: proto.String("test-ip")},
opAPI: stubOperationRegionAPI{},
opAPI: stubOperationGlobalAPI{},
},
"insert fails": {
addrAPI: stubAddressesAPI{insertErr: someErr},
opAPI: stubOperationRegionAPI{},
opAPI: stubOperationGlobalAPI{},
wantErr: true,
},
"get fails": {
addrAPI: stubAddressesAPI{getErr: someErr},
opAPI: stubOperationRegionAPI{},
opAPI: stubOperationGlobalAPI{},
wantErr: true,
},
"get address nil": {
addrAPI: stubAddressesAPI{getAddr: nil},
opAPI: stubOperationRegionAPI{},
opAPI: stubOperationGlobalAPI{},
wantErr: true,
},
"wait fails": {
addrAPI: stubAddressesAPI{},
opAPI: stubOperationRegionAPI{waitErr: someErr},
opAPI: stubOperationGlobalAPI{waitErr: someErr},
wantErr: true,
},
}
@ -581,7 +668,7 @@ func TestCreateIPAddr(t *testing.T) {
name: "name",
uid: "uid",
addressesAPI: tc.addrAPI,
operationRegionAPI: tc.opAPI,
operationGlobalAPI: tc.opAPI,
}
err := client.createIPAddr(ctx)
@ -603,33 +690,33 @@ func TestDeleteIPAddr(t *testing.T) {
testCases := map[string]struct {
addrAPI addressesAPI
opAPI operationRegionAPI
opAPI operationGlobalAPI
addrID string
wantErr bool
}{
"successful delete": {
addrAPI: stubAddressesAPI{},
opAPI: stubOperationRegionAPI{},
opAPI: stubOperationGlobalAPI{},
addrID: "name",
},
"not found": {
addrAPI: stubAddressesAPI{deleteErr: notFoundErr},
opAPI: stubOperationRegionAPI{},
opAPI: stubOperationGlobalAPI{},
addrID: "name",
},
"empty is no-op": {
addrAPI: stubAddressesAPI{},
opAPI: stubOperationRegionAPI{},
opAPI: stubOperationGlobalAPI{},
},
"delete fails": {
addrAPI: stubAddressesAPI{deleteErr: someErr},
opAPI: stubOperationRegionAPI{},
opAPI: stubOperationGlobalAPI{},
addrID: "name",
wantErr: true,
},
"wait fails": {
addrAPI: stubAddressesAPI{},
opAPI: stubOperationRegionAPI{waitErr: someErr},
opAPI: stubOperationGlobalAPI{waitErr: someErr},
addrID: "name",
wantErr: true,
},
@ -646,7 +733,7 @@ func TestDeleteIPAddr(t *testing.T) {
name: "name",
uid: "uid",
addressesAPI: tc.addrAPI,
operationRegionAPI: tc.opAPI,
operationGlobalAPI: tc.opAPI,
loadbalancerIPname: tc.addrID,
}

20
debugd/cmd/debugd/debugd.go
View File

@ -23,9 +23,7 @@ import (
platform "github.com/edgelesssys/constellation/internal/cloud/cloudprovider"
"github.com/edgelesssys/constellation/internal/deploy/ssh"
"github.com/edgelesssys/constellation/internal/deploy/user"
"github.com/edgelesssys/constellation/internal/iproute"
"github.com/edgelesssys/constellation/internal/logger"
"github.com/edgelesssys/constellation/internal/role"
"github.com/spf13/afero"
)
@ -69,9 +67,6 @@ func main() {
log.Fatalf("%s", err)
}
fetcher = gcpFetcher
if err := setLoadbalancerRoute(ctx, fetcher); err != nil {
log.Errorf("adding load balancer IP to local routing table: %s", err)
}
log.Infof("Added load balancer IP to local routing table")
case platform.QEMU:
fetcher = cloudprovider.NewQEMU()
@ -108,18 +103,3 @@ func writeDebugBanner(log *logger.Logger) {
log.Infof("Unable to print to /dev/ttyS0: %v", err)
}
}
func setLoadbalancerRoute(ctx context.Context, fetcher metadata.Fetcher) error {
ownRole, err := fetcher.Role(ctx)
if err != nil {
return err
}
if ownRole != role.ControlPlane {
return nil
}
ip, err := fetcher.DiscoverLoadbalancerIP(ctx)
if err != nil {
return err
}
return iproute.AddToLocalRoutingTable(ctx, ip)
}

2
internal/cloud/gcp/api.go
View File

@ -29,7 +29,7 @@ type subnetworkAPI interface {
}
type forwardingRulesAPI interface {
List(ctx context.Context, req *computepb.ListForwardingRulesRequest, opts ...gax.CallOption) ForwardingRuleIterator
List(ctx context.Context, req *computepb.ListGlobalForwardingRulesRequest, opts ...gax.CallOption) ForwardingRuleIterator
Close() error
}

17
internal/cloud/gcp/client.go
View File

@ -47,7 +47,7 @@ func NewClient(ctx context.Context) (*Client, error) {
if err != nil {
return nil, err
}
forwardingRulesAPI, err := compute.NewForwardingRulesRESTClient(ctx)
forwardingRulesAPI, err := compute.NewGlobalForwardingRulesRESTClient(ctx)
if err != nil {
return nil, err
}
@ -222,20 +222,14 @@ func (c *Client) RetrieveSubnetworkAliasCIDR(ctx context.Context, project, zone,
}
// RetrieveLoadBalancerEndpoint returns the endpoint of the load balancer with the constellation-uid tag.
func (c *Client) RetrieveLoadBalancerEndpoint(ctx context.Context, project, zone string) (string, error) {
func (c *Client) RetrieveLoadBalancerEndpoint(ctx context.Context, project string) (string, error) {
uid, err := c.UID()
if err != nil {
return "", err
}
region := zoneFromRegionRegex.FindString(zone)
if region == "" {
return "", fmt.Errorf("invalid zone %s", zone)
}
req := &computepb.ListForwardingRulesRequest{
req := &computepb.ListGlobalForwardingRulesRequest{
Project: project,
Region: region,
}
iter := c.forwardingRulesAPI.List(ctx, req)
for {
@ -247,10 +241,11 @@ func (c *Client) RetrieveLoadBalancerEndpoint(ctx context.Context, project, zone
return "", fmt.Errorf("retrieving load balancer IP failed: %w", err)
}
if resp.Labels["constellation-uid"] == uid {
if len(resp.Ports) == 0 {
if resp.PortRange == nil {
return "", errors.New("load balancer with searched UID has no ports")
}
return net.JoinHostPort(*resp.IPAddress, resp.Ports[0]), nil
portRange := strings.Split(*resp.PortRange, "-")
return net.JoinHostPort(*resp.IPAddress, portRange[0]), nil
}
}

12
internal/cloud/gcp/client_test.go
View File

@ -796,7 +796,7 @@ func TestRetrieveLoadBalancerEndpoint(t *testing.T) {
rules: []*computepb.ForwardingRule{
{
IPAddress: proto.String(loadBalancerIP),
Ports: []string{"100"},
PortRange: proto.String("100-100"),
Labels: map[string]string{"constellation-uid": uid},
},
},
@ -811,7 +811,7 @@ func TestRetrieveLoadBalancerEndpoint(t *testing.T) {
rules: []*computepb.ForwardingRule{
{
IPAddress: proto.String(loadBalancerIP),
Ports: []string{"100"},
PortRange: proto.String("100-100"),
},
},
},
@ -825,7 +825,7 @@ func TestRetrieveLoadBalancerEndpoint(t *testing.T) {
rules: []*computepb.ForwardingRule{
{
IPAddress: proto.String(loadBalancerIP),
Ports: []string{"100"},
PortRange: proto.String("100-100"),
Labels: map[string]string{"constellation-uid": uid},
},
},
@ -855,7 +855,7 @@ func TestRetrieveLoadBalancerEndpoint(t *testing.T) {
rules: []*computepb.ForwardingRule{
{
IPAddress: proto.String(loadBalancerIP),
Ports: []string{"100"},
PortRange: proto.String("100-100"),
Labels: map[string]string{"constellation-uid": uid},
},
},
@ -870,7 +870,7 @@ func TestRetrieveLoadBalancerEndpoint(t *testing.T) {
require := require.New(t)
client := Client{forwardingRulesAPI: tc.stubForwardingRulesClient, metadataAPI: tc.stubMetadataClient}
aliasCIDR, err := client.RetrieveLoadBalancerEndpoint(context.Background(), "project", "us-central1-a")
aliasCIDR, err := client.RetrieveLoadBalancerEndpoint(context.Background(), "project")
if tc.wantErr {
assert.Error(err)
@ -1038,7 +1038,7 @@ type stubForwardingRulesClient struct {
CloseErr error
}
func (s stubForwardingRulesClient) List(ctx context.Context, req *computepb.ListForwardingRulesRequest, opts ...gax.CallOption) ForwardingRuleIterator {
func (s stubForwardingRulesClient) List(ctx context.Context, req *computepb.ListGlobalForwardingRulesRequest, opts ...gax.CallOption) ForwardingRuleIterator {
return s.ForwardingRuleIterator
}

8
internal/cloud/gcp/metadata.go
View File

@ -33,7 +33,7 @@ type API interface {
// RetrieveSubnetworkAliasCIDR retrieves the subnetwork CIDR of the current instance.
RetrieveSubnetworkAliasCIDR(ctx context.Context, project, zone, instanceName string) (string, error)
// RetrieveLoadBalancerEndpoint retrieves the load balancer endpoint of the current instance.
RetrieveLoadBalancerEndpoint(ctx context.Context, project, zone string) (string, error)
RetrieveLoadBalancerEndpoint(ctx context.Context, project string) (string, error)
// SetInstanceMetadata sets metadata key: value of the instance specified by project, zone and instanceName.
SetInstanceMetadata(ctx context.Context, project, zone, instanceName, key, value string) error
// UnsetInstanceMetadata removes a metadata key-value pair of the instance specified by project, zone and instanceName.
@ -123,11 +123,7 @@ func (m *Metadata) GetLoadBalancerEndpoint(ctx context.Context) (string, error)
if err != nil {
return "", err
}
zone, err := m.api.RetrieveZone()
if err != nil {
return "", err
}
return m.api.RetrieveLoadBalancerEndpoint(ctx, project, zone)
return m.api.RetrieveLoadBalancerEndpoint(ctx, project)
}
// UID retrieves the UID of the constellation.

2
internal/cloud/gcp/metadata_test.go
View File

@ -293,7 +293,7 @@ func (s *stubGCPClient) RetrieveInstanceName() (string, error) {
return s.instanceName, s.retrieveInstanceNameErr
}
func (s *stubGCPClient) RetrieveLoadBalancerEndpoint(ctx context.Context, project, zone string) (string, error) {
func (s *stubGCPClient) RetrieveLoadBalancerEndpoint(ctx context.Context, project string) (string, error) {
return s.loadBalancerIP, s.retrieveLoadBalancerErr
}

8
internal/cloud/gcp/wrappers.go
View File

@ -50,17 +50,17 @@ func (c *subnetworkClient) Get(ctx context.Context, req *computepb.GetSubnetwork
}
type forwardingRulesClient struct {
*compute.ForwardingRulesClient
*compute.GlobalForwardingRulesClient
}
func (c *forwardingRulesClient) Close() error {
return c.ForwardingRulesClient.Close()
return c.GlobalForwardingRulesClient.Close()
}
func (c *forwardingRulesClient) List(ctx context.Context, req *computepb.ListForwardingRulesRequest,
func (c *forwardingRulesClient) List(ctx context.Context, req *computepb.ListGlobalForwardingRulesRequest,
opts ...gax.CallOption,
) ForwardingRuleIterator {
return c.ForwardingRulesClient.List(ctx, req)
return c.GlobalForwardingRulesClient.List(ctx, req)
}
type metadataClient struct{}

1
internal/constants/constants.go
View File

@ -47,6 +47,7 @@ const (
SSHPort = 22
NVMEOverTCPPort = 8009
DebugdPort = 4000
KonnectivityPort = 8132
// Default NodePort Range
// https://kubernetes.io/docs/concepts/services-networking/service/#type-nodeport
NodePortFrom = 30000

7
internal/constants/firewall.go
View File

@ -42,6 +42,13 @@ var (
IPRange: "0.0.0.0/0",
FromPort: KubernetesPort,
},
{
Name: "konnectivity",
Description: "konnectivity",
Protocol: "tcp",
IPRange: "0.0.0.0/0",
FromPort: KonnectivityPort,
},
}
// IngressRulesDebug is the default set of ingress rules for a Constellation cluster with debug mode.

5
internal/grpc/retry/retry.go
View File

@ -36,6 +36,11 @@ func ServiceIsUnavailable(err error) bool {
return false
}
// retry if GCP proxy LB isn't fully available yet
if strings.HasPrefix(statusErr.Message(), `connection error: desc = "transport: authentication handshake failed: EOF"`) {
return true
}
// ideally we would check the error type directly, but grpc only provides a string
return !strings.HasPrefix(statusErr.Message(), `connection error: desc = "transport: authentication handshake failed`)
}

50
internal/iproute/route.go
View File

@ -1,50 +0,0 @@
/*
Copyright (c) Edgeless Systems GmbH
SPDX-License-Identifier: AGPL-3.0-only
*/
package iproute
import (
"context"
"errors"
"fmt"
"os/exec"
"strings"
)
// AddToLocalRoutingTable adds the IP to the local routing table.
func AddToLocalRoutingTable(ctx context.Context, ip string) error {
return manipulateLocalRoutingTable(ctx, "add", ip)
}
// RemoveFromLocalRoutingTable removes the IPfrom the local routing table.
func RemoveFromLocalRoutingTable(ctx context.Context, ip string) error {
return manipulateLocalRoutingTable(ctx, "del", ip)
}
func manipulateLocalRoutingTable(ctx context.Context, action string, ip string) error {
// https://github.com/GoogleCloudPlatform/guest-agent/blob/792fce795218633bcbde505fb3457a0b24f26d37/google_guest_agent/addresses.go#L179
if !strings.Contains(ip, "/") {
ip = ip + "/32"
}
args := []string{"route", action, "to", "local", ip, "scope", "host", "dev", "ens3", "proto", "66"}
_, err := exec.CommandContext(ctx, "ip", args...).Output()
if err == nil {
return nil
}
var exitErr *exec.ExitError
if !errors.As(err, &exitErr) {
return fmt.Errorf("ip route %s: %w", action, err)
}
if exitErr.ExitCode() == 2 {
// "RTNETLINK answers: File exists" or "RTNETLINK answers: No such process"
//
// Ignore, expected in case of adding an existing route or deleting a route
// that does not exist.
return nil
}
return fmt.Errorf("ip route %s (code %v) with: %s", action, exitErr.ExitCode(), exitErr.Stderr)
}

2
internal/versions/versions.go
View File

@ -43,6 +43,8 @@ func IsPreviewK8sVersion(version ValidK8sVersion) bool {
const (
// Constellation images.
// These images are built in a way that they support all versions currently listed in VersionConfigs.
KonnectivityAgentImage = "us.gcr.io/k8s-artifacts-prod/kas-network-proxy/proxy-agent:v0.0.32"
KonnectivityServerImage = "registry.k8s.io/kas-network-proxy/proxy-server:v0.0.32"
JoinImage = "ghcr.io/edgelesssys/constellation/join-service:v0.0.2-0.20220905091720-bd6c6ce836af"
AccessManagerImage = "ghcr.io/edgelesssys/constellation/access-manager:v0.0.1"
KmsImage = "ghcr.io/edgelesssys/constellation/kmsserver:v0.0.2-0.20220831181049-47d4c9e30423"