[node operator] etcd client implementation

Signed-off-by: Malte Poll <mp@edgeless.systems>
This commit is contained in:
Malte Poll 2022-07-18 16:38:34 +02:00 committed by Malte Poll
parent bef2bcc4a9
commit 242020e304
4 changed files with 367 additions and 1 deletions

View File

@ -7,6 +7,7 @@ require (
github.com/onsi/ginkgo v1.16.5
github.com/onsi/gomega v1.18.1
github.com/stretchr/testify v1.7.5
go.etcd.io/etcd/api/v3 v3.5.4
k8s.io/api v0.24.0
k8s.io/apimachinery v0.24.0
k8s.io/client-go v0.24.0
@ -14,7 +15,17 @@ require (
sigs.k8s.io/controller-runtime v0.12.1
)
require github.com/pmezard/go-difflib v1.0.0 // indirect
require (
github.com/coreos/etcd v3.3.13+incompatible // indirect
github.com/coreos/go-semver v0.3.0 // indirect
github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e // indirect
github.com/coreos/go-systemd/v22 v22.3.2 // indirect
github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
go.etcd.io/etcd/client/pkg/v3 v3.5.4 // indirect
google.golang.org/genproto v0.0.0-20220107163113-42d7afdf6368 // indirect
google.golang.org/grpc v1.40.0 // indirect
)
require (
cloud.google.com/go v0.81.0 // indirect
@ -60,6 +71,8 @@ require (
github.com/prometheus/common v0.32.1 // indirect
github.com/prometheus/procfs v0.7.3 // indirect
github.com/spf13/pflag v1.0.5 // indirect
go.etcd.io/etcd v3.3.27+incompatible
go.etcd.io/etcd/client/v3 v3.5.4
go.uber.org/atomic v1.7.0 // indirect
go.uber.org/multierr v1.6.0 // indirect
go.uber.org/zap v1.19.1 // indirect

View File

@ -107,11 +107,21 @@ github.com/cockroachdb/datadriven v0.0.0-20200714090401-bf6692d28da5/go.mod h1:h
github.com/cockroachdb/errors v1.2.4/go.mod h1:rQD95gz6FARkaKkQXUksEje/d9a6wBJoCr5oaCLELYA=
github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f/go.mod h1:i/u985jwjWRlyHXQbwatDASoW0RMlZ/3i9yJHE2xLkI=
github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk=
github.com/coreos/etcd v3.3.13+incompatible h1:8F3hqu9fGYLBifCmRCJsicFqDx/D68Rt3q1JMazcgBQ=
github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
github.com/coreos/go-oidc v2.1.0+incompatible/go.mod h1:CgnwVTmzoESiwO9qyAFEMiHoZ1nMCKZlZ9V6mm3/LKc=
github.com/coreos/go-oidc v2.1.0+incompatible/go.mod h1:CgnwVTmzoESiwO9qyAFEMiHoZ1nMCKZlZ9V6mm3/LKc=
github.com/coreos/go-semver v0.3.0 h1:wkHLiw0WNATZnSG7epLsujiMCgPAc9xhjJ4tgnAxmfM=
github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e h1:Wf6HqHfScWJN9/ZjdUKyjop4mf3Qdd+1TvvltAvM3m8=
github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
github.com/coreos/go-systemd/v22 v22.3.2 h1:D9/bQk5vlXQFZ6Kwuu6zaiXJ9oTPe68++AzAJc1DzSI=
github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f h1:lBNOc5arjvs8E5mO2tbpBpLoyyu8B6e44T7hJy6potg=
github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA=
github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
github.com/cpuguy83/go-md2man/v2 v2.0.1/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
@ -428,6 +438,7 @@ github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDf
github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo=
github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M=
github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0=
github.com/prometheus/client_golang v1.11.1/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0=
github.com/prometheus/client_golang v1.12.1 h1:ZiaPsmm9uiBeaSMRznKsCDNtPCS0T3JVDGF+06gjBzk=
github.com/prometheus/client_golang v1.12.1/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY=
github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
@ -513,13 +524,25 @@ github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1
github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
go.etcd.io/bbolt v1.3.6/go.mod h1:qXsaaIqmgQH0T+OPdb99Bf+PKfBBQVAdyD6TY9G8XM4=
go.etcd.io/etcd v3.3.27+incompatible h1:5hMrpf6REqTHV2LW2OclNpRtxI0k9ZplMemJsMSWju0=
go.etcd.io/etcd v3.3.27+incompatible/go.mod h1:yaeTdrJi5lOmYerz05bd8+V7KubZs8YSFZfzsF9A6aI=
go.etcd.io/etcd/api/v3 v3.5.0/go.mod h1:cbVKeC6lCfl7j/8jBhAK6aIYO9XOjdptoxU/nLQcPvs=
go.etcd.io/etcd/api/v3 v3.5.0/go.mod h1:cbVKeC6lCfl7j/8jBhAK6aIYO9XOjdptoxU/nLQcPvs=
go.etcd.io/etcd/api/v3 v3.5.1/go.mod h1:cbVKeC6lCfl7j/8jBhAK6aIYO9XOjdptoxU/nLQcPvs=
go.etcd.io/etcd/api/v3 v3.5.4 h1:OHVyt3TopwtUQ2GKdd5wu3PmmipR4FTwCqoEjSyRdIc=
go.etcd.io/etcd/api/v3 v3.5.4/go.mod h1:5GB2vv4A4AOn3yk7MftYGHkUfGtDHnEraIjym4dYz5A=
go.etcd.io/etcd/client/pkg/v3 v3.5.0/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g=
go.etcd.io/etcd/client/pkg/v3 v3.5.0/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g=
go.etcd.io/etcd/client/pkg/v3 v3.5.1/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g=
go.etcd.io/etcd/client/pkg/v3 v3.5.4 h1:lrneYvz923dvC14R54XcA7FXoZ3mlGZAgmwhfm7HqOg=
go.etcd.io/etcd/client/pkg/v3 v3.5.4/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g=
go.etcd.io/etcd/client/v2 v2.305.0/go.mod h1:h9puh54ZTgAKtEbut2oe9P4L/oqKCVB6xsXlzd7alYQ=
go.etcd.io/etcd/client/v2 v2.305.0/go.mod h1:h9puh54ZTgAKtEbut2oe9P4L/oqKCVB6xsXlzd7alYQ=
go.etcd.io/etcd/client/v3 v3.5.0/go.mod h1:AIKXXVX/DQXtfTEqBryiLTUXwON+GuvO6Z7lLS/oTh0=
go.etcd.io/etcd/client/v3 v3.5.0/go.mod h1:AIKXXVX/DQXtfTEqBryiLTUXwON+GuvO6Z7lLS/oTh0=
go.etcd.io/etcd/client/v3 v3.5.1/go.mod h1:OnjH4M8OnAotwaB2l9bVgZzRFKru7/ZMoS46OtKyd3Q=
go.etcd.io/etcd/client/v3 v3.5.4 h1:p83BUL3tAYS0OT/r0qglgc3M1JjhM0diV8DSWAhVXv4=
go.etcd.io/etcd/client/v3 v3.5.4/go.mod h1:ZaRkVgBZC+L+dLCjTcF1hRXpgZXQPOvnA/Ak/gq3kiY=
go.etcd.io/etcd/pkg/v3 v3.5.0/go.mod h1:UzJGatBQ1lXChBkQF0AuAtkRQMYnHubxAEYIrC3MSsE=
go.etcd.io/etcd/raft/v3 v3.5.0/go.mod h1:UFOHSIvO/nKwd4lhkwabrTD3cqW5yVyYYf/KlD00Szc=
go.etcd.io/etcd/server/v3 v3.5.0/go.mod h1:3Ah5ruV+M+7RZr0+Y/5mNLwC+eQlni+mQmOVdCRJoS4=
@ -911,6 +934,7 @@ google.golang.org/genproto v0.0.0-20210319143718-93e7006c17a6/go.mod h1:FWY/as6D
google.golang.org/genproto v0.0.0-20210402141018-6c239bbf2bb1/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A=
google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0=
google.golang.org/genproto v0.0.0-20210831024726-fe130286e0e2/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY=
google.golang.org/genproto v0.0.0-20220107163113-42d7afdf6368 h1:Et6SkiuvnBn+SgrSYXs/BrUpGB4mbdwt4R3vaPIlicA=
google.golang.org/genproto v0.0.0-20220107163113-42d7afdf6368/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc=
google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
@ -933,6 +957,7 @@ google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAG
google.golang.org/grpc v1.36.1/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU=
google.golang.org/grpc v1.37.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM=
google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM=
google.golang.org/grpc v1.40.0 h1:AGJ0Ih4mHjSeibYkFGh1dD9KJ/eOtZ93I6hoHhukQ5Q=
google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34=
google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=

View File

@ -0,0 +1,128 @@
package etcd
import (
"context"
"errors"
"fmt"
"net"
"net/url"
"github.com/edgelesssys/constellation/operators/constellation-node-operator/internal/controlplane"
clientv3 "go.etcd.io/etcd/client/v3"
"go.etcd.io/etcd/pkg/transport"
"sigs.k8s.io/controller-runtime/pkg/client"
)
const (
// etcdListenClientPort defines the port etcd listen on for client traffic
etcdListenClientPort = "2379"
// etcdListenPeerPort defines the port etcd listen on for peer traffic
etcdListenPeerPort = "2380"
// etcdCACertName defines etcd's CA certificate name
etcdCACertName = "/etc/kubernetes/pki/etcd/ca.crt"
// etcdPeerCertName defines etcd's peer certificate name
etcdPeerCertName = "/etc/kubernetes/pki/etcd/peer.crt"
// etcdPeerKeyName defines etcd's peer key name
etcdPeerKeyName = "/etc/kubernetes/pki/etcd/peer.key"
)
var memberNotFoundErr = errors.New("member not found")
// Client is an etcd client that can be used to remove a member from an etcd cluster.
type Client struct {
etcdClient etcdClient
}
// New creates a new Client.
func New(k8sClient client.Client) (*Client, error) {
initialEndpoints, err := getInitialEndpoints(k8sClient)
if err != nil {
return nil, err
}
tlsInfo := transport.TLSInfo{
CertFile: etcdPeerCertName,
KeyFile: etcdPeerKeyName,
TrustedCAFile: etcdCACertName,
}
tlsConfig, err := tlsInfo.ClientConfig()
if err != nil {
return nil, err
}
etcdClient, err := clientv3.New(clientv3.Config{
Endpoints: initialEndpoints,
TLS: tlsConfig,
})
if err != nil {
return nil, err
}
if err = etcdClient.Sync(context.TODO()); err != nil {
return nil, fmt.Errorf("syncing endpoints with etcd: %w", err)
}
return &Client{
etcdClient: etcdClient,
}, nil
}
// Close shuts down the client's etcd connections.
func (c *Client) Close() error {
return c.etcdClient.Close()
}
// RemoveEtcdMemberFromCluster removes an etcd member from the cluster.
func (c *Client) RemoveEtcdMemberFromCluster(ctx context.Context, vpcIP string) error {
memberID, err := c.getMemberID(ctx, vpcIP)
if err != nil {
if err == memberNotFoundErr {
return nil
}
return err
}
_, err = c.etcdClient.MemberRemove(ctx, memberID)
return err
}
// getMemberID returns the member ID of the member with the given vpcIP.
func (c *Client) getMemberID(ctx context.Context, vpcIP string) (uint64, error) {
listResponse, err := c.etcdClient.MemberList(ctx)
if err != nil {
return 0, err
}
wantedPeerURL := peerURL(vpcIP, etcdListenPeerPort)
for _, member := range listResponse.Members {
for _, peerURL := range member.PeerURLs {
if peerURL == wantedPeerURL {
return member.ID, nil
}
}
}
return 0, memberNotFoundErr
}
// peerURL returns the peer etcd URL for the given vpcIP and port.
func peerURL(host, port string) string {
return (&url.URL{
Scheme: "https",
Host: net.JoinHostPort(host, port),
}).String()
}
// getInitialEndpoints returns the initial endpoints for the etcd cluster.
func getInitialEndpoints(k8sClient client.Client) ([]string, error) {
ips, err := controlplane.ListControlPlaneIPs(k8sClient)
if err != nil {
return nil, err
}
etcdEndpoints := make([]string, len(ips))
for i, ip := range ips {
etcdEndpoints[i] = net.JoinHostPort(ip, etcdListenClientPort)
}
return etcdEndpoints, nil
}
type etcdClient interface {
MemberList(ctx context.Context) (*clientv3.MemberListResponse, error)
MemberRemove(ctx context.Context, memberID uint64) (*clientv3.MemberRemoveResponse, error)
Sync(ctx context.Context) error
Close() error
}

View File

@ -0,0 +1,200 @@
package etcd
import (
"context"
"errors"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
pb "go.etcd.io/etcd/api/v3/etcdserverpb"
clientv3 "go.etcd.io/etcd/client/v3"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"sigs.k8s.io/controller-runtime/pkg/client"
)
func TestClose(t *testing.T) {
client := Client{etcdClient: &stubEtcdClient{}}
assert.NoError(t, client.Close())
}
func TestRemoveEtcdMemberFromCluster(t *testing.T) {
testCases := map[string]struct {
vpcIP string
memberListErr error
wantErr bool
}{
"removing member works": {
vpcIP: "192.0.2.1",
},
"member already removed": {
vpcIP: "192.0.2.2",
},
"listing members fails": {
memberListErr: errors.New("listing members failed"),
wantErr: true,
},
}
for name, tc := range testCases {
t.Run(name, func(t *testing.T) {
assert := assert.New(t)
require := require.New(t)
client := Client{etcdClient: &stubEtcdClient{
members: []*pb.Member{
{ID: 1, PeerURLs: []string{"https://192.0.2.1:2380"}},
},
listErr: tc.memberListErr,
}}
err := client.RemoveEtcdMemberFromCluster(context.Background(), tc.vpcIP)
if tc.wantErr {
assert.Error(err)
return
}
require.NoError(err)
})
}
}
func TestGetMemberID(t *testing.T) {
testCases := map[string]struct {
members []*pb.Member
memberListErr error
wantMemberID uint64
wantErr bool
}{
"getting member id works": {
members: []*pb.Member{
{ID: 1, PeerURLs: []string{"https://192.0.2.1:2380"}},
},
wantMemberID: 1,
},
"vpc ip has no corresponding etcd member": {
members: []*pb.Member{
{ID: 1, PeerURLs: []string{"https://192.0.2.2:2380"}},
},
wantErr: true,
},
"listing members fails": {
memberListErr: errors.New("listing members failed"),
wantErr: true,
},
}
for name, tc := range testCases {
t.Run(name, func(t *testing.T) {
assert := assert.New(t)
require := require.New(t)
client := Client{etcdClient: &stubEtcdClient{
members: tc.members,
listErr: tc.memberListErr,
}}
gotMemberID, err := client.getMemberID(context.Background(), "192.0.2.1")
if tc.wantErr {
assert.Error(err)
return
}
require.NoError(err)
assert.Equal(tc.wantMemberID, gotMemberID)
})
}
}
func TestPeerURL(t *testing.T) {
assert.Equal(t, "https://host:2380", peerURL("host", etcdListenPeerPort))
}
func TestGetInitialEndpoints(t *testing.T) {
testCases := map[string]struct {
nodes []corev1.Node
listErr error
wantEndpoints []string
wantErr bool
}{
"listing works": {
nodes: []corev1.Node{
{
ObjectMeta: metav1.ObjectMeta{
Labels: map[string]string{"node-role.kubernetes.io/control-plane": ""},
},
Status: corev1.NodeStatus{Addresses: []corev1.NodeAddress{{
Type: corev1.NodeInternalIP,
Address: "192.0.2.1",
}}},
},
{
Status: corev1.NodeStatus{Addresses: []corev1.NodeAddress{{
Type: corev1.NodeInternalIP,
Address: "192.0.2.2",
}}},
},
},
wantEndpoints: []string{"192.0.2.1:2379"},
},
"listing fails": {
listErr: errors.New("listing failed"),
wantErr: true,
},
}
for name, tc := range testCases {
t.Run(name, func(t *testing.T) {
assert := assert.New(t)
require := require.New(t)
client := &stubK8sClient{
nodes: tc.nodes,
listErr: tc.listErr,
}
gotEndpoints, err := getInitialEndpoints(client)
if tc.wantErr {
assert.Error(err)
return
}
require.NoError(err)
assert.ElementsMatch(tc.wantEndpoints, gotEndpoints)
})
}
}
type stubK8sClient struct {
nodes []corev1.Node
listErr error
client.Client
}
func (c *stubK8sClient) List(ctx context.Context, list client.ObjectList, opts ...client.ListOption) error {
list.(*corev1.NodeList).Items = c.nodes
return c.listErr
}
type stubEtcdClient struct {
members []*pb.Member
listErr error
removeErr error
syncErr error
closeErr error
}
func (c *stubEtcdClient) MemberList(ctx context.Context) (*clientv3.MemberListResponse, error) {
return &clientv3.MemberListResponse{
Members: c.members,
}, c.listErr
}
func (c *stubEtcdClient) MemberRemove(ctx context.Context, memberID uint64) (*clientv3.MemberRemoveResponse, error) {
return &clientv3.MemberRemoveResponse{
Members: c.members,
}, c.removeErr
}
func (c *stubEtcdClient) Sync(ctx context.Context) error {
return c.syncErr
}
func (c *stubEtcdClient) Close() error {
return c.closeErr
}