[node operator] etcd client implementation

Signed-off-by: Malte Poll <mp@edgeless.systems>

operators/constellation-node-operator/go.mod

@@ -7,6 +7,7 @@ require (
 	github.com/onsi/ginkgo v1.16.5
 	github.com/onsi/gomega v1.18.1
 	github.com/stretchr/testify v1.7.5
+	go.etcd.io/etcd/api/v3 v3.5.4
 	k8s.io/api v0.24.0
 	k8s.io/apimachinery v0.24.0
 	k8s.io/client-go v0.24.0
@@ -14,7 +15,17 @@ require (
 	sigs.k8s.io/controller-runtime v0.12.1
 )
 
-require github.com/pmezard/go-difflib v1.0.0 // indirect
+require (
+	github.com/coreos/etcd v3.3.13+incompatible // indirect
+	github.com/coreos/go-semver v0.3.0 // indirect
+	github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e // indirect
+	github.com/coreos/go-systemd/v22 v22.3.2 // indirect
+	github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f // indirect
+	github.com/pmezard/go-difflib v1.0.0 // indirect
+	go.etcd.io/etcd/client/pkg/v3 v3.5.4 // indirect
+	google.golang.org/genproto v0.0.0-20220107163113-42d7afdf6368 // indirect
+	google.golang.org/grpc v1.40.0 // indirect
+)
 
 require (
 	cloud.google.com/go v0.81.0 // indirect
@@ -60,6 +71,8 @@ require (
 	github.com/prometheus/common v0.32.1 // indirect
 	github.com/prometheus/procfs v0.7.3 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
+	go.etcd.io/etcd v3.3.27+incompatible
+	go.etcd.io/etcd/client/v3 v3.5.4
 	go.uber.org/atomic v1.7.0 // indirect
 	go.uber.org/multierr v1.6.0 // indirect
 	go.uber.org/zap v1.19.1 // indirect

operators/constellation-node-operator/go.sum

@@ -107,11 +107,21 @@ github.com/cockroachdb/datadriven v0.0.0-20200714090401-bf6692d28da5/go.mod h1:h
 github.com/cockroachdb/errors v1.2.4/go.mod h1:rQD95gz6FARkaKkQXUksEje/d9a6wBJoCr5oaCLELYA=
 github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f/go.mod h1:i/u985jwjWRlyHXQbwatDASoW0RMlZ/3i9yJHE2xLkI=
 github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk=
+github.com/coreos/etcd v3.3.13+incompatible h1:8F3hqu9fGYLBifCmRCJsicFqDx/D68Rt3q1JMazcgBQ=
+github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
 github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
 github.com/coreos/go-oidc v2.1.0+incompatible/go.mod h1:CgnwVTmzoESiwO9qyAFEMiHoZ1nMCKZlZ9V6mm3/LKc=
+github.com/coreos/go-oidc v2.1.0+incompatible/go.mod h1:CgnwVTmzoESiwO9qyAFEMiHoZ1nMCKZlZ9V6mm3/LKc=
+github.com/coreos/go-semver v0.3.0 h1:wkHLiw0WNATZnSG7epLsujiMCgPAc9xhjJ4tgnAxmfM=
 github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
+github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
+github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e h1:Wf6HqHfScWJN9/ZjdUKyjop4mf3Qdd+1TvvltAvM3m8=
 github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
+github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
+github.com/coreos/go-systemd/v22 v22.3.2 h1:D9/bQk5vlXQFZ6Kwuu6zaiXJ9oTPe68++AzAJc1DzSI=
 github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
+github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
+github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f h1:lBNOc5arjvs8E5mO2tbpBpLoyyu8B6e44T7hJy6potg=
 github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA=
 github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
 github.com/cpuguy83/go-md2man/v2 v2.0.1/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
@@ -428,6 +438,7 @@ github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDf
 github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo=
 github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M=
 github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0=
+github.com/prometheus/client_golang v1.11.1/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0=
 github.com/prometheus/client_golang v1.12.1 h1:ZiaPsmm9uiBeaSMRznKsCDNtPCS0T3JVDGF+06gjBzk=
 github.com/prometheus/client_golang v1.12.1/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY=
 github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
@@ -513,13 +524,25 @@ github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1
 github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
 go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
 go.etcd.io/bbolt v1.3.6/go.mod h1:qXsaaIqmgQH0T+OPdb99Bf+PKfBBQVAdyD6TY9G8XM4=
+go.etcd.io/etcd v3.3.27+incompatible h1:5hMrpf6REqTHV2LW2OclNpRtxI0k9ZplMemJsMSWju0=
+go.etcd.io/etcd v3.3.27+incompatible/go.mod h1:yaeTdrJi5lOmYerz05bd8+V7KubZs8YSFZfzsF9A6aI=
+go.etcd.io/etcd/api/v3 v3.5.0/go.mod h1:cbVKeC6lCfl7j/8jBhAK6aIYO9XOjdptoxU/nLQcPvs=
 go.etcd.io/etcd/api/v3 v3.5.0/go.mod h1:cbVKeC6lCfl7j/8jBhAK6aIYO9XOjdptoxU/nLQcPvs=
 go.etcd.io/etcd/api/v3 v3.5.1/go.mod h1:cbVKeC6lCfl7j/8jBhAK6aIYO9XOjdptoxU/nLQcPvs=
+go.etcd.io/etcd/api/v3 v3.5.4 h1:OHVyt3TopwtUQ2GKdd5wu3PmmipR4FTwCqoEjSyRdIc=
+go.etcd.io/etcd/api/v3 v3.5.4/go.mod h1:5GB2vv4A4AOn3yk7MftYGHkUfGtDHnEraIjym4dYz5A=
+go.etcd.io/etcd/client/pkg/v3 v3.5.0/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g=
 go.etcd.io/etcd/client/pkg/v3 v3.5.0/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g=
 go.etcd.io/etcd/client/pkg/v3 v3.5.1/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g=
+go.etcd.io/etcd/client/pkg/v3 v3.5.4 h1:lrneYvz923dvC14R54XcA7FXoZ3mlGZAgmwhfm7HqOg=
+go.etcd.io/etcd/client/pkg/v3 v3.5.4/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g=
+go.etcd.io/etcd/client/v2 v2.305.0/go.mod h1:h9puh54ZTgAKtEbut2oe9P4L/oqKCVB6xsXlzd7alYQ=
 go.etcd.io/etcd/client/v2 v2.305.0/go.mod h1:h9puh54ZTgAKtEbut2oe9P4L/oqKCVB6xsXlzd7alYQ=
 go.etcd.io/etcd/client/v3 v3.5.0/go.mod h1:AIKXXVX/DQXtfTEqBryiLTUXwON+GuvO6Z7lLS/oTh0=
+go.etcd.io/etcd/client/v3 v3.5.0/go.mod h1:AIKXXVX/DQXtfTEqBryiLTUXwON+GuvO6Z7lLS/oTh0=
 go.etcd.io/etcd/client/v3 v3.5.1/go.mod h1:OnjH4M8OnAotwaB2l9bVgZzRFKru7/ZMoS46OtKyd3Q=
+go.etcd.io/etcd/client/v3 v3.5.4 h1:p83BUL3tAYS0OT/r0qglgc3M1JjhM0diV8DSWAhVXv4=
+go.etcd.io/etcd/client/v3 v3.5.4/go.mod h1:ZaRkVgBZC+L+dLCjTcF1hRXpgZXQPOvnA/Ak/gq3kiY=
 go.etcd.io/etcd/pkg/v3 v3.5.0/go.mod h1:UzJGatBQ1lXChBkQF0AuAtkRQMYnHubxAEYIrC3MSsE=
 go.etcd.io/etcd/raft/v3 v3.5.0/go.mod h1:UFOHSIvO/nKwd4lhkwabrTD3cqW5yVyYYf/KlD00Szc=
 go.etcd.io/etcd/server/v3 v3.5.0/go.mod h1:3Ah5ruV+M+7RZr0+Y/5mNLwC+eQlni+mQmOVdCRJoS4=
@@ -911,6 +934,7 @@ google.golang.org/genproto v0.0.0-20210319143718-93e7006c17a6/go.mod h1:FWY/as6D
 google.golang.org/genproto v0.0.0-20210402141018-6c239bbf2bb1/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A=
 google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0=
 google.golang.org/genproto v0.0.0-20210831024726-fe130286e0e2/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY=
+google.golang.org/genproto v0.0.0-20220107163113-42d7afdf6368 h1:Et6SkiuvnBn+SgrSYXs/BrUpGB4mbdwt4R3vaPIlicA=
 google.golang.org/genproto v0.0.0-20220107163113-42d7afdf6368/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc=
 google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
 google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
@@ -933,6 +957,7 @@ google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAG
 google.golang.org/grpc v1.36.1/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU=
 google.golang.org/grpc v1.37.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM=
 google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM=
+google.golang.org/grpc v1.40.0 h1:AGJ0Ih4mHjSeibYkFGh1dD9KJ/eOtZ93I6hoHhukQ5Q=
 google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34=
 google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
 google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=

operators/constellation-node-operator/internal/etcd/etcd.go

@@ -0,0 +1,128 @@
+package etcd
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"net"
+	"net/url"
+
+	"github.com/edgelesssys/constellation/operators/constellation-node-operator/internal/controlplane"
+	clientv3 "go.etcd.io/etcd/client/v3"
+	"go.etcd.io/etcd/pkg/transport"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+)
+
+const (
+	// etcdListenClientPort defines the port etcd listen on for client traffic
+	etcdListenClientPort = "2379"
+	// etcdListenPeerPort defines the port etcd listen on for peer traffic
+	etcdListenPeerPort = "2380"
+	// etcdCACertName defines etcd's CA certificate name
+	etcdCACertName = "/etc/kubernetes/pki/etcd/ca.crt"
+	// etcdPeerCertName defines etcd's peer certificate name
+	etcdPeerCertName = "/etc/kubernetes/pki/etcd/peer.crt"
+	// etcdPeerKeyName defines etcd's peer key name
+	etcdPeerKeyName = "/etc/kubernetes/pki/etcd/peer.key"
+)
+
+var memberNotFoundErr = errors.New("member not found")
+
+// Client is an etcd client that can be used to remove a member from an etcd cluster.
+type Client struct {
+	etcdClient etcdClient
+}
+
+// New creates a new Client.
+func New(k8sClient client.Client) (*Client, error) {
+	initialEndpoints, err := getInitialEndpoints(k8sClient)
+	if err != nil {
+		return nil, err
+	}
+	tlsInfo := transport.TLSInfo{
+		CertFile:      etcdPeerCertName,
+		KeyFile:       etcdPeerKeyName,
+		TrustedCAFile: etcdCACertName,
+	}
+	tlsConfig, err := tlsInfo.ClientConfig()
+	if err != nil {
+		return nil, err
+	}
+	etcdClient, err := clientv3.New(clientv3.Config{
+		Endpoints: initialEndpoints,
+		TLS:       tlsConfig,
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	if err = etcdClient.Sync(context.TODO()); err != nil {
+		return nil, fmt.Errorf("syncing endpoints with etcd: %w", err)
+	}
+	return &Client{
+		etcdClient: etcdClient,
+	}, nil
+}
+
+// Close shuts down the client's etcd connections.
+func (c *Client) Close() error {
+	return c.etcdClient.Close()
+}
+
+// RemoveEtcdMemberFromCluster removes an etcd member from the cluster.
+func (c *Client) RemoveEtcdMemberFromCluster(ctx context.Context, vpcIP string) error {
+	memberID, err := c.getMemberID(ctx, vpcIP)
+	if err != nil {
+		if err == memberNotFoundErr {
+			return nil
+		}
+		return err
+	}
+	_, err = c.etcdClient.MemberRemove(ctx, memberID)
+	return err
+}
+
+// getMemberID returns the member ID of the member with the given vpcIP.
+func (c *Client) getMemberID(ctx context.Context, vpcIP string) (uint64, error) {
+	listResponse, err := c.etcdClient.MemberList(ctx)
+	if err != nil {
+		return 0, err
+	}
+	wantedPeerURL := peerURL(vpcIP, etcdListenPeerPort)
+	for _, member := range listResponse.Members {
+		for _, peerURL := range member.PeerURLs {
+			if peerURL == wantedPeerURL {
+				return member.ID, nil
+			}
+		}
+	}
+	return 0, memberNotFoundErr
+}
+
+// peerURL returns the peer etcd URL for the given vpcIP and port.
+func peerURL(host, port string) string {
+	return (&url.URL{
+		Scheme: "https",
+		Host:   net.JoinHostPort(host, port),
+	}).String()
+}
+
+// getInitialEndpoints returns the initial endpoints for the etcd cluster.
+func getInitialEndpoints(k8sClient client.Client) ([]string, error) {
+	ips, err := controlplane.ListControlPlaneIPs(k8sClient)
+	if err != nil {
+		return nil, err
+	}
+	etcdEndpoints := make([]string, len(ips))
+	for i, ip := range ips {
+		etcdEndpoints[i] = net.JoinHostPort(ip, etcdListenClientPort)
+	}
+	return etcdEndpoints, nil
+}
+
+type etcdClient interface {
+	MemberList(ctx context.Context) (*clientv3.MemberListResponse, error)
+	MemberRemove(ctx context.Context, memberID uint64) (*clientv3.MemberRemoveResponse, error)
+	Sync(ctx context.Context) error
+	Close() error
+}

operators/constellation-node-operator/internal/etcd/etcd_test.go

@@ -0,0 +1,200 @@
+package etcd
+
+import (
+	"context"
+	"errors"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	pb "go.etcd.io/etcd/api/v3/etcdserverpb"
+	clientv3 "go.etcd.io/etcd/client/v3"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+)
+
+func TestClose(t *testing.T) {
+	client := Client{etcdClient: &stubEtcdClient{}}
+	assert.NoError(t, client.Close())
+}
+
+func TestRemoveEtcdMemberFromCluster(t *testing.T) {
+	testCases := map[string]struct {
+		vpcIP         string
+		memberListErr error
+		wantErr       bool
+	}{
+		"removing member works": {
+			vpcIP: "192.0.2.1",
+		},
+		"member already removed": {
+			vpcIP: "192.0.2.2",
+		},
+		"listing members fails": {
+			memberListErr: errors.New("listing members failed"),
+			wantErr:       true,
+		},
+	}
+
+	for name, tc := range testCases {
+		t.Run(name, func(t *testing.T) {
+			assert := assert.New(t)
+			require := require.New(t)
+
+			client := Client{etcdClient: &stubEtcdClient{
+				members: []*pb.Member{
+					{ID: 1, PeerURLs: []string{"https://192.0.2.1:2380"}},
+				},
+				listErr: tc.memberListErr,
+			}}
+			err := client.RemoveEtcdMemberFromCluster(context.Background(), tc.vpcIP)
+			if tc.wantErr {
+				assert.Error(err)
+				return
+			}
+			require.NoError(err)
+		})
+	}
+}
+
+func TestGetMemberID(t *testing.T) {
+	testCases := map[string]struct {
+		members       []*pb.Member
+		memberListErr error
+		wantMemberID  uint64
+		wantErr       bool
+	}{
+		"getting member id works": {
+			members: []*pb.Member{
+				{ID: 1, PeerURLs: []string{"https://192.0.2.1:2380"}},
+			},
+			wantMemberID: 1,
+		},
+		"vpc ip has no corresponding etcd member": {
+			members: []*pb.Member{
+				{ID: 1, PeerURLs: []string{"https://192.0.2.2:2380"}},
+			},
+			wantErr: true,
+		},
+		"listing members fails": {
+			memberListErr: errors.New("listing members failed"),
+			wantErr:       true,
+		},
+	}
+
+	for name, tc := range testCases {
+		t.Run(name, func(t *testing.T) {
+			assert := assert.New(t)
+			require := require.New(t)
+
+			client := Client{etcdClient: &stubEtcdClient{
+				members: tc.members,
+				listErr: tc.memberListErr,
+			}}
+			gotMemberID, err := client.getMemberID(context.Background(), "192.0.2.1")
+			if tc.wantErr {
+				assert.Error(err)
+				return
+			}
+			require.NoError(err)
+			assert.Equal(tc.wantMemberID, gotMemberID)
+		})
+	}
+}
+
+func TestPeerURL(t *testing.T) {
+	assert.Equal(t, "https://host:2380", peerURL("host", etcdListenPeerPort))
+}
+
+func TestGetInitialEndpoints(t *testing.T) {
+	testCases := map[string]struct {
+		nodes         []corev1.Node
+		listErr       error
+		wantEndpoints []string
+		wantErr       bool
+	}{
+		"listing works": {
+			nodes: []corev1.Node{
+				{
+					ObjectMeta: metav1.ObjectMeta{
+						Labels: map[string]string{"node-role.kubernetes.io/control-plane": ""},
+					},
+					Status: corev1.NodeStatus{Addresses: []corev1.NodeAddress{{
+						Type:    corev1.NodeInternalIP,
+						Address: "192.0.2.1",
+					}}},
+				},
+				{
+					Status: corev1.NodeStatus{Addresses: []corev1.NodeAddress{{
+						Type:    corev1.NodeInternalIP,
+						Address: "192.0.2.2",
+					}}},
+				},
+			},
+			wantEndpoints: []string{"192.0.2.1:2379"},
+		},
+		"listing fails": {
+			listErr: errors.New("listing failed"),
+			wantErr: true,
+		},
+	}
+
+	for name, tc := range testCases {
+		t.Run(name, func(t *testing.T) {
+			assert := assert.New(t)
+			require := require.New(t)
+
+			client := &stubK8sClient{
+				nodes:   tc.nodes,
+				listErr: tc.listErr,
+			}
+			gotEndpoints, err := getInitialEndpoints(client)
+			if tc.wantErr {
+				assert.Error(err)
+				return
+			}
+			require.NoError(err)
+			assert.ElementsMatch(tc.wantEndpoints, gotEndpoints)
+		})
+	}
+}
+
+type stubK8sClient struct {
+	nodes   []corev1.Node
+	listErr error
+	client.Client
+}
+
+func (c *stubK8sClient) List(ctx context.Context, list client.ObjectList, opts ...client.ListOption) error {
+	list.(*corev1.NodeList).Items = c.nodes
+	return c.listErr
+}
+
+type stubEtcdClient struct {
+	members   []*pb.Member
+	listErr   error
+	removeErr error
+	syncErr   error
+	closeErr  error
+}
+
+func (c *stubEtcdClient) MemberList(ctx context.Context) (*clientv3.MemberListResponse, error) {
+	return &clientv3.MemberListResponse{
+		Members: c.members,
+	}, c.listErr
+}
+
+func (c *stubEtcdClient) MemberRemove(ctx context.Context, memberID uint64) (*clientv3.MemberRemoveResponse, error) {
+	return &clientv3.MemberRemoveResponse{
+		Members: c.members,
+	}, c.removeErr
+}
+
+func (c *stubEtcdClient) Sync(ctx context.Context) error {
+	return c.syncErr
+}
+
+func (c *stubEtcdClient) Close() error {
+	return c.closeErr
+}