refactor storewrapper IP handling / add coordinator IP-Block

coordinator/core/core.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"errors"
 	"net"
+	"net/netip"
 	"sync"
 	"time"
 
@@ -18,7 +19,7 @@ import (
 	"go.uber.org/zap"
 )
 
-var coordinatorVPNIP = net.IP{10, 118, 0, 1}
+var coordinatorVPNIP = netip.AddrFrom4([4]byte{10, 118, 0, 1})
 
 type Core struct {
 	state                  state.State
@@ -57,11 +58,6 @@ func NewCore(vpn VPN, kube Cluster,
 		persistentStoreFactory: persistentStoreFactory,
 		lastHeartbeats:         make(map[string]time.Time),
 	}
-
-	if err := c.data().PutLastNodeIP(coordinatorVPNIP.To4()); err != nil {
-		return nil, err
-	}
-
 	if err := c.data().IncrementPeersResourceVersion(); err != nil {
 		return nil, err
 	}
@@ -90,6 +86,11 @@ func (c *Core) GetVPNPubKey() ([]byte, error) {
 	return c.data().GetVPNKey()
 }
 
+// GetVPNPubKey returns the peer's VPN public key.
+func (c *Core) InitializeStoreIPs() error {
+	return c.data().InitializeStoreIPs()
+}
+
 // SetVPNIP sets the peer's VPN IP.
 func (c *Core) SetVPNIP(ip string) error {
 	return c.vpn.SetInterfaceIP(ip)
@@ -102,7 +103,7 @@ func (*Core) GetCoordinatorVPNIP() string {
 
 // AddAdmin adds an admin to the VPN.
 func (c *Core) AddAdmin(pubKey []byte) (string, error) {
-	vpnIP, err := c.GenerateNextIP()
+	vpnIP, err := c.GetNextNodeIP()
 	if err != nil {
 		return "", err
 	}
@@ -112,8 +113,8 @@ func (c *Core) AddAdmin(pubKey []byte) (string, error) {
 	return vpnIP, nil
 }
 
-// GenerateNextIP gets the next free IP-Addr.
+// GetNextNodeIP gets the next free IP-Addr.
-func (c *Core) GenerateNextIP() (string, error) {
+func (c *Core) GetNextNodeIP() (string, error) {
 	tx, err := c.store.BeginTransaction()
 	if err != nil {
 		return "", err
@@ -123,7 +124,7 @@ func (c *Core) GenerateNextIP() (string, error) {
 	if err != nil {
 		return "", err
 	}
-	return ip, tx.Commit()
+	return ip.String(), tx.Commit()
 }
 
 // SwitchToPersistentStore creates a new store using the persistentStoreFactory and transfers the initial temporary store into it.

coordinator/core/core_test.go

@@ -30,6 +30,7 @@ func TestAddAdmin(t *testing.T) {
 	vpn := &stubVPN{}
 	core, err := NewCore(vpn, nil, nil, nil, nil, nil, zaptest.NewLogger(t), nil, nil)
 	require.NoError(err)
+	require.NoError(core.InitializeStoreIPs())
 
 	pubKey := []byte{2, 3, 4}
 
@@ -39,41 +40,46 @@ func TestAddAdmin(t *testing.T) {
 	assert.Equal([]stubVPNPeer{{pubKey: pubKey, vpnIP: vpnIP}}, vpn.peers)
 }
 
-func TestGenerateNextIP(t *testing.T) {
+func TestGetNextNodeIP(t *testing.T) {
 	assert := assert.New(t)
 	require := require.New(t)
 
 	core, err := NewCore(&stubVPN{}, nil, nil, nil, nil, nil, zaptest.NewLogger(t), nil, nil)
 	require.NoError(err)
+	require.NoError(core.InitializeStoreIPs())
 
-	ip, err := core.GenerateNextIP()
+	ip, err := core.GetNextNodeIP()
 	assert.NoError(err)
-	assert.Equal("10.118.0.2", ip)
+	assert.Equal("10.118.0.11", ip)
 
-	ip, err = core.GenerateNextIP()
+	ip, err = core.GetNextNodeIP()
 	assert.NoError(err)
-	assert.Equal("10.118.0.3", ip)
+	assert.Equal("10.118.0.12", ip)
 
-	require.NoError(core.data().PutFreedNodeVPNIP("10.118.0.2"))
+	ip, err = core.GetNextNodeIP()
-	require.NoError(core.data().PutFreedNodeVPNIP("10.118.0.3"))
+	assert.NoError(err)
+	assert.Equal("10.118.0.13", ip)
+
+	require.NoError(core.data().PutFreedNodeVPNIP("10.118.0.12"))
+	require.NoError(core.data().PutFreedNodeVPNIP("10.118.0.13"))
 	ipsInStore := map[string]struct{}{
-		"10.118.0.3": {},
+		"10.118.0.13": {},
-		"10.118.0.2": {},
+		"10.118.0.12": {},
 	}
 
-	ip, err = core.GenerateNextIP()
+	ip, err = core.GetNextNodeIP()
 	assert.NoError(err)
 	assert.Contains(ipsInStore, ip)
 	delete(ipsInStore, ip)
 
-	ip, err = core.GenerateNextIP()
+	ip, err = core.GetNextNodeIP()
 	assert.NoError(err)
 	assert.Contains(ipsInStore, ip)
 	delete(ipsInStore, ip)
 
-	ip, err = core.GenerateNextIP()
+	ip, err = core.GetNextNodeIP()
 	assert.NoError(err)
-	assert.Equal("10.118.0.4", ip)
+	assert.Equal("10.118.0.14", ip)
 }
 
 func TestSwitchToPersistentStore(t *testing.T) {

coordinator/pubapi/coord.go

@@ -40,6 +40,10 @@ func (a *API) ActivateAsCoordinator(in *pubproto.ActivateAsCoordinatorRequest, s
 	// AdvanceState MUST be called before any other functions that are not sanity checks or otherwise required
 	// This ensures the node is marked as initialzed before the node is in a state that allows code execution
 	// Any new additions to ActivateAsNode MUST come after
+	if err := a.core.InitializeStoreIPs(); err != nil {
+		return status.Errorf(codes.Internal, "failed to initialize store IPs %v", err)
+	}
+
 	ownerID, clusterID, err := a.core.GetIDs(in.MasterSecret)
 	if err != nil {
 		return status.Errorf(codes.Internal, "%v", err)
@@ -166,7 +170,7 @@ func (a *API) activateNodes(logToCLI logFunc, nodePublicEndpoints []string, coor
 	// Activate all nodes.
 	for num, nodePublicEndpoint := range nodePublicEndpoints {
 		logToCLI("activating node %3d out of %3d nodes", num+1, len(nodePublicEndpoints))
-		nodeVPNIP, err := a.core.GenerateNextIP()
+		nodeVPNIP, err := a.core.GetNextNodeIP()
 		if err != nil {
 			a.logger.Error("generation of vpn ips failed", zap.Error(err))
 			return err

coordinator/pubapi/core.go

@@ -12,8 +12,9 @@ type Core interface {
 	GetVPNPubKey() ([]byte, error)
 	SetVPNIP(string) error
 	GetCoordinatorVPNIP() string
+	InitializeStoreIPs() error
 	AddAdmin(pubKey []byte) (string, error)
-	GenerateNextIP() (string, error)
+	GetNextNodeIP() (string, error)
 	SwitchToPersistentStore() error
 	GetIDs(masterSecret []byte) (ownerID []byte, clusterID []byte, err error)
 	SetUpKMS(ctx context.Context, storageURI, kmsURI, kekID string, useExisting bool) error

coordinator/pubapi/core_test.go

@@ -41,6 +41,10 @@ func (c *fakeCore) SetVPNIP(ip string) error {
 	return c.setVPNIPErr
 }
 
+func (c *fakeCore) InitializeStoreIPs() error {
+	return nil
+}
+
 func (*fakeCore) GetCoordinatorVPNIP() string {
 	return "192.0.2.100"
 }
@@ -50,7 +54,7 @@ func (c *fakeCore) AddAdmin(pubKey []byte) (string, error) {
 	return "192.0.2.99", nil
 }
 
-func (c *fakeCore) GenerateNextIP() (string, error) {
+func (c *fakeCore) GetNextNodeIP() (string, error) {
 	c.nextIP++
 	return fmt.Sprintf("192.0.2.%v", 100+c.nextIP), nil
 }

coordinator/storewrapper/storewrapper.go

@@ -5,7 +5,7 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
-	"net"
+	"net/netip"
 	"strconv"
 	"strings"
 
@@ -15,18 +15,29 @@ import (
 	kubeadm "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta3"
 )
 
+// variables which will be used as a store-prefix start with prefix[...].
+// variables which will be used as a store-key start with key[...].
 const (
-	requestMasterSecret          = "masterSecret"
+	keyHighestAvailableCoordinatorIP = "highestAvailableCoordinatorIP"
-	requestClusterID             = "clusterID"
+	keyHighestAvailableNodeIP        = "highestAvailableNodeIP"
-	requestVPNPubKey             = "vpnKey"
+	keyKubernetesJoinCommand         = "kubeJoin"
-	requestKubernetesJoinCommand = "kubeJoin"
+	keyPeersResourceVersion          = "peersResourceVersion"
-	requestKubeConfig            = "kubeConfig"
+	keyMasterSecret                  = "masterSecret"
-	requestKEKID                 = "kekID"
+	keyKubeConfig                    = "kubeConfig"
-	peerLocationPrefix           = "PeerPrefix"
+	keyClusterID                     = "clusterID"
-	peersResourceVersion         = "peersResourceVersion"
+	keyVPNPubKey                     = "vpnKey"
-	adminLocation                = "externalAdmin"
+	keyKEKID                         = "kekID"
-	freeNodeIP                   = "freeNodeVPNIPs"
+	prefixFreeCoordinatorIPs         = "freeCoordinatorVPNIPs"
-	lastNodeIP                   = "LastNodeIPPrefix"
+	prefixAdminLocation              = "externalAdminsData"
+	prefixPeerLocation               = "peerPrefix"
+	prefixFreeNodeIPs                = "freeNodeVPNIPs"
+)
+
+var (
+	coordinatorIPRangeStart = netip.AddrFrom4([4]byte{10, 118, 0, 1})
+	coordinatorIPRangeEnd   = netip.AddrFrom4([4]byte{10, 118, 0, 10})
+	nodeIPRangeStart        = netip.AddrFrom4([4]byte{10, 118, 0, 11})
+	nodeIPRangeEnd          = netip.AddrFrom4([4]byte{10, 118, 255, 254})
 )
 
 // StoreWrapper is a wrapper for the store interface.
@@ -62,34 +73,34 @@ func (s StoreWrapper) PutState(currState state.State) error {
 
 // GetVPNKey returns the VPN pubKey from Store.
 func (s StoreWrapper) GetVPNKey() ([]byte, error) {
-	return s.Store.Get(requestVPNPubKey)
+	return s.Store.Get(keyVPNPubKey)
 }
 
 // PutVPNKey saves the VPN pubKey to store.
 func (s StoreWrapper) PutVPNKey(key []byte) error {
-	return s.Store.Put(requestVPNPubKey, key)
+	return s.Store.Put(keyVPNPubKey, key)
 }
 
 // PutPeer puts a single peer in the store, with a unique key derived form the VPNIP.
 func (s StoreWrapper) PutPeer(peer peer.Peer) error {
 	if len(peer.VPNIP) == 0 {
-		return fmt.Errorf("unique ID of peer not set")
+		return errors.New("unique ID of peer not set")
 	}
 	jsonPeer, err := json.Marshal(peer)
 	if err != nil {
 		return err
 	}
-	return s.Store.Put(peerLocationPrefix+peer.VPNIP, jsonPeer)
+	return s.Store.Put(prefixPeerLocation+peer.VPNIP, jsonPeer)
 }
 
 // RemovePeer removes a peer from the store.
 func (s StoreWrapper) RemovePeer(peer peer.Peer) error {
-	return s.Store.Delete(peerLocationPrefix + peer.VPNIP)
+	return s.Store.Delete(prefixPeerLocation + peer.VPNIP)
 }
 
 // GetPeer returns a peer requested by the given VPN IP address.
 func (s StoreWrapper) GetPeer(vpnIP string) (peer.Peer, error) {
-	bytePeer, err := s.Store.Get(peerLocationPrefix + vpnIP)
+	bytePeer, err := s.Store.Get(prefixPeerLocation + vpnIP)
 	if err != nil {
 		return peer.Peer{}, err
 	}
@@ -100,7 +111,7 @@ func (s StoreWrapper) GetPeer(vpnIP string) (peer.Peer, error) {
 
 // GetPeers returns all peers in the store.
 func (s StoreWrapper) GetPeers() ([]peer.Peer, error) {
-	return s.getPeersByPrefix(peerLocationPrefix)
+	return s.getPeersByPrefix(prefixPeerLocation)
 }
 
 // IncrementPeersResourceVersion increments the version of the stored peers.
@@ -113,12 +124,12 @@ func (s StoreWrapper) IncrementPeersResourceVersion() error {
 	} else if err != nil {
 		return err
 	}
-	return s.Store.Put(peersResourceVersion, []byte(strconv.Itoa(val+1)))
+	return s.Store.Put(keyPeersResourceVersion, []byte(strconv.Itoa(val+1)))
 }
 
 // GetPeersResourceVersion returns the current version of the stored peers.
 func (s StoreWrapper) GetPeersResourceVersion() (int, error) {
-	raw, err := s.Store.Get(peersResourceVersion)
+	raw, err := s.Store.Get(keyPeersResourceVersion)
 	if err != nil {
 		return 0, err
 	}
@@ -137,7 +148,7 @@ func (s StoreWrapper) UpdatePeers(peers []peer.Peer) (added, removed []peer.Peer
 		updatedPeers[p.VPNIP] = p
 	}
 
-	it, err := s.Store.Iterator(peerLocationPrefix)
+	it, err := s.Store.Iterator(prefixPeerLocation)
 	if err != nil {
 		return nil, nil, err
 	}
@@ -177,7 +188,7 @@ func (s StoreWrapper) UpdatePeers(peers []peer.Peer) (added, removed []peer.Peer
 
 	// perform remove and add
 	for _, p := range removed {
-		if err := s.Store.Delete(peerLocationPrefix + p.VPNIP); err != nil {
+		if err := s.Store.Delete(prefixPeerLocation + p.VPNIP); err != nil {
 			return nil, nil, err
 		}
 	}
@@ -186,7 +197,7 @@ func (s StoreWrapper) UpdatePeers(peers []peer.Peer) (added, removed []peer.Peer
 		if err != nil {
 			return nil, nil, err
 		}
-		if err := s.Store.Put(peerLocationPrefix+p.VPNIP, data); err != nil {
+		if err := s.Store.Put(prefixPeerLocation+p.VPNIP, data); err != nil {
 			return nil, nil, err
 		}
 	}
@@ -200,7 +211,29 @@ func (s StoreWrapper) PutAdmin(peer peer.AdminData) error {
 	if err != nil {
 		return err
 	}
-	return s.Store.Put(adminLocation+peer.VPNIP, jsonPeer)
+	return s.Store.Put(prefixAdminLocation+peer.VPNIP, jsonPeer)
+}
+
+// GetAdmin gets a single admin from the store.
+// TODO: extend if we want to have multiple admins.
+func (s StoreWrapper) GetAdmin() (peer.AdminData, error) {
+	iter, err := s.Store.Iterator(prefixAdminLocation)
+	if err != nil {
+		return peer.AdminData{}, err
+	}
+	key, err := iter.GetNext()
+	if err != nil {
+		return peer.AdminData{}, err
+	}
+	value, err := s.Store.Get(key)
+	if err != nil {
+		return peer.AdminData{}, err
+	}
+	var adminData peer.AdminData
+	if err := json.Unmarshal(value, &adminData); err != nil {
+		return peer.AdminData{}, err
+	}
+	return adminData, nil
 }
 
 func (s StoreWrapper) getPeersByPrefix(prefix string) ([]peer.Peer, error) {
@@ -228,30 +261,9 @@ func (s StoreWrapper) getPeersByPrefix(prefix string) ([]peer.Peer, error) {
 	return peers, nil
 }
 
-// PutFreedNodeVPNIP stores a VPNIP in the store at a unique key.
-func (s StoreWrapper) PutFreedNodeVPNIP(vpnIP string) error {
-	return s.Store.Put(freeNodeIP+vpnIP, nil)
-}
-
-// GetFreedNodeVPNIP reclaims a VPNIP from the store and removes it from there.
-func (s StoreWrapper) GetFreedNodeVPNIP() (string, error) {
-	iter, err := s.Store.Iterator(freeNodeIP)
-	if err != nil {
-		return "", err
-	}
-	if !iter.HasNext() {
-		return "", nil
-	}
-	vpnIP, err := iter.GetNext()
-	if err != nil {
-		return "", err
-	}
-	return strings.TrimPrefix(vpnIP, freeNodeIP), s.Store.Delete(vpnIP)
-}
-
 // GetKubernetesJoinArgs returns the Kubernetes join command from store.
 func (s StoreWrapper) GetKubernetesJoinArgs() (*kubeadm.BootstrapTokenDiscovery, error) {
-	rawJoinCommand, err := s.Store.Get(requestKubernetesJoinCommand)
+	rawJoinCommand, err := s.Store.Get(keyKubernetesJoinCommand)
 	if err != nil {
 		return nil, err
 	}
@@ -268,91 +280,167 @@ func (s StoreWrapper) PutKubernetesJoinArgs(args *kubeadm.BootstrapTokenDiscover
 	if err != nil {
 		return err
 	}
-	return s.Store.Put(requestKubernetesJoinCommand, j)
+	return s.Store.Put(keyKubernetesJoinCommand, j)
 }
 
 // GetKubernetesConfig returns the Kubernetes kubeconfig file to authenticate with the kubernetes API.
 func (s StoreWrapper) GetKubernetesConfig() ([]byte, error) {
-	return s.Store.Get(requestKubeConfig)
+	return s.Store.Get(keyKubeConfig)
 }
 
 // PutKubernetesConfig saves the Kubernetes kubeconfig file command to store.
 func (s StoreWrapper) PutKubernetesConfig(kubeConfig []byte) error {
-	return s.Store.Put(requestKubeConfig, kubeConfig)
+	return s.Store.Put(keyKubeConfig, kubeConfig)
 }
 
 // GetMasterSecret returns the Constellation master secret from store.
 func (s StoreWrapper) GetMasterSecret() ([]byte, error) {
-	return s.Store.Get(requestMasterSecret)
+	return s.Store.Get(keyMasterSecret)
 }
 
 // PutMasterSecret saves the Constellation master secret to store.
 func (s StoreWrapper) PutMasterSecret(masterSecret []byte) error {
-	return s.Store.Put(requestMasterSecret, masterSecret)
+	return s.Store.Put(keyMasterSecret, masterSecret)
 }
 
 // GetKEKID returns the key encryption key ID from store.
 func (s StoreWrapper) GetKEKID() (string, error) {
-	kekID, err := s.Store.Get(requestKEKID)
+	kekID, err := s.Store.Get(keyKEKID)
 	return string(kekID), err
 }
 
 // PutKEKID saves the key encryption key ID to store.
 func (s StoreWrapper) PutKEKID(kekID string) error {
-	return s.Store.Put(requestKEKID, []byte(kekID))
+	return s.Store.Put(keyKEKID, []byte(kekID))
 }
 
 // GetClusterID returns the unique identifier of the cluster from store.
 func (s StoreWrapper) GetClusterID() ([]byte, error) {
-	return s.Store.Get(requestClusterID)
+	return s.Store.Get(keyClusterID)
 }
 
 // PutClusterID saves the unique identifier of the cluster to store.
 func (s StoreWrapper) PutClusterID(clusterID []byte) error {
-	return s.Store.Put(requestClusterID, clusterID)
+	return s.Store.Put(keyClusterID, clusterID)
 }
 
-// PutLastNodeIP puts the last used ip to the store.
+func (s StoreWrapper) InitializeStoreIPs() error {
-func (s StoreWrapper) PutLastNodeIP(ip []byte) error {
+	if err := s.PutNextNodeIP(nodeIPRangeStart); err != nil {
-	return s.Store.Put(lastNodeIP, ip)
+		return err
+	}
+	return s.PutNextCoordinatorIP(coordinatorIPRangeStart)
 }
 
-// GetLastNodeIP gets the last used ip from the store.
+// PutNextCoordinatorIP puts the last used ip into the store.
-func (s StoreWrapper) GetLastNodeIP() ([]byte, error) {
+func (s StoreWrapper) PutNextCoordinatorIP(ip netip.Addr) error {
-	return s.Store.Get(lastNodeIP)
+	return s.Store.Put(keyHighestAvailableCoordinatorIP, ip.AsSlice())
 }
 
-// generateNextNodeIP generates addresses from a /16 subnet.
+// getNextCoordinatorIP generates addresses from a /16 subnet.
-func (s StoreWrapper) generateNextNodeIP() (string, error) {
+func (s StoreWrapper) getNextCoordinatorIP() (netip.Addr, error) {
-	ip, err := s.GetLastNodeIP()
+	byteIP, err := s.Store.Get(keyHighestAvailableCoordinatorIP)
 	if err != nil {
-		return "", fmt.Errorf("could not obtain IP from store")
+		return netip.Addr{}, errors.New("could not obtain IP from store")
 	}
-	if ip[3] < 255 && ip[2] < 255 || ip[3] < 254 {
+	ip, ok := netip.AddrFromSlice(byteIP)
-		ip[3]++
+	if !ok {
-	} else if ip[2] < 255 {
+		return netip.Addr{}, fmt.Errorf("ip addr malformed %v", byteIP)
-		ip[3] = 0
-		ip[2]++
-	} else {
-		return "", fmt.Errorf("no IPs left to assign")
 	}
+	if !ip.IsValid() || ip.Compare(coordinatorIPRangeEnd) == 1 {
+		return netip.Addr{}, errors.New("no ips left to assign")
+	}
+	nextIP := ip.Next()
+	if err := s.PutNextCoordinatorIP(nextIP); err != nil {
+		return netip.Addr{}, errors.New("could not put IP to store")
+	}
+	return ip, nil
+}
 
-	err = s.PutLastNodeIP(ip)
+// PopNextFreeCoordinatorIP return the next free IP, these could be a old one from a removed peer
-	if err != nil {
+// or a newly generated IP.
-		return "", fmt.Errorf("could not put IP to store")
+func (s StoreWrapper) PopNextFreeCoordinatorIP() (netip.Addr, error) {
+	vpnIP, err := s.getFreedVPNIP(prefixFreeCoordinatorIPs)
+	var noElementsError *store.NoElementsLeftError
+	if errors.As(err, &noElementsError) {
+		return s.getNextCoordinatorIP()
 	}
-	return net.IPv4(ip[0], ip[1], ip[2], ip[3]).String(), nil
+	if err != nil {
+		return netip.Addr{}, err
+	}
+	return vpnIP, nil
+}
+
+// PutFreedCoordinatorVPNIP puts a already generated VPNIP (IP < highestAvailableCoordinatorIP ),
+// which is currently unused, into the store.
+// The IP is saved at a specific prefix and will be used with priority when we
+// request a new Coordinator IP.
+func (s StoreWrapper) PutFreedCoordinatorVPNIP(vpnIP string) error {
+	return s.Store.Put(prefixFreeCoordinatorIPs+vpnIP, nil)
+}
+
+// PutNextNodeIP puts the last used ip into the store.
+func (s StoreWrapper) PutNextNodeIP(ip netip.Addr) error {
+	return s.Store.Put(keyHighestAvailableNodeIP, ip.AsSlice())
+}
+
+// getNextNodeIP generates addresses from a /16 subnet.
+func (s StoreWrapper) getNextNodeIP() (netip.Addr, error) {
+	byteIP, err := s.Store.Get(keyHighestAvailableNodeIP)
+	if err != nil {
+		return netip.Addr{}, errors.New("could not obtain IP from store")
+	}
+	ip, ok := netip.AddrFromSlice(byteIP)
+	if !ok {
+		return netip.Addr{}, fmt.Errorf("ip addr malformed %v", byteIP)
+	}
+	if !ip.IsValid() || ip.Compare(nodeIPRangeEnd) == 1 {
+		return netip.Addr{}, errors.New("no ips left to assign")
+	}
+	nextIP := ip.Next()
+
+	if err := s.PutNextNodeIP(nextIP); err != nil {
+		return netip.Addr{}, errors.New("could not put IP to store")
+	}
+	return ip, nil
 }
 
 // PopNextFreeNodeIP return the next free IP, these could be a old one from a removed peer
 // or a newly generated IP.
-func (s StoreWrapper) PopNextFreeNodeIP() (string, error) {
+func (s StoreWrapper) PopNextFreeNodeIP() (netip.Addr, error) {
-	vpnIP, err := s.GetFreedNodeVPNIP()
+	vpnIP, err := s.getFreedVPNIP(prefixFreeNodeIPs)
-	if err != nil {
+	var noElementsError *store.NoElementsLeftError
-		return "", err
+	if errors.As(err, &noElementsError) {
+		return s.getNextNodeIP()
 	}
-	if len(vpnIP) == 0 {
+	if err != nil {
-		return s.generateNextNodeIP()
+		return netip.Addr{}, err
 	}
 	return vpnIP, nil
 }
+
+// PutFreedNodeVPNIP puts a already generated VPNIP (IP < highestAvailableNodeIP ),
+// which is currently unused, into the store.
+// The IP is saved at a specific prefix and will be used with priority when we
+// request a new Node IP.
+func (s StoreWrapper) PutFreedNodeVPNIP(vpnIP string) error {
+	return s.Store.Put(prefixFreeNodeIPs+vpnIP, nil)
+}
+
+// getFreedVPNIP reclaims a VPNIP from the store and removes it from there.
+func (s StoreWrapper) getFreedVPNIP(prefix string) (netip.Addr, error) {
+	iter, err := s.Store.Iterator(prefix)
+	if err != nil {
+		return netip.Addr{}, err
+	}
+	vpnIPWithPrefix, err := iter.GetNext()
+	if err != nil {
+		return netip.Addr{}, err
+	}
+	stringVPNIP := strings.TrimPrefix(vpnIPWithPrefix, prefix)
+	vpnIP, err := netip.ParseAddr(stringVPNIP)
+	if err != nil {
+		return netip.Addr{}, fmt.Errorf("ip addr malformed %v, %w", stringVPNIP, err)
+	}
+
+	return vpnIP, s.Store.Delete(vpnIPWithPrefix)
+}

coordinator/storewrapper/storewrapper_test.go

@@ -1,6 +1,7 @@
 package storewrapper
 
 import (
+	"net/netip"
 	"testing"
 
 	"github.com/edgelesssys/constellation/coordinator/peer"
@@ -142,96 +143,76 @@ func TestStoreWrapperPeerInterface(t *testing.T) {
 	assert.Error(stwrapper.PutPeer(invalidPeer))
 }
 
-func TestStoreWrapperGetVPNIP(t *testing.T) {
+func TestGenerateNextNodeIP(t *testing.T) {
 	assert := assert.New(t)
 	require := require.New(t)
 
 	stor := store.NewStdStore()
 	stwrapper := StoreWrapper{Store: stor}
-	require.NoError(stwrapper.PutFreedNodeVPNIP("203.0.113.1"))
+	require.NoError(stwrapper.PutNextNodeIP(netip.AddrFrom4([4]byte{10, 118, 0, 11})))
-	require.NoError(stwrapper.PutFreedNodeVPNIP("203.0.113.2"))
-	ipsInStore := map[string]struct{}{
-		"203.0.113.1": {},
-		"203.0.113.2": {},
-	}
 
-	ip, err := stwrapper.GetFreedNodeVPNIP()
+	ip, err := stwrapper.getNextNodeIP()
-	require.NoError(err)
-	assert.Contains(ipsInStore, ip)
-	delete(ipsInStore, ip)
-
-	ip, err = stwrapper.GetFreedNodeVPNIP()
-	require.NoError(err)
-	assert.Contains(ipsInStore, ip)
-	delete(ipsInStore, ip)
-
-	ip, err = stwrapper.GetFreedNodeVPNIP()
 	assert.NoError(err)
-	assert.Len(ip, 0)
+	assert.Equal(netip.AddrFrom4([4]byte{10, 118, 0, 11}), ip)
-}
 
-func TestGenerateNextIP(t *testing.T) {
+	ip, err = stwrapper.getNextNodeIP()
-	assert := assert.New(t)
-	require := require.New(t)
-
-	stor := store.NewStdStore()
-	stwrapper := StoreWrapper{Store: stor}
-	require.NoError(stwrapper.PutLastNodeIP([]byte{10, 118, 0, 1}))
-
-	ip, err := stwrapper.generateNextNodeIP()
 	assert.NoError(err)
-	assert.Equal(ip, "10.118.0.2")
+	assert.Equal(netip.AddrFrom4([4]byte{10, 118, 0, 12}), ip)
 
-	ip, err = stwrapper.generateNextNodeIP()
+	ip, err = stwrapper.getNextNodeIP()
 	assert.NoError(err)
-	assert.Equal(ip, "10.118.0.3")
+	assert.Equal(netip.AddrFrom4([4]byte{10, 118, 0, 13}), ip)
 
-	for i := 0; i < 256*256-7; i++ {
+	for i := 0; i < 256*256-17; i++ {
-		ip, err = stwrapper.generateNextNodeIP()
+		ip, err = stwrapper.getNextNodeIP()
 		assert.NoError(err)
 		assert.NotEmpty(ip)
 	}
 
-	ip, err = stwrapper.generateNextNodeIP()
+	ip, err = stwrapper.getNextNodeIP()
 	assert.NoError(err)
-	assert.Equal(ip, "10.118.255.253")
+	assert.Equal(netip.AddrFrom4([4]byte{10, 118, 255, 253}), ip)
 
-	ip, err = stwrapper.generateNextNodeIP()
+	ip, err = stwrapper.getNextNodeIP()
 	assert.NoError(err)
-	assert.Equal(ip, "10.118.255.254")
+	assert.Equal(netip.AddrFrom4([4]byte{10, 118, 255, 254}), ip)
 
 	// 10.118.255.255 (broadcast IP) should not be returned
-	ip, err = stwrapper.generateNextNodeIP()
+	ip, err = stwrapper.getNextNodeIP()
 	assert.Error(err)
 	assert.Empty(ip)
 
 	// error should still persist
-	ip, err = stwrapper.generateNextNodeIP()
+	ip, err = stwrapper.getNextNodeIP()
 	assert.Error(err)
 	assert.Empty(ip)
 }
 
-func TestPopNextFreeIP(t *testing.T) {
+func TestPopNextFreeNodeIP(t *testing.T) {
 	assert := assert.New(t)
 	require := require.New(t)
 
 	stor := store.NewStdStore()
 	stwrapper := StoreWrapper{Store: stor}
-	require.NoError(stwrapper.PutLastNodeIP([]byte{10, 118, 0, 1}))
+	require.NoError(stwrapper.PutNextNodeIP(netip.AddrFrom4([4]byte{10, 118, 0, 11})))
 
 	ip, err := stwrapper.PopNextFreeNodeIP()
 	assert.NoError(err)
-	assert.Equal("10.118.0.2", ip)
+	assert.Equal(netip.AddrFrom4([4]byte{10, 118, 0, 11}), ip)
 
 	ip, err = stwrapper.PopNextFreeNodeIP()
 	assert.NoError(err)
-	assert.Equal("10.118.0.3", ip)
+	assert.Equal(netip.AddrFrom4([4]byte{10, 118, 0, 12}), ip)
 
-	require.NoError(stwrapper.PutFreedNodeVPNIP("10.118.0.3"))
+	ip, err = stwrapper.PopNextFreeNodeIP()
-	require.NoError(stwrapper.PutFreedNodeVPNIP("10.118.0.2"))
+	assert.NoError(err)
-	ipsInStore := map[string]struct{}{
+	assert.Equal(netip.AddrFrom4([4]byte{10, 118, 0, 13}), ip)
-		"10.118.0.3": {},
+
-		"10.118.0.2": {},
+	require.NoError(stwrapper.PutFreedNodeVPNIP("10.118.0.13"))
+	require.NoError(stwrapper.PutFreedNodeVPNIP("10.118.0.12"))
+	ipsInStore := map[netip.Addr]struct{}{
+		netip.AddrFrom4([4]byte{10, 118, 0, 12}): {},
+		netip.AddrFrom4([4]byte{10, 118, 0, 13}): {},
 	}
 
 	ip, err = stwrapper.PopNextFreeNodeIP()
@@ -246,5 +227,122 @@ func TestPopNextFreeIP(t *testing.T) {
 
 	ip, err = stwrapper.PopNextFreeNodeIP()
 	assert.NoError(err)
-	assert.Equal("10.118.0.4", ip)
+	assert.Equal(netip.AddrFrom4([4]byte{10, 118, 0, 14}), ip)
+}
+
+func TestGenerateNextCoordinatorIP(t *testing.T) {
+	assert := assert.New(t)
+	require := require.New(t)
+
+	stor := store.NewStdStore()
+	stwrapper := StoreWrapper{Store: stor}
+	require.NoError(stwrapper.PutNextCoordinatorIP(netip.AddrFrom4([4]byte{10, 118, 0, 1})))
+
+	ip, err := stwrapper.getNextCoordinatorIP()
+	assert.NoError(err)
+	assert.Equal(netip.AddrFrom4([4]byte{10, 118, 0, 1}), ip)
+
+	ip, err = stwrapper.getNextCoordinatorIP()
+	assert.NoError(err)
+	assert.Equal(netip.AddrFrom4([4]byte{10, 118, 0, 2}), ip)
+
+	ip, err = stwrapper.getNextCoordinatorIP()
+	assert.NoError(err)
+	assert.Equal(netip.AddrFrom4([4]byte{10, 118, 0, 3}), ip)
+
+	for i := 0; i < 7; i++ {
+		ip, err = stwrapper.getNextCoordinatorIP()
+		assert.NoError(err)
+		assert.NotEmpty(ip)
+	}
+
+	// 10.118.0.11 (first Node IP) should not be returned
+	ip, err = stwrapper.getNextCoordinatorIP()
+	assert.Error(err)
+	assert.Empty(ip)
+
+	// error should still persist
+	ip, err = stwrapper.getNextCoordinatorIP()
+	assert.Error(err)
+	assert.Empty(ip)
+}
+
+func TestPopNextFreeCoordinatorIP(t *testing.T) {
+	assert := assert.New(t)
+	require := require.New(t)
+
+	stor := store.NewStdStore()
+	stwrapper := StoreWrapper{Store: stor}
+	require.NoError(stwrapper.PutNextCoordinatorIP(netip.AddrFrom4([4]byte{10, 118, 0, 1})))
+
+	ip, err := stwrapper.PopNextFreeCoordinatorIP()
+	assert.NoError(err)
+	assert.Equal(netip.AddrFrom4([4]byte{10, 118, 0, 1}), ip)
+
+	ip, err = stwrapper.PopNextFreeCoordinatorIP()
+	assert.NoError(err)
+	assert.Equal(netip.AddrFrom4([4]byte{10, 118, 0, 2}), ip)
+
+	ip, err = stwrapper.PopNextFreeCoordinatorIP()
+	assert.NoError(err)
+	assert.Equal(netip.AddrFrom4([4]byte{10, 118, 0, 3}), ip)
+
+	for i := 0; i < 7; i++ {
+		_, err = stwrapper.PopNextFreeCoordinatorIP()
+		require.NoError(err)
+	}
+
+	ip, err = stwrapper.PopNextFreeCoordinatorIP()
+	assert.Error(err)
+	assert.Empty(ip)
+
+	require.NoError(stwrapper.PutFreedCoordinatorVPNIP("10.118.0.3"))
+	require.NoError(stwrapper.PutFreedCoordinatorVPNIP("10.118.0.2"))
+	ipsInStore := map[netip.Addr]struct{}{
+		netip.AddrFrom4([4]byte{10, 118, 0, 3}): {},
+		netip.AddrFrom4([4]byte{10, 118, 0, 2}): {},
+	}
+
+	ip, err = stwrapper.PopNextFreeCoordinatorIP()
+	assert.NoError(err)
+	assert.Contains(ipsInStore, ip)
+	delete(ipsInStore, ip)
+
+	ip, err = stwrapper.PopNextFreeCoordinatorIP()
+	assert.NoError(err)
+	assert.Contains(ipsInStore, ip)
+	delete(ipsInStore, ip)
+
+	ip, err = stwrapper.PopNextFreeCoordinatorIP()
+	assert.Error(err)
+	assert.Equal(netip.Addr{}, ip)
+}
+
+func TestGetFreedVPNIP(t *testing.T) {
+	assert := assert.New(t)
+	require := require.New(t)
+
+	stor := store.NewStdStore()
+	stwrapper := StoreWrapper{Store: stor}
+	require.NoError(stwrapper.PutFreedCoordinatorVPNIP("203.0.113.1"))
+	require.NoError(stwrapper.PutFreedCoordinatorVPNIP("203.0.113.2"))
+	ipsInStore := map[netip.Addr]struct{}{
+		netip.AddrFrom4([4]byte{203, 0, 113, 1}): {},
+		netip.AddrFrom4([4]byte{203, 0, 113, 2}): {},
+	}
+
+	ip, err := stwrapper.getFreedVPNIP(prefixFreeCoordinatorIPs)
+	require.NoError(err)
+	assert.Contains(ipsInStore, ip)
+	delete(ipsInStore, ip)
+
+	ip, err = stwrapper.getFreedVPNIP(prefixFreeCoordinatorIPs)
+	require.NoError(err)
+	assert.Contains(ipsInStore, ip)
+	delete(ipsInStore, ip)
+
+	ip, err = stwrapper.getFreedVPNIP(prefixFreeCoordinatorIPs)
+	var noElementsError *store.NoElementsLeftError
+	assert.ErrorAs(err, &noElementsError)
+	assert.Equal(netip.Addr{}, ip)
 }