AB#2114 Add QEMU metadata API (#237)

* Add QEMU metadata API

* API server is started automatically when using terraform to deploy a QEMU cluster

* Enable QEMU metadata usage for disk-mapper, debugd and the Coordinator

Signed-off-by: Daniel Weiße <dw@edgeless.systems>

hack/qemu-metadata-api/Dockerfile

@@ -0,0 +1,31 @@
+FROM fedora:36 as build
+
+RUN dnf -y update && \
+    dnf -y install libvirt-devel @development-tools pkg-config wget git && \
+    dnf clean all
+
+ARG GO_VER=1.18.3
+RUN wget https://go.dev/dl/go${GO_VER}.linux-amd64.tar.gz && \
+    tar -C /usr/local -xzf go${GO_VER}.linux-amd64.tar.gz && \
+    rm go${GO_VER}.linux-amd64.tar.gz
+ENV PATH ${PATH}:/usr/local/go/bin
+
+WORKDIR /qemu-metadata-api
+COPY go.mod ./
+COPY go.sum ./
+RUN go mod download all
+
+WORKDIR /qemu-metadata-api/hack
+COPY hack/go.mod ./go.mod
+COPY hack/go.sum ./go.sum
+RUN go mod download all
+
+COPY . /qemu-metadata-api
+WORKDIR /qemu-metadata-api/hack/qemu-metadata-api
+RUN go build -o api-server ./main.go
+
+FROM fedora:36 as release
+RUN  dnf -y install libvirt-devel && \
+    dnf clean all
+COPY --from=build /qemu-metadata-api/hack/qemu-metadata-api/api-server /server
+ENTRYPOINT [ "/server" ]

hack/qemu-metadata-api/README.md

@@ -0,0 +1,50 @@
+# QEMU metadata API
+
+This program provides a metadata API for Constellation on QEMU.
+
+## Dependencies
+
+To interact with QEMU `libvirt` is required.
+Install the C libraries:
+
+On Ubuntu:
+
+```shell
+sudo apt install libvirt-dev
+```
+
+On Fedora:
+
+```shell
+sudo dnf install libvirt-devel
+```
+
+## Firewalld
+
+If your system uses `firewalld` virtmanager will add itself to the firewall rules managed by `firewalld`.
+Your VMs might be unable to communicate with the host.
+
+To fix this open port `8080` (the default port for the QEMU metadata API) for the `libvirt` zone:
+
+```shell
+# Open the port
+sudo firewall-cmd --zone libvirt --add-port 8080/tcp --permanent
+```
+
+## Docker image
+
+Build the image:
+
+```shell
+DOCKER_BUILDKIT=1 docker build -t ghcr.io/edgelesssys/constellation/qemu-metadata-api:latest -f hack/qemu-metadata-api/Dockerfile .
+```
+
+A container of the image is automatically started by Terraform.
+You can also run the image manually using the following command:
+
+```shell
+docker run -it --rm \
+    --network host \
+    -v /var/run/libvirt/libvirt-sock:/var/run/libvirt/libvirt-sock \
+    ghcr.io/edgelesssys/constellation/qemu-metadata-api:latest
+```

hack/qemu-metadata-api/main.go

@@ -0,0 +1,30 @@
+package main
+
+import (
+	"flag"
+
+	"github.com/edgelesssys/constellation/hack/qemu-metadata-api/server"
+	"github.com/edgelesssys/constellation/hack/qemu-metadata-api/virtwrapper"
+	"github.com/edgelesssys/constellation/internal/logger"
+	"go.uber.org/zap"
+	"go.uber.org/zap/zapcore"
+	"libvirt.org/go/libvirt"
+)
+
+func main() {
+	bindPort := flag.String("port", "8080", "Port to bind to")
+	flag.Parse()
+
+	log := logger.New(logger.JSONLog, zapcore.InfoLevel)
+
+	conn, err := libvirt.NewConnect("qemu:///system")
+	if err != nil {
+		log.With(zap.Error(err)).Fatalf("Failed to connect to libvirt")
+	}
+	defer conn.Close()
+
+	serv := server.New(log, &virtwrapper.Connect{Conn: conn})
+	if err := serv.ListenAndServe(*bindPort); err != nil {
+		log.With(zap.Error(err)).Fatalf("Failed to serve")
+	}
+}

hack/qemu-metadata-api/server/server.go

@@ -0,0 +1,138 @@
+package server
+
+import (
+	"encoding/json"
+	"fmt"
+	"net"
+	"net/http"
+	"strings"
+
+	"github.com/edgelesssys/constellation/coordinator/cloudprovider/cloudtypes"
+	"github.com/edgelesssys/constellation/coordinator/role"
+	"github.com/edgelesssys/constellation/hack/qemu-metadata-api/virtwrapper"
+	"github.com/edgelesssys/constellation/internal/logger"
+	"go.uber.org/zap"
+)
+
+type Server struct {
+	log  *logger.Logger
+	virt virConnect
+}
+
+func New(log *logger.Logger, conn virConnect) *Server {
+	return &Server{
+		log:  log,
+		virt: conn,
+	}
+}
+
+func (s *Server) ListenAndServe(port string) error {
+	mux := http.NewServeMux()
+	mux.Handle("/self", http.HandlerFunc(s.listSelf))
+	mux.Handle("/peers", http.HandlerFunc(s.listPeers))
+
+	server := http.Server{
+		Handler: mux,
+	}
+
+	lis, err := net.Listen("tcp", net.JoinHostPort("", port))
+	if err != nil {
+		return err
+	}
+
+	s.log.Infof("Starting QEMU metadata API on %s", lis.Addr())
+	return server.Serve(lis)
+}
+
+// listSelf returns peer information about the instance issuing the request.
+func (s *Server) listSelf(w http.ResponseWriter, r *http.Request) {
+	log := s.log.With(zap.String("peer", r.RemoteAddr))
+	log.Infof("Serving GET request for /self")
+
+	remoteIP, _, err := net.SplitHostPort(r.RemoteAddr)
+	if err != nil {
+		log.With(zap.Error(err)).Errorf("Failed to parse remote address")
+		http.Error(w, fmt.Sprintf("Failed to parse remote address: %s\n", err), http.StatusInternalServerError)
+		return
+	}
+
+	peers, err := s.listAll()
+	if err != nil {
+		log.With(zap.Error(err)).Errorf("Failed to list peer metadata")
+		http.Error(w, err.Error(), http.StatusInternalServerError)
+		return
+	}
+
+	for _, peer := range peers {
+		for _, ip := range peer.PublicIPs {
+			if ip == remoteIP {
+				w.Header().Set("Content-Type", "application/json")
+				if err := json.NewEncoder(w).Encode(peer); err != nil {
+					http.Error(w, err.Error(), http.StatusInternalServerError)
+					return
+				}
+				log.Infof("Request successful")
+				return
+			}
+		}
+	}
+
+	log.Errorf("Failed to find peer in active leases")
+	http.Error(w, "No matching peer found", http.StatusNotFound)
+}
+
+// listPeers returns a list of all active peers.
+func (s *Server) listPeers(w http.ResponseWriter, r *http.Request) {
+	log := s.log.With(zap.String("peer", r.RemoteAddr))
+	log.Infof("Serving GET request for /peers")
+
+	peers, err := s.listAll()
+	if err != nil {
+		log.With(zap.Error(err)).Errorf("Failed to list peer metadata")
+		http.Error(w, err.Error(), http.StatusInternalServerError)
+		return
+	}
+
+	w.Header().Set("Content-Type", "application/json")
+	if err := json.NewEncoder(w).Encode(peers); err != nil {
+		http.Error(w, err.Error(), http.StatusInternalServerError)
+		return
+	}
+	log.Infof("Request successful")
+}
+
+// listAll returns a list of all active peers.
+func (s *Server) listAll() ([]cloudtypes.Instance, error) {
+	net, err := s.virt.LookupNetworkByName("constellation")
+	if err != nil {
+		return nil, err
+	}
+	defer net.Free()
+
+	leases, err := net.GetDHCPLeases()
+	if err != nil {
+		return nil, err
+	}
+	var peers []cloudtypes.Instance
+
+	for _, lease := range leases {
+		instanceRole := role.Node
+		if strings.HasPrefix(lease.Hostname, "control-plane") {
+			instanceRole = role.Coordinator
+		}
+
+		peers = append(peers, cloudtypes.Instance{
+			Name:       lease.Hostname,
+			Role:       instanceRole,
+			PrivateIPs: []string{lease.IPaddr},
+			PublicIPs:  []string{lease.IPaddr},
+			ProviderID: "qemu:///hostname/" + lease.Hostname,
+		})
+	}
+
+	return peers, nil
+}
+
+type virConnect interface {
+	LookupNetworkByName(name string) (*virtwrapper.Network, error)
+}

hack/qemu-metadata-api/server/server_test.go

@@ -0,0 +1,249 @@
+package server
+
+import (
+	"encoding/json"
+	"errors"
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+
+	"github.com/edgelesssys/constellation/coordinator/cloudprovider/cloudtypes"
+	"github.com/edgelesssys/constellation/hack/qemu-metadata-api/virtwrapper"
+	"github.com/edgelesssys/constellation/internal/logger"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"libvirt.org/go/libvirt"
+)
+
+func TestListAll(t *testing.T) {
+	someErr := errors.New("error")
+
+	testCases := map[string]struct {
+		wantErr bool
+		connect *stubConnect
+	}{
+		"success": {
+			connect: &stubConnect{
+				network: stubNetwork{
+					leases: []libvirt.NetworkDHCPLease{
+						{
+							IPaddr:   "192.0.100.1",
+							Hostname: "control-plane-0",
+						},
+						{
+							IPaddr:   "192.0.100.2",
+							Hostname: "control-plane-1",
+						},
+						{
+							IPaddr:   "192.0.200.1",
+							Hostname: "worker-0",
+						},
+					},
+				},
+			},
+		},
+		"LookupNetworkByName error": {
+			connect: &stubConnect{
+				getNetworkErr: someErr,
+			},
+			wantErr: true,
+		},
+		"GetDHCPLeases error": {
+			connect: &stubConnect{
+				network: stubNetwork{
+					getLeaseErr: someErr,
+				},
+			},
+			wantErr: true,
+		},
+	}
+
+	for name, tc := range testCases {
+		t.Run(name, func(t *testing.T) {
+			assert := assert.New(t)
+
+			server := New(logger.NewTest(t), tc.connect)
+
+			res, err := server.listAll()
+
+			if tc.wantErr {
+				assert.Error(err)
+				return
+			}
+			assert.NoError(err)
+			assert.Len(tc.connect.network.leases, len(res))
+		})
+	}
+}
+
+func TestListSelf(t *testing.T) {
+	someErr := errors.New("error")
+
+	testCases := map[string]struct {
+		remoteAddr string
+		connect    *stubConnect
+		wantErr    bool
+	}{
+		"success": {
+			remoteAddr: "192.0.100.1:1234",
+			connect: &stubConnect{
+				network: stubNetwork{
+					leases: []libvirt.NetworkDHCPLease{
+						{
+							IPaddr:   "192.0.100.1",
+							Hostname: "control-plane-0",
+						},
+					},
+				},
+			},
+		},
+		"listAll error": {
+			remoteAddr: "192.0.100.1:1234",
+			connect: &stubConnect{
+				getNetworkErr: someErr,
+			},
+			wantErr: true,
+		},
+		"remoteAddr error": {
+			remoteAddr: "",
+			connect: &stubConnect{
+				network: stubNetwork{
+					leases: []libvirt.NetworkDHCPLease{
+						{
+							IPaddr:   "192.0.100.1",
+							Hostname: "control-plane-0",
+						},
+					},
+				},
+			},
+			wantErr: true,
+		},
+		"peer not found": {
+			remoteAddr: "192.0.200.1:1234",
+			connect: &stubConnect{
+				network: stubNetwork{
+					leases: []libvirt.NetworkDHCPLease{
+						{
+							IPaddr:   "192.0.100.1",
+							Hostname: "control-plane-0",
+						},
+					},
+				},
+			},
+			wantErr: true,
+		},
+	}
+
+	for name, tc := range testCases {
+		t.Run(name, func(t *testing.T) {
+			assert := assert.New(t)
+			require := require.New(t)
+
+			server := New(logger.NewTest(t), tc.connect)
+
+			req, err := http.NewRequest(http.MethodGet, "http://192.0.0.1/self", nil)
+			require.NoError(err)
+			req.RemoteAddr = tc.remoteAddr
+
+			w := httptest.NewRecorder()
+			server.listSelf(w, req)
+
+			if tc.wantErr {
+				assert.NotEqual(http.StatusOK, w.Code)
+				return
+			}
+			assert.Equal(http.StatusOK, w.Code)
+			metadataRaw, err := io.ReadAll(w.Body)
+			require.NoError(err)
+
+			var metadata cloudtypes.Instance
+			require.NoError(json.Unmarshal(metadataRaw, &metadata))
+			assert.Equal(tc.connect.network.leases[0].Hostname, metadata.Name)
+			assert.Equal(tc.connect.network.leases[0].IPaddr, metadata.PublicIPs[0])
+		})
+	}
+}
+
+func TestListPeers(t *testing.T) {
+	testCases := map[string]struct {
+		remoteAddr string
+		connect    *stubConnect
+		wantErr    bool
+	}{
+		"success": {
+			remoteAddr: "192.0.100.1:1234",
+			connect: &stubConnect{
+				network: stubNetwork{
+					leases: []libvirt.NetworkDHCPLease{
+						{
+							IPaddr:   "192.0.100.1",
+							Hostname: "control-plane-0",
+						},
+						{
+							IPaddr:   "192.0.200.1",
+							Hostname: "worker-0",
+						},
+					},
+				},
+			},
+		},
+		"listAll error": {
+			remoteAddr: "192.0.100.1:1234",
+			connect: &stubConnect{
+				getNetworkErr: errors.New("error"),
+			},
+			wantErr: true,
+		},
+	}
+
+	for name, tc := range testCases {
+		t.Run(name, func(t *testing.T) {
+			assert := assert.New(t)
+			require := require.New(t)
+
+			server := New(logger.NewTest(t), tc.connect)
+
+			req, err := http.NewRequest(http.MethodGet, "http://192.0.0.1/peers", nil)
+			require.NoError(err)
+			req.RemoteAddr = tc.remoteAddr
+
+			w := httptest.NewRecorder()
+			server.listPeers(w, req)
+
+			if tc.wantErr {
+				assert.NotEqual(http.StatusOK, w.Code)
+				return
+			}
+			assert.Equal(http.StatusOK, w.Code)
+			metadataRaw, err := io.ReadAll(w.Body)
+			require.NoError(err)
+
+			var metadata []cloudtypes.Instance
+			require.NoError(json.Unmarshal(metadataRaw, &metadata))
+			assert.Len(metadata, len(tc.connect.network.leases))
+		})
+	}
+}
+
+type stubConnect struct {
+	network       stubNetwork
+	getNetworkErr error
+}
+
+func (c stubConnect) LookupNetworkByName(name string) (*virtwrapper.Network, error) {
+	return &virtwrapper.Network{Net: c.network}, c.getNetworkErr
+}
+
+type stubNetwork struct {
+	leases      []libvirt.NetworkDHCPLease
+	getLeaseErr error
+}
+
+func (n stubNetwork) GetDHCPLeases() ([]libvirt.NetworkDHCPLease, error) {
+	return n.leases, n.getLeaseErr
+}
+
+func (n stubNetwork) Free() error {
+	return nil
+}

hack/qemu-metadata-api/virtwrapper/virtwrapper.go

@@ -0,0 +1,32 @@
+package virtwrapper
+
+import "libvirt.org/go/libvirt"
+
+type Connect struct {
+	Conn *libvirt.Connect
+}
+
+func (c *Connect) LookupNetworkByName(name string) (*Network, error) {
+	net, err := c.Conn.LookupNetworkByName(name)
+	if err != nil {
+		return nil, err
+	}
+	return &Network{Net: net}, nil
+}
+
+type Network struct {
+	Net virNetwork
+}
+
+func (n *Network) GetDHCPLeases() ([]libvirt.NetworkDHCPLease, error) {
+	return n.Net.GetDHCPLeases()
+}
+
+func (n *Network) Free() {
+	_ = n.Net.Free()
+}
+
+type virNetwork interface {
+	GetDHCPLeases() ([]libvirt.NetworkDHCPLease, error)
+	Free() error
+}