AB#2114 Add QEMU metadata API (#237)

* Add QEMU metadata API

* API server is started automatically when using terraform to deploy a QEMU cluster

* Enable QEMU metadata usage for disk-mapper, debugd and the Coordinator

Signed-off-by: Daniel Weiße <dw@edgeless.systems>
This commit is contained in:
Daniel Weiße 2022-06-30 11:14:26 +02:00 committed by GitHub
parent b0aafd0c2a
commit 040e498b42
17 changed files with 648 additions and 23 deletions

View File

@ -21,7 +21,7 @@ jobs:
go-version: 1.18 go-version: 1.18
- name: Install Dependencies - name: Install Dependencies
run: sudo apt-get update && sudo apt-get install -y pkg-config libcryptsetup12 libcryptsetup-dev run: sudo apt-get update && sudo apt-get install -y pkg-config libcryptsetup12 libcryptsetup-dev libvirt-dev
- name: Test main module - name: Test main module
run: go test -race -count=3 ./... run: go test -race -count=3 ./...

View File

@ -2,42 +2,73 @@ package qemu
import ( import (
"context" "context"
"encoding/json"
"errors"
"io"
"net/http"
"net/url"
"github.com/edgelesssys/constellation/coordinator/cloudprovider/cloudtypes" "github.com/edgelesssys/constellation/coordinator/cloudprovider/cloudtypes"
"github.com/edgelesssys/constellation/coordinator/role" "github.com/edgelesssys/constellation/coordinator/role"
) )
// Metadata implements core.ProviderMetadata interface for QEMU (currently not supported). const qemuMetadataEndpoint = "10.42.0.1:8080"
// Metadata implements core.ProviderMetadata interface for QEMU.
type Metadata struct{} type Metadata struct{}
// Supported is used to determine if metadata API is implemented for this cloud provider. // Supported is used to determine if metadata API is implemented for this cloud provider.
func (m *Metadata) Supported() bool { func (m *Metadata) Supported() bool {
return false return true
} }
// List retrieves all instances belonging to the current constellation. // List retrieves all instances belonging to the current constellation.
func (m *Metadata) List(ctx context.Context) ([]cloudtypes.Instance, error) { func (m *Metadata) List(ctx context.Context) ([]cloudtypes.Instance, error) {
panic("function *Metadata.List not implemented") instancesRaw, err := m.retrieveMetadata(ctx, "/peers")
if err != nil {
return nil, err
}
var instances []cloudtypes.Instance
err = json.Unmarshal(instancesRaw, &instances)
return instances, err
} }
// Self retrieves the current instance. // Self retrieves the current instance.
func (m *Metadata) Self(ctx context.Context) (cloudtypes.Instance, error) { func (m *Metadata) Self(ctx context.Context) (cloudtypes.Instance, error) {
panic("function *Metdata.Self not implemented") instanceRaw, err := m.retrieveMetadata(ctx, "/self")
if err != nil {
return cloudtypes.Instance{}, err
}
var instance cloudtypes.Instance
err = json.Unmarshal(instanceRaw, &instance)
return instance, err
} }
// GetInstance retrieves an instance using its providerID. // GetInstance retrieves an instance using its providerID.
func (m Metadata) GetInstance(ctx context.Context, providerID string) (cloudtypes.Instance, error) { func (m Metadata) GetInstance(ctx context.Context, providerID string) (cloudtypes.Instance, error) {
panic("function *Metadata.GetInstance not implemented") instances, err := m.List(ctx)
if err != nil {
return cloudtypes.Instance{}, err
}
for _, instance := range instances {
if instance.ProviderID == providerID {
return instance, nil
}
}
return cloudtypes.Instance{}, errors.New("instance not found")
} }
// SignalRole signals the constellation role via cloud provider metadata (if supported by the CSP and deployment type, otherwise does nothing). // SignalRole signals the constellation role via cloud provider metadata (if supported by the CSP and deployment type, otherwise does nothing).
func (m Metadata) SignalRole(ctx context.Context, role role.Role) error { func (m Metadata) SignalRole(ctx context.Context, role role.Role) error {
panic("function *Metadata.SignalRole not implemented") return nil
} }
// SetVPNIP stores the internally used VPN IP in cloud provider metadata (if supported and required for autoscaling by the CSP, otherwise does nothing). // SetVPNIP stores the internally used VPN IP in cloud provider metadata (if supported and required for autoscaling by the CSP, otherwise does nothing).
func (m Metadata) SetVPNIP(ctx context.Context, vpnIP string) error { func (m Metadata) SetVPNIP(ctx context.Context, vpnIP string) error {
panic("function *Metadata.SetVPNIP not implemented") return nil
} }
// SupportsLoadBalancer returns true if the cloud provider supports load balancers. // SupportsLoadBalancer returns true if the cloud provider supports load balancers.
@ -52,5 +83,25 @@ func (m Metadata) GetLoadBalancerIP(ctx context.Context) (string, error) {
// GetSubnetworkCIDR retrieves the subnetwork CIDR from cloud provider metadata. // GetSubnetworkCIDR retrieves the subnetwork CIDR from cloud provider metadata.
func (m Metadata) GetSubnetworkCIDR(ctx context.Context) (string, error) { func (m Metadata) GetSubnetworkCIDR(ctx context.Context) (string, error) {
panic("function *Metadata.GetSubnetworkCIDR not implemented") return "10.244.0.0/16", nil
}
func (m Metadata) retrieveMetadata(ctx context.Context, uri string) ([]byte, error) {
url := &url.URL{
Scheme: "http",
Host: qemuMetadataEndpoint,
Path: uri,
}
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url.String(), nil)
if err != nil {
return nil, err
}
res, err := (&http.Client{}).Do(req)
if err != nil {
return nil, err
}
defer res.Body.Close()
return io.ReadAll(res.Body)
} }

View File

@ -162,9 +162,10 @@ func main() {
issuer = qemu.NewIssuer() issuer = qemu.NewIssuer()
validator = qemu.NewValidator(pcrs) validator = qemu.NewValidator(pcrs)
// no support for cloud services in qemu // no support for cloud logging in qemu
metadata := &qemucloud.Metadata{}
cloudLogger = &logging.NopLogger{} cloudLogger = &logging.NopLogger{}
metadata := &qemucloud.Metadata{}
pcrsJSON, err := json.Marshal(pcrs) pcrsJSON, err := json.Marshal(pcrs)
if err != nil { if err != nil {
log.Fatal(err) log.Fatal(err)

View File

@ -124,7 +124,7 @@ func (k *KubernetesUtil) SetupPodNetwork(ctx context.Context, in SetupPodNetwork
case "azure": case "azure":
return k.setupAzurePodNetwork(ctx, in.ProviderID, in.SubnetworkPodCIDR) return k.setupAzurePodNetwork(ctx, in.ProviderID, in.SubnetworkPodCIDR)
case "qemu": case "qemu":
return k.setupQemuPodNetwork(ctx) return k.setupQemuPodNetwork(ctx, in.SubnetworkPodCIDR)
default: default:
return fmt.Errorf("unsupported cloud provider %q", in.CloudProvider) return fmt.Errorf("unsupported cloud provider %q", in.CloudProvider)
} }
@ -213,8 +213,8 @@ func (k *KubernetesUtil) FixCilium(nodeNameK8s string) {
} }
} }
func (k *KubernetesUtil) setupQemuPodNetwork(ctx context.Context) error { func (k *KubernetesUtil) setupQemuPodNetwork(ctx context.Context, subnetworkPodCIDR string) error {
ciliumInstall := exec.CommandContext(ctx, "cilium", "install", "--encryption", "wireguard", "--helm-set", "ipam.operator.clusterPoolIPv4PodCIDRList=10.244.0.0/16,endpointRoutes.enabled=true") ciliumInstall := exec.CommandContext(ctx, "cilium", "install", "--encryption", "wireguard", "--helm-set", "ipam.operator.clusterPoolIPv4PodCIDRList="+subnetworkPodCIDR+",endpointRoutes.enabled=true")
ciliumInstall.Env = append(os.Environ(), "KUBECONFIG="+kubeConfig) ciliumInstall.Env = append(os.Environ(), "KUBECONFIG="+kubeConfig)
out, err := ciliumInstall.CombinedOutput() out, err := ciliumInstall.CombinedOutput()
if err != nil { if err != nil {

View File

@ -3,7 +3,6 @@ package main
import ( import (
"net" "net"
"os" "os"
"strings"
"sync" "sync"
"github.com/edgelesssys/constellation/debugd/coordinator" "github.com/edgelesssys/constellation/debugd/coordinator"
@ -12,6 +11,7 @@ import (
"github.com/edgelesssys/constellation/debugd/debugd/metadata/cloudprovider" "github.com/edgelesssys/constellation/debugd/debugd/metadata/cloudprovider"
"github.com/edgelesssys/constellation/debugd/debugd/metadata/fallback" "github.com/edgelesssys/constellation/debugd/debugd/metadata/fallback"
"github.com/edgelesssys/constellation/debugd/debugd/server" "github.com/edgelesssys/constellation/debugd/debugd/server"
platform "github.com/edgelesssys/constellation/internal/cloud/cloudprovider"
"github.com/edgelesssys/constellation/internal/deploy/ssh" "github.com/edgelesssys/constellation/internal/deploy/ssh"
"github.com/edgelesssys/constellation/internal/deploy/user" "github.com/edgelesssys/constellation/internal/deploy/user"
"github.com/edgelesssys/constellation/internal/logger" "github.com/edgelesssys/constellation/internal/logger"
@ -34,22 +34,24 @@ func main() {
download := deploy.New(log.Named("download"), &net.Dialer{}, serviceManager, streamer) download := deploy.New(log.Named("download"), &net.Dialer{}, serviceManager, streamer)
var fetcher metadata.Fetcher var fetcher metadata.Fetcher
constellationCSP := strings.ToLower(os.Getenv("CONSTEL_CSP")) csp := os.Getenv("CONSTEL_CSP")
switch constellationCSP { switch platform.FromString(csp) {
case "azure": case platform.Azure:
azureFetcher, err := cloudprovider.NewAzure(ctx) azureFetcher, err := cloudprovider.NewAzure(ctx)
if err != nil { if err != nil {
panic(err) panic(err)
} }
fetcher = azureFetcher fetcher = azureFetcher
case "gcp": case platform.GCP:
gcpFetcher, err := cloudprovider.NewGCP(ctx) gcpFetcher, err := cloudprovider.NewGCP(ctx)
if err != nil { if err != nil {
panic(err) panic(err)
} }
fetcher = gcpFetcher fetcher = gcpFetcher
case platform.QEMU:
fetcher = cloudprovider.NewQEMU()
default: default:
log.Errorf("Unknown / unimplemented cloud provider CONSTEL_CSP=%v. Using fallback", constellationCSP) log.Errorf("Unknown / unimplemented cloud provider CONSTEL_CSP=%v. Using fallback", csp)
fetcher = fallback.Fetcher{} fetcher = fallback.Fetcher{}
} }
sched := metadata.NewScheduler(log.Named("scheduler"), fetcher, ssh, download) sched := metadata.NewScheduler(log.Named("scheduler"), fetcher, ssh, download)

View File

@ -7,6 +7,7 @@ import (
azurecloud "github.com/edgelesssys/constellation/coordinator/cloudprovider/azure" azurecloud "github.com/edgelesssys/constellation/coordinator/cloudprovider/azure"
"github.com/edgelesssys/constellation/coordinator/cloudprovider/cloudtypes" "github.com/edgelesssys/constellation/coordinator/cloudprovider/cloudtypes"
gcpcloud "github.com/edgelesssys/constellation/coordinator/cloudprovider/gcp" gcpcloud "github.com/edgelesssys/constellation/coordinator/cloudprovider/gcp"
qemucloud "github.com/edgelesssys/constellation/coordinator/cloudprovider/qemu"
"github.com/edgelesssys/constellation/internal/deploy/ssh" "github.com/edgelesssys/constellation/internal/deploy/ssh"
) )
@ -47,6 +48,12 @@ func NewAzure(ctx context.Context) (*Fetcher, error) {
}, nil }, nil
} }
func NewQEMU() *Fetcher {
return &Fetcher{
metaAPI: &qemucloud.Metadata{},
}
}
// DiscoverDebugdIPs will query the metadata of all instances and return any ips of instances already set up for debugging. // DiscoverDebugdIPs will query the metadata of all instances and return any ips of instances already set up for debugging.
func (f *Fetcher) DiscoverDebugdIPs(ctx context.Context) ([]string, error) { func (f *Fetcher) DiscoverDebugdIPs(ctx context.Context) ([]string, error) {
self, err := f.metaAPI.Self(ctx) self, err := f.metaAPI.Self(ctx)

View File

@ -43,8 +43,10 @@ require (
github.com/spf13/afero v1.8.2 github.com/spf13/afero v1.8.2
github.com/spf13/cobra v1.5.0 github.com/spf13/cobra v1.5.0
github.com/stretchr/testify v1.7.1 github.com/stretchr/testify v1.7.1
go.uber.org/zap v1.21.0
google.golang.org/grpc v1.46.2 google.golang.org/grpc v1.46.2
gopkg.in/yaml.v3 v3.0.0-20220512140231-539c8e751b99 gopkg.in/yaml.v3 v3.0.0-20220512140231-539c8e751b99
libvirt.org/go/libvirt v1.8004.0
) )
require ( require (
@ -138,7 +140,6 @@ require (
go.opencensus.io v0.23.0 // indirect go.opencensus.io v0.23.0 // indirect
go.uber.org/atomic v1.9.0 // indirect go.uber.org/atomic v1.9.0 // indirect
go.uber.org/multierr v1.8.0 // indirect go.uber.org/multierr v1.8.0 // indirect
go.uber.org/zap v1.21.0 // indirect
golang.org/x/crypto v0.0.0-20220315160706-3147a52a75dd // indirect golang.org/x/crypto v0.0.0-20220315160706-3147a52a75dd // indirect
golang.org/x/net v0.0.0-20220225172249-27dd8689420f // indirect golang.org/x/net v0.0.0-20220225172249-27dd8689420f // indirect
golang.org/x/oauth2 v0.0.0-20220309155454-6242fa91716a // indirect golang.org/x/oauth2 v0.0.0-20220309155454-6242fa91716a // indirect

View File

@ -1639,6 +1639,8 @@ honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWh
honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
libvirt.org/go/libvirt v1.8004.0 h1:SKa5hQNKQfc1VjU4LqLMorqPCxC1lplnz8LwLiMrPyM=
libvirt.org/go/libvirt v1.8004.0/go.mod h1:1WiFE8EjZfq+FCVog+rvr1yatKbKZ9FaFMZgEqxEJqQ=
pack.ag/amqp v0.11.2/go.mod h1:4/cbmt4EJXSKlG6LCfWHoqmN0uFdy5i/+YFz+fTfhV4= pack.ag/amqp v0.11.2/go.mod h1:4/cbmt4EJXSKlG6LCfWHoqmN0uFdy5i/+YFz+fTfhV4=
rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=

View File

@ -0,0 +1,31 @@
FROM fedora:36 as build
RUN dnf -y update && \
dnf -y install libvirt-devel @development-tools pkg-config wget git && \
dnf clean all
ARG GO_VER=1.18.3
RUN wget https://go.dev/dl/go${GO_VER}.linux-amd64.tar.gz && \
tar -C /usr/local -xzf go${GO_VER}.linux-amd64.tar.gz && \
rm go${GO_VER}.linux-amd64.tar.gz
ENV PATH ${PATH}:/usr/local/go/bin
WORKDIR /qemu-metadata-api
COPY go.mod ./
COPY go.sum ./
RUN go mod download all
WORKDIR /qemu-metadata-api/hack
COPY hack/go.mod ./go.mod
COPY hack/go.sum ./go.sum
RUN go mod download all
COPY . /qemu-metadata-api
WORKDIR /qemu-metadata-api/hack/qemu-metadata-api
RUN go build -o api-server ./main.go
FROM fedora:36 as release
RUN dnf -y install libvirt-devel && \
dnf clean all
COPY --from=build /qemu-metadata-api/hack/qemu-metadata-api/api-server /server
ENTRYPOINT [ "/server" ]

View File

@ -0,0 +1,50 @@
# QEMU metadata API
This program provides a metadata API for Constellation on QEMU.
## Dependencies
To interact with QEMU `libvirt` is required.
Install the C libraries:
On Ubuntu:
```shell
sudo apt install libvirt-dev
```
On Fedora:
```shell
sudo dnf install libvirt-devel
```
## Firewalld
If your system uses `firewalld` virtmanager will add itself to the firewall rules managed by `firewalld`.
Your VMs might be unable to communicate with the host.
To fix this open port `8080` (the default port for the QEMU metadata API) for the `libvirt` zone:
```shell
# Open the port
sudo firewall-cmd --zone libvirt --add-port 8080/tcp --permanent
```
## Docker image
Build the image:
```shell
DOCKER_BUILDKIT=1 docker build -t ghcr.io/edgelesssys/constellation/qemu-metadata-api:latest -f hack/qemu-metadata-api/Dockerfile .
```
A container of the image is automatically started by Terraform.
You can also run the image manually using the following command:
```shell
docker run -it --rm \
--network host \
-v /var/run/libvirt/libvirt-sock:/var/run/libvirt/libvirt-sock \
ghcr.io/edgelesssys/constellation/qemu-metadata-api:latest
```

View File

@ -0,0 +1,30 @@
package main
import (
"flag"
"github.com/edgelesssys/constellation/hack/qemu-metadata-api/server"
"github.com/edgelesssys/constellation/hack/qemu-metadata-api/virtwrapper"
"github.com/edgelesssys/constellation/internal/logger"
"go.uber.org/zap"
"go.uber.org/zap/zapcore"
"libvirt.org/go/libvirt"
)
func main() {
bindPort := flag.String("port", "8080", "Port to bind to")
flag.Parse()
log := logger.New(logger.JSONLog, zapcore.InfoLevel)
conn, err := libvirt.NewConnect("qemu:///system")
if err != nil {
log.With(zap.Error(err)).Fatalf("Failed to connect to libvirt")
}
defer conn.Close()
serv := server.New(log, &virtwrapper.Connect{Conn: conn})
if err := serv.ListenAndServe(*bindPort); err != nil {
log.With(zap.Error(err)).Fatalf("Failed to serve")
}
}

View File

@ -0,0 +1,138 @@
package server
import (
"encoding/json"
"fmt"
"net"
"net/http"
"strings"
"github.com/edgelesssys/constellation/coordinator/cloudprovider/cloudtypes"
"github.com/edgelesssys/constellation/coordinator/role"
"github.com/edgelesssys/constellation/hack/qemu-metadata-api/virtwrapper"
"github.com/edgelesssys/constellation/internal/logger"
"go.uber.org/zap"
)
type Server struct {
log *logger.Logger
virt virConnect
}
func New(log *logger.Logger, conn virConnect) *Server {
return &Server{
log: log,
virt: conn,
}
}
func (s *Server) ListenAndServe(port string) error {
mux := http.NewServeMux()
mux.Handle("/self", http.HandlerFunc(s.listSelf))
mux.Handle("/peers", http.HandlerFunc(s.listPeers))
server := http.Server{
Handler: mux,
}
lis, err := net.Listen("tcp", net.JoinHostPort("", port))
if err != nil {
return err
}
s.log.Infof("Starting QEMU metadata API on %s", lis.Addr())
return server.Serve(lis)
}
// listSelf returns peer information about the instance issuing the request.
func (s *Server) listSelf(w http.ResponseWriter, r *http.Request) {
log := s.log.With(zap.String("peer", r.RemoteAddr))
log.Infof("Serving GET request for /self")
remoteIP, _, err := net.SplitHostPort(r.RemoteAddr)
if err != nil {
log.With(zap.Error(err)).Errorf("Failed to parse remote address")
http.Error(w, fmt.Sprintf("Failed to parse remote address: %s\n", err), http.StatusInternalServerError)
return
}
peers, err := s.listAll()
if err != nil {
log.With(zap.Error(err)).Errorf("Failed to list peer metadata")
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
for _, peer := range peers {
for _, ip := range peer.PublicIPs {
if ip == remoteIP {
w.Header().Set("Content-Type", "application/json")
if err := json.NewEncoder(w).Encode(peer); err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
log.Infof("Request successful")
return
}
}
}
log.Errorf("Failed to find peer in active leases")
http.Error(w, "No matching peer found", http.StatusNotFound)
}
// listPeers returns a list of all active peers.
func (s *Server) listPeers(w http.ResponseWriter, r *http.Request) {
log := s.log.With(zap.String("peer", r.RemoteAddr))
log.Infof("Serving GET request for /peers")
peers, err := s.listAll()
if err != nil {
log.With(zap.Error(err)).Errorf("Failed to list peer metadata")
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
w.Header().Set("Content-Type", "application/json")
if err := json.NewEncoder(w).Encode(peers); err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
log.Infof("Request successful")
}
// listAll returns a list of all active peers.
func (s *Server) listAll() ([]cloudtypes.Instance, error) {
net, err := s.virt.LookupNetworkByName("constellation")
if err != nil {
return nil, err
}
defer net.Free()
leases, err := net.GetDHCPLeases()
if err != nil {
return nil, err
}
var peers []cloudtypes.Instance
for _, lease := range leases {
instanceRole := role.Node
if strings.HasPrefix(lease.Hostname, "control-plane") {
instanceRole = role.Coordinator
}
peers = append(peers, cloudtypes.Instance{
Name: lease.Hostname,
Role: instanceRole,
PrivateIPs: []string{lease.IPaddr},
PublicIPs: []string{lease.IPaddr},
ProviderID: "qemu:///hostname/" + lease.Hostname,
})
}
return peers, nil
}
type virConnect interface {
LookupNetworkByName(name string) (*virtwrapper.Network, error)
}

View File

@ -0,0 +1,249 @@
package server
import (
"encoding/json"
"errors"
"io"
"net/http"
"net/http/httptest"
"testing"
"github.com/edgelesssys/constellation/coordinator/cloudprovider/cloudtypes"
"github.com/edgelesssys/constellation/hack/qemu-metadata-api/virtwrapper"
"github.com/edgelesssys/constellation/internal/logger"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"libvirt.org/go/libvirt"
)
func TestListAll(t *testing.T) {
someErr := errors.New("error")
testCases := map[string]struct {
wantErr bool
connect *stubConnect
}{
"success": {
connect: &stubConnect{
network: stubNetwork{
leases: []libvirt.NetworkDHCPLease{
{
IPaddr: "192.0.100.1",
Hostname: "control-plane-0",
},
{
IPaddr: "192.0.100.2",
Hostname: "control-plane-1",
},
{
IPaddr: "192.0.200.1",
Hostname: "worker-0",
},
},
},
},
},
"LookupNetworkByName error": {
connect: &stubConnect{
getNetworkErr: someErr,
},
wantErr: true,
},
"GetDHCPLeases error": {
connect: &stubConnect{
network: stubNetwork{
getLeaseErr: someErr,
},
},
wantErr: true,
},
}
for name, tc := range testCases {
t.Run(name, func(t *testing.T) {
assert := assert.New(t)
server := New(logger.NewTest(t), tc.connect)
res, err := server.listAll()
if tc.wantErr {
assert.Error(err)
return
}
assert.NoError(err)
assert.Len(tc.connect.network.leases, len(res))
})
}
}
func TestListSelf(t *testing.T) {
someErr := errors.New("error")
testCases := map[string]struct {
remoteAddr string
connect *stubConnect
wantErr bool
}{
"success": {
remoteAddr: "192.0.100.1:1234",
connect: &stubConnect{
network: stubNetwork{
leases: []libvirt.NetworkDHCPLease{
{
IPaddr: "192.0.100.1",
Hostname: "control-plane-0",
},
},
},
},
},
"listAll error": {
remoteAddr: "192.0.100.1:1234",
connect: &stubConnect{
getNetworkErr: someErr,
},
wantErr: true,
},
"remoteAddr error": {
remoteAddr: "",
connect: &stubConnect{
network: stubNetwork{
leases: []libvirt.NetworkDHCPLease{
{
IPaddr: "192.0.100.1",
Hostname: "control-plane-0",
},
},
},
},
wantErr: true,
},
"peer not found": {
remoteAddr: "192.0.200.1:1234",
connect: &stubConnect{
network: stubNetwork{
leases: []libvirt.NetworkDHCPLease{
{
IPaddr: "192.0.100.1",
Hostname: "control-plane-0",
},
},
},
},
wantErr: true,
},
}
for name, tc := range testCases {
t.Run(name, func(t *testing.T) {
assert := assert.New(t)
require := require.New(t)
server := New(logger.NewTest(t), tc.connect)
req, err := http.NewRequest(http.MethodGet, "http://192.0.0.1/self", nil)
require.NoError(err)
req.RemoteAddr = tc.remoteAddr
w := httptest.NewRecorder()
server.listSelf(w, req)
if tc.wantErr {
assert.NotEqual(http.StatusOK, w.Code)
return
}
assert.Equal(http.StatusOK, w.Code)
metadataRaw, err := io.ReadAll(w.Body)
require.NoError(err)
var metadata cloudtypes.Instance
require.NoError(json.Unmarshal(metadataRaw, &metadata))
assert.Equal(tc.connect.network.leases[0].Hostname, metadata.Name)
assert.Equal(tc.connect.network.leases[0].IPaddr, metadata.PublicIPs[0])
})
}
}
func TestListPeers(t *testing.T) {
testCases := map[string]struct {
remoteAddr string
connect *stubConnect
wantErr bool
}{
"success": {
remoteAddr: "192.0.100.1:1234",
connect: &stubConnect{
network: stubNetwork{
leases: []libvirt.NetworkDHCPLease{
{
IPaddr: "192.0.100.1",
Hostname: "control-plane-0",
},
{
IPaddr: "192.0.200.1",
Hostname: "worker-0",
},
},
},
},
},
"listAll error": {
remoteAddr: "192.0.100.1:1234",
connect: &stubConnect{
getNetworkErr: errors.New("error"),
},
wantErr: true,
},
}
for name, tc := range testCases {
t.Run(name, func(t *testing.T) {
assert := assert.New(t)
require := require.New(t)
server := New(logger.NewTest(t), tc.connect)
req, err := http.NewRequest(http.MethodGet, "http://192.0.0.1/peers", nil)
require.NoError(err)
req.RemoteAddr = tc.remoteAddr
w := httptest.NewRecorder()
server.listPeers(w, req)
if tc.wantErr {
assert.NotEqual(http.StatusOK, w.Code)
return
}
assert.Equal(http.StatusOK, w.Code)
metadataRaw, err := io.ReadAll(w.Body)
require.NoError(err)
var metadata []cloudtypes.Instance
require.NoError(json.Unmarshal(metadataRaw, &metadata))
assert.Len(metadata, len(tc.connect.network.leases))
})
}
}
type stubConnect struct {
network stubNetwork
getNetworkErr error
}
func (c stubConnect) LookupNetworkByName(name string) (*virtwrapper.Network, error) {
return &virtwrapper.Network{Net: c.network}, c.getNetworkErr
}
type stubNetwork struct {
leases []libvirt.NetworkDHCPLease
getLeaseErr error
}
func (n stubNetwork) GetDHCPLeases() ([]libvirt.NetworkDHCPLease, error) {
return n.leases, n.getLeaseErr
}
func (n stubNetwork) Free() error {
return nil
}

View File

@ -0,0 +1,32 @@
package virtwrapper
import "libvirt.org/go/libvirt"
type Connect struct {
Conn *libvirt.Connect
}
func (c *Connect) LookupNetworkByName(name string) (*Network, error) {
net, err := c.Conn.LookupNetworkByName(name)
if err != nil {
return nil, err
}
return &Network{Net: net}, nil
}
type Network struct {
Net virNetwork
}
func (n *Network) GetDHCPLeases() ([]libvirt.NetworkDHCPLease, error) {
return n.Net.GetDHCPLeases()
}
func (n *Network) Free() {
_ = n.Net.Free()
}
type virNetwork interface {
GetDHCPLeases() ([]libvirt.NetworkDHCPLease, error)
Free() error
}

View File

@ -10,6 +10,7 @@ import (
azurecloud "github.com/edgelesssys/constellation/coordinator/cloudprovider/azure" azurecloud "github.com/edgelesssys/constellation/coordinator/cloudprovider/azure"
gcpcloud "github.com/edgelesssys/constellation/coordinator/cloudprovider/gcp" gcpcloud "github.com/edgelesssys/constellation/coordinator/cloudprovider/gcp"
qemucloud "github.com/edgelesssys/constellation/coordinator/cloudprovider/qemu"
"github.com/edgelesssys/constellation/coordinator/core" "github.com/edgelesssys/constellation/coordinator/core"
"github.com/edgelesssys/constellation/internal/attestation/azure" "github.com/edgelesssys/constellation/internal/attestation/azure"
"github.com/edgelesssys/constellation/internal/attestation/gcp" "github.com/edgelesssys/constellation/internal/attestation/gcp"
@ -67,8 +68,7 @@ func main() {
case "qemu": case "qemu":
diskPath = qemuStateDiskPath diskPath = qemuStateDiskPath
issuer = qemu.NewIssuer() issuer = qemu.NewIssuer()
log.Warnf("cloud services are not supported on QEMU") metadata = &qemucloud.Metadata{}
metadata = &core.ProviderMetadataFake{}
default: default:
diskPathErr = fmt.Errorf("csp %q is not supported by Constellation", *csp) diskPathErr = fmt.Errorf("csp %q is not supported by Constellation", *csp)

View File

@ -6,6 +6,7 @@ Prerequisite:
- [qcow2 constellation image](/image/) - [qcow2 constellation image](/image/)
- [setup](#setup-libvirt--terraform) - [setup](#setup-libvirt--terraform)
- [qemu-metadata-api](/hack/qemu-metadata-api/README.md)
Optional: Write a `terraform.tfvars` file in the terraform workspace (`terraform/libvirt`), defining required variables and overriding optional variables. Optional: Write a `terraform.tfvars` file in the terraform workspace (`terraform/libvirt`), defining required variables and overriding optional variables.
See [variables.tf](./variables.tf) for a description of all available variables. See [variables.tf](./variables.tf) for a description of all available variables.

View File

@ -4,6 +4,10 @@ terraform {
source = "dmacvicar/libvirt" source = "dmacvicar/libvirt"
version = "0.6.14" version = "0.6.14"
} }
docker = {
source = "kreuzwerker/docker"
version = "2.17.0"
}
} }
} }
@ -11,6 +15,32 @@ provider "libvirt" {
uri = "qemu:///session" uri = "qemu:///session"
} }
provider "docker" {
host = "unix:///var/run/docker.sock"
registry_auth {
address = "ghcr.io"
config_file = pathexpand("~/.docker/config.json")
}
}
resource "docker_image" "qemu-metadata" {
name = "ghcr.io/edgelesssys/constellation/qemu-metadata-api:latest"
keep_locally = true
}
resource "docker_container" "qemu-metadata" {
name = "qemu-metadata"
image = docker_image.qemu-metadata.latest
network_mode = "host"
rm = true
mounts {
source = "/var/run/libvirt/libvirt-sock"
target = "/var/run/libvirt/libvirt-sock"
type = "bind"
}
}
module "control_plane" { module "control_plane" {
source = "./modules/instance_group" source = "./modules/instance_group"
role = "control-plane" role = "control-plane"