Git-Mirrors/constellation
1
0
Fork 0
mirror of https://github.com/edgelesssys/constellation.git synced 2025-09-27 20:10:51 -04:00
Code Issues Projects Releases Packages Wiki Activity

AB#2439 Containerized libvirt (#191)

Browse source 
Signed-off-by: Daniel Weiße <dw@edgeless.systems>
This commit is contained in:
Daniel Weiße 2022-10-05 09:11:30 +02:00 • committed by GitHub
parent abe40de3e5
commit 2ea695896f
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
20 changed files with 746 additions and 50 deletions
Download patch file Download diff file Expand all files Collapse all files

5
cli/internal/cloudcmd/clients.go
View file

@ -32,3 +32,8 @@ type azureclient interface {
CreateInstances(ctx context.Context, input azurecl.CreateInstancesInput) error
TerminateResourceGroupResources(ctx context.Context) error
}
type libvirtRunner interface {
Start(ctx context.Context, containerName, imageName string) error
Stop(ctx context.Context) error
}

17
cli/internal/cloudcmd/clients_test.go
View file

@ -225,3 +225,20 @@ func (c *stubTerraformClient) CleanUpWorkspace() error {
func (c *stubTerraformClient) RemoveInstaller() {
c.removeInstallerCalled = true
}
type stubLibvirtRunner struct {
startCalled bool
stopCalled bool
startErr error
stopErr error
}
func (r *stubLibvirtRunner) Start(_ context.Context, _, _ string) error {
r.startCalled = true
return r.startErr
}
func (r *stubLibvirtRunner) Stop(context.Context) error {
r.stopCalled = true
return r.stopErr
}

66
cli/internal/cloudcmd/create.go
View file

@ -10,9 +10,13 @@ import (
"context"
"fmt"
"io"
"net/url"
"os"
"runtime"
"strings"
azurecl "github.com/edgelesssys/constellation/v2/cli/internal/azure/client"
"github.com/edgelesssys/constellation/v2/cli/internal/libvirt"
"github.com/edgelesssys/constellation/v2/cli/internal/terraform"
"github.com/edgelesssys/constellation/v2/internal/cloud/cloudprovider"
"github.com/edgelesssys/constellation/v2/internal/cloud/cloudtypes"
@ -26,6 +30,7 @@ type Creator struct {
out io.Writer
newTerraformClient func(ctx context.Context, provider cloudprovider.Provider) (terraformClient, error)
newAzureClient func(subscriptionID, tenantID, name, location, resourceGroup string) (azureclient, error)
newLibvirtRunner func() libvirtRunner
}
// NewCreator creates a new creator.
@ -38,6 +43,9 @@ func NewCreator(out io.Writer) *Creator {
newAzureClient: func(subscriptionID, tenantID, name, location, resourceGroup string) (azureclient, error) {
return azurecl.NewInitialized(subscriptionID, tenantID, name, location, resourceGroup)
},
newLibvirtRunner: func() libvirtRunner {
return libvirt.New()
},
}
}
@ -81,7 +89,8 @@ func (c *Creator) Create(ctx context.Context, provider cloudprovider.Provider, c
return state.ConstellationState{}, err
}
defer cl.RemoveInstaller()
return c.createQEMU(ctx, cl, name, config, controlPlaneCount, workerCount)
lv := c.newLibvirtRunner()
return c.createQEMU(ctx, cl, lv, name, config, controlPlaneCount, workerCount)
default:
return state.ConstellationState{}, fmt.Errorf("unsupported cloud provider: %s", provider)
}
@ -153,10 +162,48 @@ func (c *Creator) createAzure(ctx context.Context, cl azureclient, config *confi
return cl.GetState(), nil
}
func (c *Creator) createQEMU(ctx context.Context, cl terraformClient, name string, config *config.Config,
func (c *Creator) createQEMU(ctx context.Context, cl terraformClient, lv libvirtRunner, name string, config *config.Config,
controlPlaneCount, workerCount int,
) (stat state.ConstellationState, retErr error) {
defer rollbackOnError(context.Background(), c.out, &retErr, &rollbackerTerraform{client: cl})
defer rollbackOnError(context.Background(), c.out, &retErr, &rollbackerQEMU{client: cl, libvirt: lv})
libvirtURI := config.Provider.QEMU.LibvirtURI
libvirtSocketPath := "."
switch {
// if no libvirt URI is specified, start a libvirt container
case libvirtURI == "":
if err := lv.Start(ctx, name, config.Provider.QEMU.LibvirtContainerImage); err != nil {
return state.ConstellationState{}, err
}
// non standard port to avoid conflict with host libvirt
// changes here should also be reflected in the Dockerfile in "cli/internal/libvirt/Dockerfile"
libvirtURI = "qemu+tcp://localhost:16599/system"
// socket for system URI should be in /var/run/libvirt/libvirt-sock
case libvirtURI == "qemu:///system":
libvirtSocketPath = "/var/run/libvirt/libvirt-sock"
// socket for session URI should be in /run/user/<uid>/libvirt/libvirt-sock
case libvirtURI == "qemu:///session":
libvirtSocketPath = fmt.Sprintf("/run/user/%d/libvirt/libvirt-sock", os.Getuid())
// if a unix socket is specified we need to parse the URI to get the socket path
case strings.HasPrefix(libvirtURI, "qemu+unix://"):
unixURI, err := url.Parse(strings.TrimPrefix(libvirtURI, "qemu+unix://"))
if err != nil {
return state.ConstellationState{}, err
}
libvirtSocketPath = unixURI.Query().Get("socket")
if libvirtSocketPath == "" {
return state.ConstellationState{}, fmt.Errorf("socket path not specified in qemu+unix URI: %s", libvirtURI)
}
}
metadataLibvirtURI := libvirtURI
if libvirtSocketPath != "." {
metadataLibvirtURI = "qemu:///system"
}
vars := &terraform.QEMUVariables{
CommonVariables: terraform.CommonVariables{
@ -165,11 +212,14 @@ func (c *Creator) createQEMU(ctx context.Context, cl terraformClient, name strin
CountWorkers: workerCount,
StateDiskSizeGB: config.StateDiskSizeGB,
},
ImagePath: config.Provider.QEMU.Image,
ImageFormat: config.Provider.QEMU.ImageFormat,
CPUCount: config.Provider.QEMU.VCPUs,
MemorySizeMiB: config.Provider.QEMU.Memory,
MetadataAPIImage: config.Provider.QEMU.MetadataAPIImage,
LibvirtURI: libvirtURI,
LibvirtSocketPath: libvirtSocketPath,
ImagePath: config.Provider.QEMU.Image,
ImageFormat: config.Provider.QEMU.ImageFormat,
CPUCount: config.Provider.QEMU.VCPUs,
MemorySizeMiB: config.Provider.QEMU.Memory,
MetadataAPIImage: config.Provider.QEMU.MetadataAPIImage,
MetadataLibvirtURI: metadataLibvirtURI,
}
if err := cl.CreateCluster(ctx, name, vars); err != nil {

50
cli/internal/cloudcmd/create_test.go
View file

@ -10,6 +10,7 @@ import (
"bytes"
"context"
"errors"
"runtime"
"testing"
"github.com/edgelesssys/constellation/v2/internal/cloud/cloudprovider"
@ -20,11 +21,18 @@ import (
)
func TestCreator(t *testing.T) {
failOnNonAMD64 := (runtime.GOARCH != "amd64") || (runtime.GOOS != "linux")
wantGCPState := state.ConstellationState{
CloudProvider: cloudprovider.GCP.String(),
LoadBalancerIP: "192.0.2.1",
}
wantQEMUState := state.ConstellationState{
CloudProvider: cloudprovider.QEMU.String(),
LoadBalancerIP: "192.0.2.1",
}
wantAzureState := state.ConstellationState{
CloudProvider: cloudprovider.Azure.String(),
AzureControlPlaneInstances: cloudtypes.Instances{
@ -49,6 +57,7 @@ func TestCreator(t *testing.T) {
newTfClientErr error
azureclient azureclient
newAzureClientErr error
libvirt *stubLibvirtRunner
provider cloudprovider.Provider
config *config.Config
wantState state.ConstellationState
@ -61,7 +70,7 @@ func TestCreator(t *testing.T) {
config: config.Default(),
wantState: wantGCPState,
},
"gcp newGCPClient error": {
"gcp newTerraformClient error": {
newTfClientErr: someErr,
provider: cloudprovider.GCP,
config: config.Default(),
@ -74,6 +83,37 @@ func TestCreator(t *testing.T) {
wantErr: true,
wantRollback: true,
},
"qemu": {
tfClient: &stubTerraformClient{state: wantQEMUState},
libvirt: &stubLibvirtRunner{},
provider: cloudprovider.QEMU,
config: config.Default(),
wantState: wantQEMUState,
wantErr: failOnNonAMD64,
},
"qemu newTerraformClient error": {
newTfClientErr: someErr,
libvirt: &stubLibvirtRunner{},
provider: cloudprovider.QEMU,
config: config.Default(),
wantErr: true,
},
"qemu create cluster error": {
tfClient: &stubTerraformClient{createClusterErr: someErr},
libvirt: &stubLibvirtRunner{},
provider: cloudprovider.QEMU,
config: config.Default(),
wantErr: true,
wantRollback: !failOnNonAMD64, // if we run on non-AMD64/linux, we don't get to a point where rollback is needed
},
"qemu start libvirt error": {
tfClient: &stubTerraformClient{state: wantQEMUState},
libvirt: &stubLibvirtRunner{startErr: someErr},
provider: cloudprovider.QEMU,
config: config.Default(),
wantErr: true,
wantRollback: !failOnNonAMD64,
},
"azure": {
azureclient: &fakeAzureClient{},
provider: cloudprovider.Azure,
@ -126,6 +166,9 @@ func TestCreator(t *testing.T) {
newAzureClient: func(subscriptionID, tenantID, name, location, resourceGroup string) (azureclient, error) {
return tc.azureclient, tc.newAzureClientErr
},
newLibvirtRunner: func() libvirtRunner {
return tc.libvirt
},
}
state, err := creator.Create(context.Background(), tc.provider, tc.config, "name", "type", 2, 3)
@ -135,7 +178,10 @@ func TestCreator(t *testing.T) {
if tc.wantRollback {
switch tc.provider {
case cloudprovider.QEMU:
fallthrough
cl := tc.tfClient.(*stubTerraformClient)
assert.True(cl.destroyClusterCalled)
assert.True(cl.cleanUpWorkspaceCalled)
assert.True(tc.libvirt.stopCalled)
case cloudprovider.GCP:
cl := tc.tfClient.(*stubTerraformClient)
assert.True(cl.destroyClusterCalled)

13
cli/internal/cloudcmd/rollback.go
View file

@ -52,3 +52,16 @@ func (r *rollbackerTerraform) rollback(ctx context.Context) error {
err = multierr.Append(err, r.client.CleanUpWorkspace())
return err
}
type rollbackerQEMU struct {
client terraformClient
libvirt libvirtRunner
}
func (r *rollbackerQEMU) rollback(ctx context.Context) error {
var err error
err = multierr.Append(err, r.client.DestroyCluster(ctx))
err = multierr.Append(err, r.libvirt.Stop(ctx))
err = multierr.Append(err, r.client.CleanUpWorkspace())
return err
}

26
cli/internal/cloudcmd/terminate.go
View file

@ -11,6 +11,7 @@ import (
"fmt"
azurecl "github.com/edgelesssys/constellation/v2/cli/internal/azure/client"
"github.com/edgelesssys/constellation/v2/cli/internal/libvirt"
"github.com/edgelesssys/constellation/v2/cli/internal/terraform"
"github.com/edgelesssys/constellation/v2/internal/cloud/cloudprovider"
"github.com/edgelesssys/constellation/v2/internal/state"
@ -20,6 +21,7 @@ import (
type Terminator struct {
newTerraformClient func(ctx context.Context) (terraformClient, error)
newAzureClient func(subscriptionID, tenantID string) (azureclient, error)
newLibvirtRunner func() libvirtRunner
}
// NewTerminator create a new cloud terminator.
@ -31,6 +33,9 @@ func NewTerminator() *Terminator {
newAzureClient: func(subscriptionID, tenantID string) (azureclient, error) {
return azurecl.NewFromDefault(subscriptionID, tenantID)
},
newLibvirtRunner: func() libvirtRunner {
return libvirt.New()
},
}
}
@ -45,14 +50,20 @@ func (t *Terminator) Terminate(ctx context.Context, state state.ConstellationSta
}
return t.terminateAzure(ctx, cl, state)
case cloudprovider.GCP:
fallthrough
case cloudprovider.QEMU:
cl, err := t.newTerraformClient(ctx)
if err != nil {
return err
}
defer cl.RemoveInstaller()
return t.terminateTerraform(ctx, cl)
case cloudprovider.QEMU:
cl, err := t.newTerraformClient(ctx)
if err != nil {
return err
}
defer cl.RemoveInstaller()
libvirt := t.newLibvirtRunner()
return t.terminateQEMU(ctx, cl, libvirt)
default:
return fmt.Errorf("unsupported provider: %s", provider)
}
@ -68,6 +79,15 @@ func (t *Terminator) terminateTerraform(ctx context.Context, cl terraformClient)
if err := cl.DestroyCluster(ctx); err != nil {
return err
}
return cl.CleanUpWorkspace()
}
func (t *Terminator) terminateQEMU(ctx context.Context, cl terraformClient, lv libvirtRunner) error {
if err := cl.DestroyCluster(ctx); err != nil {
return err
}
if err := lv.Stop(ctx); err != nil {
return err
}
return cl.CleanUpWorkspace()
}

18
cli/internal/cloudcmd/terminate_test.go
View file

@ -38,6 +38,7 @@ func TestTerminator(t *testing.T) {
newTfClientErr error
azureclient azureclient
newAzureClientErr error
libvirt *stubLibvirtRunner
state state.ConstellationState
wantErr bool
}{
@ -62,15 +63,24 @@ func TestTerminator(t *testing.T) {
},
"qemu": {
tfClient: &stubTerraformClient{},
libvirt: &stubLibvirtRunner{},
state: state.ConstellationState{CloudProvider: cloudprovider.QEMU.String()},
},
"qemu destroy cluster error": {
tfClient: &stubTerraformClient{destroyClusterErr: someErr},
libvirt: &stubLibvirtRunner{},
state: state.ConstellationState{CloudProvider: cloudprovider.QEMU.String()},
wantErr: true,
},
"qemu clean up workspace error": {
tfClient: &stubTerraformClient{cleanUpWorkspaceErr: someErr},
libvirt: &stubLibvirtRunner{},
state: state.ConstellationState{CloudProvider: cloudprovider.QEMU.String()},
wantErr: true,
},
"qemu stop libvirt error": {
tfClient: &stubTerraformClient{},
libvirt: &stubLibvirtRunner{stopErr: someErr},
state: state.ConstellationState{CloudProvider: cloudprovider.QEMU.String()},
wantErr: true,
},
@ -105,6 +115,9 @@ func TestTerminator(t *testing.T) {
newAzureClient: func(subscriptionID, tenantID string) (azureclient, error) {
return tc.azureclient, tc.newAzureClientErr
},
newLibvirtRunner: func() libvirtRunner {
return tc.libvirt
},
}
err := terminator.Terminate(context.Background(), tc.state)
@ -114,9 +127,10 @@ func TestTerminator(t *testing.T) {
} else {
assert.NoError(err)
switch cloudprovider.FromString(tc.state.CloudProvider) {
case cloudprovider.GCP:
fallthrough
case cloudprovider.QEMU:
assert.True(tc.libvirt.stopCalled)
fallthrough
case cloudprovider.GCP:
cl := tc.tfClient.(*stubTerraformClient)
assert.True(cl.destroyClusterCalled)
assert.True(cl.removeInstallerCalled)

23
cli/internal/libvirt/Dockerfile Normal file
View file

@ -0,0 +1,23 @@
FROM fedora@sha256:486fd5578f93fbc57a519e34ad4b7cac927c3f8a95409baedf0c19e9f287c207 AS deploy
RUN dnf -y update && \
dnf -y install dnf-plugins-core \
libvirt-daemon-config-network \
libvirt-daemon-kvm \
qemu-kvm \
swtpm \
swtpm-tools \
xsltproc \
libvirt-client && \
dnf clean all
# Prevent cgroup issues on Fedora and configure libvirt
RUN echo "cgroup_controllers = []" >> /etc/libvirt/qemu.conf && \
echo "listen_tls = 0" >> /etc/libvirt/libvirtd.conf && \
echo "listen_tcp = 1" >> /etc/libvirt/libvirtd.conf && \
echo "tcp_port = \"16599\"" >> /etc/libvirt/libvirtd.conf && \
echo "listen_addr = \"localhost\"" >> /etc/libvirt/libvirtd.conf && \
echo "auth_tcp = \"none\"" >> /etc/libvirt/libvirtd.conf
COPY --chmod=755 ./cli/internal/libvirt/start.sh /start.sh
ENTRYPOINT ["/start.sh"]

30
cli/internal/libvirt/README.md Normal file
View file

@ -0,0 +1,30 @@
# Containerized libvirt
To avoid dependency issues with the libvirt setup of the host, we provide a containerized libvirt instance.
If no libvirt connection string is provided in the Constellation config file during create,
this container is deployed to provide a libvirt daemon for orchestrating Constellation nodes in QEMU.
The container will listen for libvirt connections on `localhost:16599`.
Connecting to the libvirt daemon running in the container and manage the deployment using `virsh` run the following:
```shell
virsh -c "qemu+tcp://localhost:16599/system"
```
## Docker image
Build the image:
```shell
DOCKER_BUILDKIT=1 docker build -t ghcr.io/edgelesssys/constellation/libvirt:latest -f cli/internal/libvirt/Dockerfile .
```
A container of the image is automatically started by the CLI.
You can also run the image manually using the following command:
```shell
docker run -it --rm \
--network host \
--privileged true \
ghcr.io/edgelesssys/constellation/libvirt:latest
```

97
cli/internal/libvirt/libvirt.go Normal file
View file

@ -0,0 +1,97 @@
/*
Copyright (c) Edgeless Systems GmbH
SPDX-License-Identifier: AGPL-3.0-only
*/
package libvirt
import (
"context"
"errors"
"github.com/docker/docker/api/types"
"github.com/docker/docker/api/types/container"
docker "github.com/docker/docker/client"
"github.com/edgelesssys/constellation/v2/internal/file"
"github.com/spf13/afero"
)
// Runner handles starting and stopping of containerized libvirt instances.
type Runner struct {
nameFile string
file file.Handler
}
// New creates a new LibvirtRunner.
func New() *Runner {
return &Runner{
nameFile: "libvirt.name",
file: file.NewHandler(afero.NewOsFs()),
}
}
// Start starts a containerized libvirt instance.
func (r *Runner) Start(ctx context.Context, name, imageName string) error {
docker, err := docker.NewClientWithOpts(docker.FromEnv, docker.WithAPIVersionNegotiation())
if err != nil {
return err
}
defer docker.Close()
containerName := name + "-libvirt"
if _, err := docker.ContainerCreate(ctx,
&container.Config{
Image: imageName,
},
&container.HostConfig{
NetworkMode: container.NetworkMode("host"),
AutoRemove: true,
// container has to be "privileged" so libvirt has access to proc fs
Privileged: true,
},
nil,
nil,
containerName,
); err != nil {
return err
}
if err := docker.ContainerStart(ctx, containerName, types.ContainerStartOptions{}); err != nil {
_ = docker.ContainerRemove(ctx, containerName, types.ContainerRemoveOptions{Force: true})
return err
}
if err := r.file.Write(r.nameFile, []byte(containerName)); err != nil {
_ = docker.ContainerRemove(ctx, containerName, types.ContainerRemoveOptions{Force: true})
return err
}
// time.Sleep(15 * time.Second)
return nil
}
// Stop stops a containerized libvirt instance.
func (r *Runner) Stop(ctx context.Context) error {
name, err := r.file.Read(r.nameFile)
if err != nil {
if errors.Is(err, afero.ErrFileNotFound) {
return nil
}
return err
}
docker, err := docker.NewClientWithOpts(docker.FromEnv, docker.WithAPIVersionNegotiation())
if err != nil {
return err
}
defer docker.Close()
if err := docker.ContainerRemove(ctx, string(name), types.ContainerRemoveOptions{Force: true}); err != nil {
return err
}
if err := r.file.Remove(r.nameFile); err != nil && !errors.Is(err, afero.ErrFileNotFound) {
return err
}
return nil
}

7
cli/internal/libvirt/start.sh Executable file
View file

@ -0,0 +1,7 @@
#!/bin/bash
# Start libvirt daemon
libvirtd --daemon --listen
virtlogd --daemon
sleep infinity

12
cli/internal/terraform/terraform/qemu/main.tf
View file

@ -12,7 +12,7 @@ terraform {
}
provider "libvirt" {
uri = "qemu:///session"
uri = var.libvirt_uri
}
provider "docker" {
@ -24,22 +24,24 @@ provider "docker" {
}
}
resource "docker_image" "qemu-metadata" {
resource "docker_image" "qemu_metadata" {
name = var.metadata_api_image
keep_locally = true
}
resource "docker_container" "qemu-metadata" {
resource "docker_container" "qemu_metadata" {
name = "${var.name}-qemu-metadata"
image = docker_image.qemu-metadata.latest
image = docker_image.qemu_metadata.latest
network_mode = "host"
rm = true
command = [
"--network",
"${var.name}-network",
"--libvirt-uri",
"${var.metadata_libvirt_uri}",
]
mounts {
source = "/var/run/libvirt/libvirt-sock"
source = abspath(var.libvirt_socket_path)
target = "/var/run/libvirt/libvirt-sock"
type = "bind"
}

15
cli/internal/terraform/terraform/qemu/variables.tf
View file

@ -1,3 +1,8 @@
variable "libvirt_uri" {
type = string
description = "libvirt socket uri"
}
variable "constellation_coreos_image" {
type = string
description = "constellation OS file path"
@ -45,6 +50,16 @@ variable "metadata_api_image" {
description = "container image of the QEMU metadata api server"
}
variable "metadata_libvirt_uri" {
type = string
description = "libvirt uri for the metadata api server"
}
variable "libvirt_socket_path" {
type = string
description = "path to libvirt socket in case of unix socket"
}
variable "name" {
type = string
default = "constellation"

11
cli/internal/terraform/variables.go
View file

@ -83,6 +83,10 @@ type QEMUVariables struct {
// CommonVariables contains common variables.
CommonVariables
// LibvirtURI is the libvirt connection URI.
LibvirtURI string
// LibvirtSocketPath is the path to the libvirt socket in case of unix socket.
LibvirtSocketPath string
// CPUCount is the number of CPUs to allocate to each node.
CPUCount int
// MemorySizeMiB is the amount of memory to allocate to each node, in MiB.
@ -93,17 +97,24 @@ type QEMUVariables struct {
ImageFormat string
// MetadataAPIImage is the container image to use for the metadata API.
MetadataAPIImage string
// MetadataLibvirtURI is the libvirt connection URI used by the metadata container.
// In case of unix socket, this should be "qemu:///system".
// Other wise it should be the same as LibvirtURI.
MetadataLibvirtURI string
}
// String returns a string representation of the variables, formatted as Terraform variables.
func (v *QEMUVariables) String() string {
b := &strings.Builder{}
b.WriteString(v.CommonVariables.String())
writeLinef(b, "libvirt_uri = %q", v.LibvirtURI)
writeLinef(b, "libvirt_socket_path = %q", v.LibvirtSocketPath)
writeLinef(b, "constellation_coreos_image = %q", v.ImagePath)
writeLinef(b, "image_format = %q", v.ImageFormat)
writeLinef(b, "vcpus = %d", v.CPUCount)
writeLinef(b, "memory = %d", v.MemorySizeMiB)
writeLinef(b, "metadata_api_image = %q", v.MetadataAPIImage)
writeLinef(b, "metadata_libvirt_uri = %q", v.MetadataLibvirtURI)
return b.String()
}