Git-Mirrors/constellation
1
0
Fork 0
mirror of https://github.com/edgelesssys/constellation.git synced 2025-08-03 04:26:20 -04:00
Code Issues Projects Releases Packages Wiki Activity

AB#2439 Containerized libvirt (#191)

Browse source 
Signed-off-by: Daniel Weiße <dw@edgeless.systems>
This commit is contained in:
Daniel Weiße 2022-10-05 09:11:30 +02:00 • committed by GitHub
parent abe40de3e5
commit 2ea695896f
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
20 changed files with 746 additions and 50 deletions
Download patch file Download diff file Expand all files Collapse all files

5
cli/internal/cloudcmd/clients.go
View file

@ -32,3 +32,8 @@ type azureclient interface {
CreateInstances(ctx context.Context, input azurecl.CreateInstancesInput) error
TerminateResourceGroupResources(ctx context.Context) error
}
type libvirtRunner interface {
Start(ctx context.Context, containerName, imageName string) error
Stop(ctx context.Context) error
}

17
cli/internal/cloudcmd/clients_test.go
View file

@ -225,3 +225,20 @@ func (c *stubTerraformClient) CleanUpWorkspace() error {
func (c *stubTerraformClient) RemoveInstaller() {
c.removeInstallerCalled = true
}
type stubLibvirtRunner struct {
startCalled bool
stopCalled bool
startErr error
stopErr error
}
func (r *stubLibvirtRunner) Start(_ context.Context, _, _ string) error {
r.startCalled = true
return r.startErr
}
func (r *stubLibvirtRunner) Stop(context.Context) error {
r.stopCalled = true
return r.stopErr
}

66
cli/internal/cloudcmd/create.go
View file

@ -10,9 +10,13 @@ import (
"context"
"fmt"
"io"
"net/url"
"os"
"runtime"
"strings"
azurecl "github.com/edgelesssys/constellation/v2/cli/internal/azure/client"
"github.com/edgelesssys/constellation/v2/cli/internal/libvirt"
"github.com/edgelesssys/constellation/v2/cli/internal/terraform"
"github.com/edgelesssys/constellation/v2/internal/cloud/cloudprovider"
"github.com/edgelesssys/constellation/v2/internal/cloud/cloudtypes"
@ -26,6 +30,7 @@ type Creator struct {
out io.Writer
newTerraformClient func(ctx context.Context, provider cloudprovider.Provider) (terraformClient, error)
newAzureClient func(subscriptionID, tenantID, name, location, resourceGroup string) (azureclient, error)
newLibvirtRunner func() libvirtRunner
}
// NewCreator creates a new creator.
@ -38,6 +43,9 @@ func NewCreator(out io.Writer) *Creator {
newAzureClient: func(subscriptionID, tenantID, name, location, resourceGroup string) (azureclient, error) {
return azurecl.NewInitialized(subscriptionID, tenantID, name, location, resourceGroup)
},
newLibvirtRunner: func() libvirtRunner {
return libvirt.New()
},
}
}
@ -81,7 +89,8 @@ func (c *Creator) Create(ctx context.Context, provider cloudprovider.Provider, c
return state.ConstellationState{}, err
}
defer cl.RemoveInstaller()
return c.createQEMU(ctx, cl, name, config, controlPlaneCount, workerCount)
lv := c.newLibvirtRunner()
return c.createQEMU(ctx, cl, lv, name, config, controlPlaneCount, workerCount)
default:
return state.ConstellationState{}, fmt.Errorf("unsupported cloud provider: %s", provider)
}
@ -153,10 +162,48 @@ func (c *Creator) createAzure(ctx context.Context, cl azureclient, config *confi
return cl.GetState(), nil
}
func (c *Creator) createQEMU(ctx context.Context, cl terraformClient, name string, config *config.Config,
func (c *Creator) createQEMU(ctx context.Context, cl terraformClient, lv libvirtRunner, name string, config *config.Config,
controlPlaneCount, workerCount int,
) (stat state.ConstellationState, retErr error) {
defer rollbackOnError(context.Background(), c.out, &retErr, &rollbackerTerraform{client: cl})
defer rollbackOnError(context.Background(), c.out, &retErr, &rollbackerQEMU{client: cl, libvirt: lv})
libvirtURI := config.Provider.QEMU.LibvirtURI
libvirtSocketPath := "."
switch {
// if no libvirt URI is specified, start a libvirt container
case libvirtURI == "":
if err := lv.Start(ctx, name, config.Provider.QEMU.LibvirtContainerImage); err != nil {
return state.ConstellationState{}, err
}
// non standard port to avoid conflict with host libvirt
// changes here should also be reflected in the Dockerfile in "cli/internal/libvirt/Dockerfile"
libvirtURI = "qemu+tcp://localhost:16599/system"
// socket for system URI should be in /var/run/libvirt/libvirt-sock
case libvirtURI == "qemu:///system":
libvirtSocketPath = "/var/run/libvirt/libvirt-sock"
// socket for session URI should be in /run/user/<uid>/libvirt/libvirt-sock
case libvirtURI == "qemu:///session":
libvirtSocketPath = fmt.Sprintf("/run/user/%d/libvirt/libvirt-sock", os.Getuid())
// if a unix socket is specified we need to parse the URI to get the socket path
case strings.HasPrefix(libvirtURI, "qemu+unix://"):
unixURI, err := url.Parse(strings.TrimPrefix(libvirtURI, "qemu+unix://"))
if err != nil {
return state.ConstellationState{}, err
}
libvirtSocketPath = unixURI.Query().Get("socket")
if libvirtSocketPath == "" {
return state.ConstellationState{}, fmt.Errorf("socket path not specified in qemu+unix URI: %s", libvirtURI)
}
}
metadataLibvirtURI := libvirtURI
if libvirtSocketPath != "." {
metadataLibvirtURI = "qemu:///system"
}
vars := &terraform.QEMUVariables{
CommonVariables: terraform.CommonVariables{
@ -165,11 +212,14 @@ func (c *Creator) createQEMU(ctx context.Context, cl terraformClient, name strin
CountWorkers: workerCount,
StateDiskSizeGB: config.StateDiskSizeGB,
},
ImagePath: config.Provider.QEMU.Image,
ImageFormat: config.Provider.QEMU.ImageFormat,
CPUCount: config.Provider.QEMU.VCPUs,
MemorySizeMiB: config.Provider.QEMU.Memory,
MetadataAPIImage: config.Provider.QEMU.MetadataAPIImage,
LibvirtURI: libvirtURI,
LibvirtSocketPath: libvirtSocketPath,
ImagePath: config.Provider.QEMU.Image,
ImageFormat: config.Provider.QEMU.ImageFormat,
CPUCount: config.Provider.QEMU.VCPUs,
MemorySizeMiB: config.Provider.QEMU.Memory,
MetadataAPIImage: config.Provider.QEMU.MetadataAPIImage,
MetadataLibvirtURI: metadataLibvirtURI,
}
if err := cl.CreateCluster(ctx, name, vars); err != nil {

50
cli/internal/cloudcmd/create_test.go
View file

@ -10,6 +10,7 @@ import (
"bytes"
"context"
"errors"
"runtime"
"testing"
"github.com/edgelesssys/constellation/v2/internal/cloud/cloudprovider"
@ -20,11 +21,18 @@ import (
)
func TestCreator(t *testing.T) {
failOnNonAMD64 := (runtime.GOARCH != "amd64") || (runtime.GOOS != "linux")
wantGCPState := state.ConstellationState{
CloudProvider: cloudprovider.GCP.String(),
LoadBalancerIP: "192.0.2.1",
}
wantQEMUState := state.ConstellationState{
CloudProvider: cloudprovider.QEMU.String(),
LoadBalancerIP: "192.0.2.1",
}
wantAzureState := state.ConstellationState{
CloudProvider: cloudprovider.Azure.String(),
AzureControlPlaneInstances: cloudtypes.Instances{
@ -49,6 +57,7 @@ func TestCreator(t *testing.T) {
newTfClientErr error
azureclient azureclient
newAzureClientErr error
libvirt *stubLibvirtRunner
provider cloudprovider.Provider
config *config.Config
wantState state.ConstellationState
@ -61,7 +70,7 @@ func TestCreator(t *testing.T) {
config: config.Default(),
wantState: wantGCPState,
},
"gcp newGCPClient error": {
"gcp newTerraformClient error": {
newTfClientErr: someErr,
provider: cloudprovider.GCP,
config: config.Default(),
@ -74,6 +83,37 @@ func TestCreator(t *testing.T) {
wantErr: true,
wantRollback: true,
},
"qemu": {
tfClient: &stubTerraformClient{state: wantQEMUState},
libvirt: &stubLibvirtRunner{},
provider: cloudprovider.QEMU,
config: config.Default(),
wantState: wantQEMUState,
wantErr: failOnNonAMD64,
},
"qemu newTerraformClient error": {
newTfClientErr: someErr,
libvirt: &stubLibvirtRunner{},
provider: cloudprovider.QEMU,
config: config.Default(),
wantErr: true,
},
"qemu create cluster error": {
tfClient: &stubTerraformClient{createClusterErr: someErr},
libvirt: &stubLibvirtRunner{},
provider: cloudprovider.QEMU,
config: config.Default(),
wantErr: true,
wantRollback: !failOnNonAMD64, // if we run on non-AMD64/linux, we don't get to a point where rollback is needed
},
"qemu start libvirt error": {
tfClient: &stubTerraformClient{state: wantQEMUState},
libvirt: &stubLibvirtRunner{startErr: someErr},
provider: cloudprovider.QEMU,
config: config.Default(),
wantErr: true,
wantRollback: !failOnNonAMD64,
},
"azure": {
azureclient: &fakeAzureClient{},
provider: cloudprovider.Azure,
@ -126,6 +166,9 @@ func TestCreator(t *testing.T) {
newAzureClient: func(subscriptionID, tenantID, name, location, resourceGroup string) (azureclient, error) {
return tc.azureclient, tc.newAzureClientErr
},
newLibvirtRunner: func() libvirtRunner {
return tc.libvirt
},
}
state, err := creator.Create(context.Background(), tc.provider, tc.config, "name", "type", 2, 3)
@ -135,7 +178,10 @@ func TestCreator(t *testing.T) {
if tc.wantRollback {
switch tc.provider {
case cloudprovider.QEMU:
fallthrough
cl := tc.tfClient.(*stubTerraformClient)
assert.True(cl.destroyClusterCalled)
assert.True(cl.cleanUpWorkspaceCalled)
assert.True(tc.libvirt.stopCalled)
case cloudprovider.GCP:
cl := tc.tfClient.(*stubTerraformClient)
assert.True(cl.destroyClusterCalled)

13
cli/internal/cloudcmd/rollback.go
View file

@ -52,3 +52,16 @@ func (r *rollbackerTerraform) rollback(ctx context.Context) error {
err = multierr.Append(err, r.client.CleanUpWorkspace())
return err
}
type rollbackerQEMU struct {
client terraformClient
libvirt libvirtRunner
}
func (r *rollbackerQEMU) rollback(ctx context.Context) error {
var err error
err = multierr.Append(err, r.client.DestroyCluster(ctx))
err = multierr.Append(err, r.libvirt.Stop(ctx))
err = multierr.Append(err, r.client.CleanUpWorkspace())
return err
}

26
cli/internal/cloudcmd/terminate.go
View file

@ -11,6 +11,7 @@ import (
"fmt"
azurecl "github.com/edgelesssys/constellation/v2/cli/internal/azure/client"
"github.com/edgelesssys/constellation/v2/cli/internal/libvirt"
"github.com/edgelesssys/constellation/v2/cli/internal/terraform"
"github.com/edgelesssys/constellation/v2/internal/cloud/cloudprovider"
"github.com/edgelesssys/constellation/v2/internal/state"
@ -20,6 +21,7 @@ import (
type Terminator struct {
newTerraformClient func(ctx context.Context) (terraformClient, error)
newAzureClient func(subscriptionID, tenantID string) (azureclient, error)
newLibvirtRunner func() libvirtRunner
}
// NewTerminator create a new cloud terminator.
@ -31,6 +33,9 @@ func NewTerminator() *Terminator {
newAzureClient: func(subscriptionID, tenantID string) (azureclient, error) {
return azurecl.NewFromDefault(subscriptionID, tenantID)
},
newLibvirtRunner: func() libvirtRunner {
return libvirt.New()
},
}
}
@ -45,14 +50,20 @@ func (t *Terminator) Terminate(ctx context.Context, state state.ConstellationSta
}
return t.terminateAzure(ctx, cl, state)
case cloudprovider.GCP:
fallthrough
case cloudprovider.QEMU:
cl, err := t.newTerraformClient(ctx)
if err != nil {
return err
}
defer cl.RemoveInstaller()
return t.terminateTerraform(ctx, cl)
case cloudprovider.QEMU:
cl, err := t.newTerraformClient(ctx)
if err != nil {
return err
}
defer cl.RemoveInstaller()
libvirt := t.newLibvirtRunner()
return t.terminateQEMU(ctx, cl, libvirt)
default:
return fmt.Errorf("unsupported provider: %s", provider)
}
@ -68,6 +79,15 @@ func (t *Terminator) terminateTerraform(ctx context.Context, cl terraformClient)
if err := cl.DestroyCluster(ctx); err != nil {
return err
}
return cl.CleanUpWorkspace()
}
func (t *Terminator) terminateQEMU(ctx context.Context, cl terraformClient, lv libvirtRunner) error {
if err := cl.DestroyCluster(ctx); err != nil {
return err
}
if err := lv.Stop(ctx); err != nil {
return err
}
return cl.CleanUpWorkspace()
}

18
cli/internal/cloudcmd/terminate_test.go
View file

@ -38,6 +38,7 @@ func TestTerminator(t *testing.T) {
newTfClientErr error
azureclient azureclient
newAzureClientErr error
libvirt *stubLibvirtRunner
state state.ConstellationState
wantErr bool
}{
@ -62,15 +63,24 @@ func TestTerminator(t *testing.T) {
},
"qemu": {
tfClient: &stubTerraformClient{},
libvirt: &stubLibvirtRunner{},
state: state.ConstellationState{CloudProvider: cloudprovider.QEMU.String()},
},
"qemu destroy cluster error": {
tfClient: &stubTerraformClient{destroyClusterErr: someErr},
libvirt: &stubLibvirtRunner{},
state: state.ConstellationState{CloudProvider: cloudprovider.QEMU.String()},
wantErr: true,
},
"qemu clean up workspace error": {
tfClient: &stubTerraformClient{cleanUpWorkspaceErr: someErr},
libvirt: &stubLibvirtRunner{},
state: state.ConstellationState{CloudProvider: cloudprovider.QEMU.String()},
wantErr: true,
},
"qemu stop libvirt error": {
tfClient: &stubTerraformClient{},
libvirt: &stubLibvirtRunner{stopErr: someErr},
state: state.ConstellationState{CloudProvider: cloudprovider.QEMU.String()},
wantErr: true,
},
@ -105,6 +115,9 @@ func TestTerminator(t *testing.T) {
newAzureClient: func(subscriptionID, tenantID string) (azureclient, error) {
return tc.azureclient, tc.newAzureClientErr
},
newLibvirtRunner: func() libvirtRunner {
return tc.libvirt
},
}
err := terminator.Terminate(context.Background(), tc.state)
@ -114,9 +127,10 @@ func TestTerminator(t *testing.T) {
} else {
assert.NoError(err)
switch cloudprovider.FromString(tc.state.CloudProvider) {
case cloudprovider.GCP:
fallthrough
case cloudprovider.QEMU:
assert.True(tc.libvirt.stopCalled)
fallthrough
case cloudprovider.GCP:
cl := tc.tfClient.(*stubTerraformClient)
assert.True(cl.destroyClusterCalled)
assert.True(cl.removeInstallerCalled)