diff --git a/CHANGELOG.md b/CHANGELOG.md index d6fcce1f3..671d78005 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,6 +22,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- Mini Constellation: Try out Constellation locally without any cloud subscription required just with one command: `constellation mini up` - Loadbalancer for control-plane recovery - K8s conformance mode - Local cluster creation based on QEMU diff --git a/cli/cmd/root.go b/cli/cmd/root.go index 8b3c74875..415a5a630 100644 --- a/cli/cmd/root.go +++ b/cli/cmd/root.go @@ -44,6 +44,7 @@ func NewRootCmd() *cobra.Command { rootCmd.AddCommand(cmd.NewConfigCmd()) rootCmd.AddCommand(cmd.NewCreateCmd()) rootCmd.AddCommand(cmd.NewInitCmd()) + rootCmd.AddCommand(cmd.NewMiniCmd()) rootCmd.AddCommand(cmd.NewVerifyCmd()) rootCmd.AddCommand(cmd.NewUpgradeCmd()) rootCmd.AddCommand(cmd.NewRecoverCmd()) diff --git a/cli/internal/cloudcmd/create.go b/cli/internal/cloudcmd/create.go index 337ef11ae..2f5355bfb 100644 --- a/cli/internal/cloudcmd/create.go +++ b/cli/internal/cloudcmd/create.go @@ -176,9 +176,7 @@ func (c *Creator) createQEMU(ctx context.Context, cl terraformClient, lv libvirt if err := lv.Start(ctx, name, config.Provider.QEMU.LibvirtContainerImage); err != nil { return state.ConstellationState{}, err } - // non standard port to avoid conflict with host libvirt - // changes here should also be reflected in the Dockerfile in "cli/internal/libvirt/Dockerfile" - libvirtURI = "qemu+tcp://localhost:16599/system" + libvirtURI = libvirt.LibvirtTCPConnectURI // socket for system URI should be in /var/run/libvirt/libvirt-sock case libvirtURI == "qemu:///system": diff --git a/cli/internal/cmd/create_test.go b/cli/internal/cmd/create_test.go index 25ef89700..1600166a2 100644 --- a/cli/internal/cmd/create_test.go +++ b/cli/internal/cmd/create_test.go @@ -14,6 +14,7 @@ import ( "testing" "github.com/edgelesssys/constellation/v2/internal/cloud/cloudprovider" + "github.com/edgelesssys/constellation/v2/internal/config" "github.com/edgelesssys/constellation/v2/internal/constants" "github.com/edgelesssys/constellation/v2/internal/file" "github.com/edgelesssys/constellation/v2/internal/state" @@ -23,11 +24,17 @@ import ( ) func TestCreate(t *testing.T) { + fsWithDefaultConfig := func(require *require.Assertions, provider cloudprovider.Provider) afero.Fs { + fs := afero.NewMemMapFs() + file := file.NewHandler(fs) + require.NoError(file.WriteYAML(constants.ConfigFilename, defaultConfigWithExpectedMeasurements(t, config.Default(), provider))) + return fs + } testState := state.ConstellationState{Name: "test", LoadBalancerIP: "192.0.2.1"} someErr := errors.New("failed") testCases := map[string]struct { - setupFs func(*require.Assertions) afero.Fs + setupFs func(*require.Assertions, cloudprovider.Provider) afero.Fs creator *stubCloudCreator provider cloudprovider.Provider yesFlag bool @@ -40,7 +47,7 @@ func TestCreate(t *testing.T) { wantAbbort bool }{ "create": { - setupFs: func(require *require.Assertions) afero.Fs { return afero.NewMemMapFs() }, + setupFs: fsWithDefaultConfig, creator: &stubCloudCreator{state: testState}, provider: cloudprovider.GCP, controllerCountFlag: intPtr(1), @@ -48,7 +55,7 @@ func TestCreate(t *testing.T) { yesFlag: true, }, "interactive": { - setupFs: func(require *require.Assertions) afero.Fs { return afero.NewMemMapFs() }, + setupFs: fsWithDefaultConfig, creator: &stubCloudCreator{state: testState}, provider: cloudprovider.Azure, controllerCountFlag: intPtr(2), @@ -56,7 +63,7 @@ func TestCreate(t *testing.T) { stdin: "yes\n", }, "interactive abort": { - setupFs: func(require *require.Assertions) afero.Fs { return afero.NewMemMapFs() }, + setupFs: fsWithDefaultConfig, creator: &stubCloudCreator{}, provider: cloudprovider.GCP, controllerCountFlag: intPtr(1), @@ -65,7 +72,7 @@ func TestCreate(t *testing.T) { wantAbbort: true, }, "interactive error": { - setupFs: func(require *require.Assertions) afero.Fs { return afero.NewMemMapFs() }, + setupFs: fsWithDefaultConfig, creator: &stubCloudCreator{}, provider: cloudprovider.GCP, controllerCountFlag: intPtr(1), @@ -74,7 +81,7 @@ func TestCreate(t *testing.T) { wantErr: true, }, "flag name to long": { - setupFs: func(require *require.Assertions) afero.Fs { return afero.NewMemMapFs() }, + setupFs: fsWithDefaultConfig, creator: &stubCloudCreator{}, provider: cloudprovider.GCP, controllerCountFlag: intPtr(1), @@ -83,7 +90,7 @@ func TestCreate(t *testing.T) { wantErr: true, }, "flag control-plane-count invalid": { - setupFs: func(require *require.Assertions) afero.Fs { return afero.NewMemMapFs() }, + setupFs: fsWithDefaultConfig, creator: &stubCloudCreator{}, provider: cloudprovider.GCP, controllerCountFlag: intPtr(0), @@ -91,7 +98,7 @@ func TestCreate(t *testing.T) { wantErr: true, }, "flag worker-count invalid": { - setupFs: func(require *require.Assertions) afero.Fs { return afero.NewMemMapFs() }, + setupFs: fsWithDefaultConfig, creator: &stubCloudCreator{}, provider: cloudprovider.GCP, controllerCountFlag: intPtr(3), @@ -99,24 +106,25 @@ func TestCreate(t *testing.T) { wantErr: true, }, "flag control-plane-count missing": { - setupFs: func(require *require.Assertions) afero.Fs { return afero.NewMemMapFs() }, + setupFs: fsWithDefaultConfig, creator: &stubCloudCreator{}, provider: cloudprovider.GCP, workerCountFlag: intPtr(3), wantErr: true, }, "flag worker-count missing": { - setupFs: func(require *require.Assertions) afero.Fs { return afero.NewMemMapFs() }, + setupFs: fsWithDefaultConfig, creator: &stubCloudCreator{}, provider: cloudprovider.GCP, controllerCountFlag: intPtr(3), wantErr: true, }, "old state in directory": { - setupFs: func(require *require.Assertions) afero.Fs { + setupFs: func(require *require.Assertions, csp cloudprovider.Provider) afero.Fs { fs := afero.NewMemMapFs() fileHandler := file.NewHandler(fs) require.NoError(fileHandler.Write(constants.StateFilename, []byte{1}, file.OptNone)) + require.NoError(fileHandler.WriteYAML(constants.ConfigFilename, defaultConfigWithExpectedMeasurements(t, config.Default(), csp))) return fs }, creator: &stubCloudCreator{}, @@ -127,10 +135,11 @@ func TestCreate(t *testing.T) { wantErr: true, }, "old adminConf in directory": { - setupFs: func(require *require.Assertions) afero.Fs { + setupFs: func(require *require.Assertions, csp cloudprovider.Provider) afero.Fs { fs := afero.NewMemMapFs() fileHandler := file.NewHandler(fs) require.NoError(fileHandler.Write(constants.AdminConfFilename, []byte{1}, file.OptNone)) + require.NoError(fileHandler.WriteYAML(constants.ConfigFilename, defaultConfigWithExpectedMeasurements(t, config.Default(), csp))) return fs }, creator: &stubCloudCreator{}, @@ -141,10 +150,11 @@ func TestCreate(t *testing.T) { wantErr: true, }, "old masterSecret in directory": { - setupFs: func(require *require.Assertions) afero.Fs { + setupFs: func(require *require.Assertions, csp cloudprovider.Provider) afero.Fs { fs := afero.NewMemMapFs() fileHandler := file.NewHandler(fs) require.NoError(fileHandler.Write(constants.MasterSecretFilename, []byte{1}, file.OptNone)) + require.NoError(fileHandler.WriteYAML(constants.ConfigFilename, defaultConfigWithExpectedMeasurements(t, config.Default(), csp))) return fs }, creator: &stubCloudCreator{}, @@ -155,17 +165,17 @@ func TestCreate(t *testing.T) { wantErr: true, }, "config does not exist": { - setupFs: func(require *require.Assertions) afero.Fs { return afero.NewMemMapFs() }, + setupFs: fsWithDefaultConfig, creator: &stubCloudCreator{}, provider: cloudprovider.GCP, controllerCountFlag: intPtr(1), workerCountFlag: intPtr(1), yesFlag: true, - configFlag: constants.ConfigFilename, + configFlag: "/does/not/exist", wantErr: true, }, "create error": { - setupFs: func(require *require.Assertions) afero.Fs { return afero.NewMemMapFs() }, + setupFs: fsWithDefaultConfig, creator: &stubCloudCreator{createErr: someErr}, provider: cloudprovider.GCP, controllerCountFlag: intPtr(1), @@ -174,8 +184,10 @@ func TestCreate(t *testing.T) { wantErr: true, }, "write state error": { - setupFs: func(require *require.Assertions) afero.Fs { + setupFs: func(require *require.Assertions, csp cloudprovider.Provider) afero.Fs { fs := afero.NewMemMapFs() + fileHandler := file.NewHandler(fs) + require.NoError(fileHandler.WriteYAML(constants.ConfigFilename, defaultConfigWithExpectedMeasurements(t, config.Default(), csp))) return afero.NewReadOnlyFs(fs) }, creator: &stubCloudCreator{}, @@ -196,7 +208,7 @@ func TestCreate(t *testing.T) { cmd.SetOut(&bytes.Buffer{}) cmd.SetErr(&bytes.Buffer{}) cmd.SetIn(bytes.NewBufferString(tc.stdin)) - cmd.Flags().String("config", "", "") // register persisten flag manually + cmd.Flags().String("config", constants.ConfigFilename, "") // register persistent flag manually if tc.yesFlag { require.NoError(cmd.Flags().Set("yes", "true")) } @@ -213,7 +225,7 @@ func TestCreate(t *testing.T) { require.NoError(cmd.Flags().Set("worker-nodes", strconv.Itoa(*tc.workerCountFlag))) } - fileHandler := file.NewHandler(tc.setupFs(require)) + fileHandler := file.NewHandler(tc.setupFs(require, tc.provider)) err := create(cmd, tc.creator, fileHandler) diff --git a/cli/internal/cmd/helmloader.go b/cli/internal/cmd/helmloader.go index 46c7c8c6a..f80458e3b 100644 --- a/cli/internal/cmd/helmloader.go +++ b/cli/internal/cmd/helmloader.go @@ -6,14 +6,16 @@ SPDX-License-Identifier: AGPL-3.0-only package cmd +import "github.com/edgelesssys/constellation/v2/internal/cloud/cloudprovider" + type helmLoader interface { - Load(csp string, conformanceMode bool) ([]byte, error) + Load(csp cloudprovider.Provider, conformanceMode bool) ([]byte, error) } type stubHelmLoader struct { loadErr error } -func (d *stubHelmLoader) Load(csp string, conformanceMode bool) ([]byte, error) { +func (d *stubHelmLoader) Load(csp cloudprovider.Provider, conformanceMode bool) ([]byte, error) { return nil, d.loadErr } diff --git a/cli/internal/cmd/init.go b/cli/internal/cmd/init.go index 3852a2394..d9969cc2f 100644 --- a/cli/internal/cmd/init.go +++ b/cli/internal/cmd/init.go @@ -114,7 +114,7 @@ func initialize(cmd *cobra.Command, newDialer func(validator *cloudcmd.Validator return err } - helmDeployments, err := helmLoader.Load(provider.String(), flags.conformance) + helmDeployments, err := helmLoader.Load(provider, flags.conformance) if err != nil { return fmt.Errorf("loading Helm charts: %w", err) } diff --git a/cli/internal/cmd/init_test.go b/cli/internal/cmd/init_test.go index 38ecce4de..a87e455c1 100644 --- a/cli/internal/cmd/init_test.go +++ b/cli/internal/cmd/init_test.go @@ -60,8 +60,7 @@ func TestInitialize(t *testing.T) { AzureResourceGroup: "test", } testQemuState := &state.ConstellationState{ - CloudProvider: "QEMU", - QEMUWorkerInstances: cloudtypes.Instances{"id-0": {}, "id-1": {}}, + CloudProvider: "QEMU", } testInitResp := &initproto.InitResponse{ Kubeconfig: []byte("kubeconfig"), diff --git a/cli/internal/cmd/mini.go b/cli/internal/cmd/mini.go new file mode 100644 index 000000000..0d130f764 --- /dev/null +++ b/cli/internal/cmd/mini.go @@ -0,0 +1,24 @@ +/* +Copyright (c) Edgeless Systems GmbH + +SPDX-License-Identifier: AGPL-3.0-only +*/ + +package cmd + +import "github.com/spf13/cobra" + +// NewMiniCmd creates a new cobra.Command for managing mini Constellation clusters. +func NewMiniCmd() *cobra.Command { + cmd := &cobra.Command{ + Use: "mini", + Short: "Manage mini Constellation clusters", + Long: "Manage mini Constellation clusters.", + Args: cobra.ExactArgs(0), + } + + cmd.AddCommand(newMiniUpCmd()) + cmd.AddCommand(newMiniDownCmd()) + + return cmd +} diff --git a/cli/internal/cmd/minidown.go b/cli/internal/cmd/minidown.go new file mode 100644 index 000000000..303c3fe34 --- /dev/null +++ b/cli/internal/cmd/minidown.go @@ -0,0 +1,60 @@ +/* +Copyright (c) Edgeless Systems GmbH + +SPDX-License-Identifier: AGPL-3.0-only +*/ + +package cmd + +import ( + "errors" + "fmt" + "os" + + "github.com/edgelesssys/constellation/v2/internal/cloud/cloudprovider" + "github.com/edgelesssys/constellation/v2/internal/constants" + "github.com/edgelesssys/constellation/v2/internal/file" + "github.com/edgelesssys/constellation/v2/internal/state" + "github.com/spf13/afero" + "github.com/spf13/cobra" + "go.uber.org/multierr" +) + +func newMiniDownCmd() *cobra.Command { + cmd := &cobra.Command{ + Use: "down", + Short: "Destroy a mini Constellation cluster", + Long: "Destroy a mini Constellation cluster.", + Args: cobra.ExactArgs(0), + RunE: runDown, + } + + return cmd +} + +func runDown(cmd *cobra.Command, args []string) error { + if err := checkForMiniCluster(file.NewHandler(afero.NewOsFs())); err != nil { + return fmt.Errorf("failed to destroy cluster: %w. Are you in the correct working directory?", err) + } + + err := runTerminate(cmd, args) + if removeErr := os.Remove(constants.MasterSecretFilename); removeErr != nil && !os.IsNotExist(removeErr) { + err = multierr.Append(err, removeErr) + } + return err +} + +func checkForMiniCluster(fileHandler file.Handler) error { + var state state.ConstellationState + if err := fileHandler.ReadJSON(constants.StateFilename, &state); err != nil { + return err + } + if cloudprovider.FromString(state.CloudProvider) != cloudprovider.QEMU { + return errors.New("cluster is not a QEMU based Constellation") + } + if state.Name != "mini" { + return errors.New("cluster is not a mini Constellation cluster") + } + + return nil +} diff --git a/cli/internal/cmd/miniup.go b/cli/internal/cmd/miniup.go new file mode 100644 index 000000000..dbfa32eae --- /dev/null +++ b/cli/internal/cmd/miniup.go @@ -0,0 +1,279 @@ +/* +Copyright (c) Edgeless Systems GmbH + +SPDX-License-Identifier: AGPL-3.0-only +*/ + +package cmd + +import ( + "bufio" + "context" + "errors" + "fmt" + "io" + "net" + "net/http" + "os" + "runtime" + "strings" + + "github.com/edgelesssys/constellation/v2/cli/internal/cloudcmd" + "github.com/edgelesssys/constellation/v2/cli/internal/helm" + "github.com/edgelesssys/constellation/v2/cli/internal/libvirt" + "github.com/edgelesssys/constellation/v2/internal/cloud/cloudprovider" + "github.com/edgelesssys/constellation/v2/internal/config" + "github.com/edgelesssys/constellation/v2/internal/constants" + "github.com/edgelesssys/constellation/v2/internal/file" + "github.com/edgelesssys/constellation/v2/internal/grpc/dialer" + "github.com/edgelesssys/constellation/v2/internal/license" + "github.com/edgelesssys/constellation/v2/internal/versions" + "github.com/schollz/progressbar/v3" + "github.com/spf13/afero" + "github.com/spf13/cobra" + "golang.org/x/sys/unix" +) + +func newMiniUpCmd() *cobra.Command { + cmd := &cobra.Command{ + Use: "up", + Short: "Create and initialize a new mini Constellation cluster", + Long: "Create and initialize a new mini Constellation cluster.\n" + + "A mini cluster consists of a single control-plane and worker node, hosted using QEMU/KVM.\n", + Args: cobra.ExactArgs(0), + RunE: runUp, + } + + // override global flag so we don't have a default value for the config + cmd.Flags().String("config", "", "path to the config file to use for the cluster") + + return cmd +} + +func runUp(cmd *cobra.Command, args []string) error { + if err := checkSystemRequirements(cmd.OutOrStdout()); err != nil { + return fmt.Errorf("system requirements not met: %w", err) + } + + fileHandler := file.NewHandler(afero.NewOsFs()) + + // create config if not passed as flag and set default values + config, err := prepareConfig(cmd, fileHandler) + if err != nil { + return fmt.Errorf("preparing config: %w", err) + } + + // create cluster + spinner := newSpinner(cmd, "Creating cluster in QEMU ", false) + spinner.Start() + err = createMiniCluster(cmd.Context(), fileHandler, cloudcmd.NewCreator(cmd.OutOrStdout()), config) + spinner.Stop() + if err != nil { + return fmt.Errorf("creating cluster: %w", err) + } + cmd.Println("Cluster successfully created.") + connectURI := config.Provider.QEMU.LibvirtURI + if connectURI == "" { + connectURI = libvirt.LibvirtTCPConnectURI + } + cmd.Println("Connect to the VMs by executing:") + cmd.Printf("\tvirsh -c %s\n\n", connectURI) + + // initialize cluster + if err := initializeMiniCluster(cmd, fileHandler); err != nil { + return fmt.Errorf("initializing cluster: %w", err) + } + return nil +} + +// checkSystemRequirements checks if the system meets the requirements for running a mini Constellation cluster. +// We do so by verifying that the host: +// - arch/os is linux/amd64. +// - has access to /dev/kvm. +// - has at least 4 CPU cores. +// - has at least 4GB of memory. +// - has at least 20GB of free disk space. +func checkSystemRequirements(out io.Writer) error { + // check arch/os + if runtime.GOARCH != "amd64" || runtime.GOOS != "linux" { + return fmt.Errorf("creation of a QEMU based Constellation is not supported for %s/%s", runtime.GOOS, runtime.GOARCH) + } + + // check if /dev/kvm exists + if _, err := os.Stat("/dev/kvm"); err != nil { + return fmt.Errorf("unable to access KVM device: %w", err) + } + + // check CPU cores + if runtime.NumCPU() < 4 { + return fmt.Errorf("insufficient CPU cores: %d, at least 4 cores are required by mini Constellation", runtime.NumCPU()) + } + if runtime.NumCPU() < 6 { + fmt.Fprintf(out, "WARNING: Only %d CPU cores available. This may cause performance issues.\n", runtime.NumCPU()) + } + + // check memory + f, err := os.Open("/proc/meminfo") + if err != nil { + return fmt.Errorf("determining available memory: failed to open /proc/meminfo: %w", err) + } + defer f.Close() + var memKB int + scanner := bufio.NewScanner(f) + for scanner.Scan() { + if strings.HasPrefix(scanner.Text(), "MemTotal:") { + _, err = fmt.Sscanf(scanner.Text(), "MemTotal:%d", &memKB) + if err != nil { + return fmt.Errorf("determining available memory: failed to parse /proc/meminfo: %w", err) + } + } + } + memGB := memKB / 1024 / 1024 + if memGB < 4 { + return fmt.Errorf("insufficient memory: %dGB, at least 4GB of memory are required by mini Constellation", memGB) + } + if memGB < 6 { + fmt.Fprintln(out, "WARNING: Less than 6GB of memory available. This may cause performance issues.") + } + + var stat unix.Statfs_t + if err := unix.Statfs(".", &stat); err != nil { + return err + } + freeSpaceGB := stat.Bavail * uint64(stat.Bsize) / 1024 / 1024 / 1024 + if freeSpaceGB < 20 { + return fmt.Errorf("insufficient disk space: %dGB, at least 20GB of disk space are required by mini Constellation", freeSpaceGB) + } + + return nil +} + +// prepareConfig reads a given config, or creates a new minimal QEMU config. +func prepareConfig(cmd *cobra.Command, fileHandler file.Handler) (*config.Config, error) { + configPath, err := cmd.Flags().GetString("config") + if err != nil { + return nil, err + } + + // check for existing config + if configPath != "" { + config, err := readConfig(cmd.OutOrStdout(), fileHandler, configPath) + if err != nil { + return nil, err + } + if config.GetProvider() != cloudprovider.QEMU { + return nil, errors.New("invalid provider for mini constellation cluster") + } + return config, nil + } + + // download image to current directory if it doesn't exist + const imagePath = "./constellation.qcow2" + if _, err := os.Stat(imagePath); err == nil { + cmd.Printf("Using existing image at %s\n\n", imagePath) + } else if errors.Is(err, os.ErrNotExist) { + cmd.Printf("Downloading image to %s\n", imagePath) + if err := installImage(cmd.Context(), cmd.OutOrStdout(), versions.ConstellationQEMUImageURL, imagePath); err != nil { + return nil, fmt.Errorf("downloading image to %s: %w", imagePath, err) + } + } else { + return nil, fmt.Errorf("checking if image exists: %w", err) + } + + if err := cmd.Flags().Set("config", constants.ConfigFilename); err != nil { + return nil, err + } + config := config.Default() + config.RemoveProviderExcept(cloudprovider.QEMU) + config.StateDiskSizeGB = 8 + config.Provider.QEMU.Image = imagePath + + return config, fileHandler.WriteYAML(constants.ConfigFilename, config, file.OptOverwrite) +} + +// createMiniCluster creates a new cluster using the given config. +func createMiniCluster(ctx context.Context, fileHandler file.Handler, creator cloudCreator, config *config.Config) error { + state, err := creator.Create(ctx, cloudprovider.QEMU, config, "mini", "", 1, 1) + if err != nil { + return err + } + if err := fileHandler.WriteJSON(constants.StateFilename, state); err != nil { + return err + } + + return writeIPtoIDFile(fileHandler, state) +} + +// initializeMiniCluster initializes a QEMU cluster. +func initializeMiniCluster(cmd *cobra.Command, fileHandler file.Handler) (retErr error) { + // clean up cluster resources if initialization fails + defer func() { + if retErr != nil { + cmd.Printf("An error occurred: %s\n", retErr) + cmd.Println("Attempting to roll back.") + _ = runDown(cmd, []string{}) + cmd.Printf("Rollback succeeded.\n\n") + } + }() + newDialer := func(validator *cloudcmd.Validator) *dialer.Dialer { + return dialer.New(nil, validator.V(cmd), &net.Dialer{}) + } + helmLoader := &helm.ChartLoader{} + + cmd.Flags().String("master-secret", "", "") + cmd.Flags().String("endpoint", "", "") + cmd.Flags().Bool("conformance", false, "") + + if err := initialize(cmd, newDialer, fileHandler, helmLoader, license.NewClient()); err != nil { + return err + } + return nil +} + +// installImage downloads the image from sourceURL to the destination. +func installImage(ctx context.Context, out io.Writer, sourceURL, destination string) error { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, sourceURL, nil) + if err != nil { + return fmt.Errorf("creating request: %w", err) + } + + resp, err := http.DefaultClient.Do(req) + if err != nil { + return fmt.Errorf("downloading image: %w", err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("downloading image: %s", resp.Status) + } + + f, err := os.OpenFile(destination, os.O_CREATE|os.O_WRONLY, 0o644) + if err != nil { + return err + } + defer f.Close() + + bar := progressbar.NewOptions64( + resp.ContentLength, + progressbar.OptionSetWriter(out), + progressbar.OptionShowBytes(true), + progressbar.OptionSetPredictTime(true), + progressbar.OptionFullWidth(), + progressbar.OptionSetTheme(progressbar.Theme{ + Saucer: "=", + SaucerHead: ">", + SaucerPadding: " ", + BarStart: "[", + BarEnd: "]", + }), + progressbar.OptionClearOnFinish(), + progressbar.OptionOnCompletion(func() { fmt.Fprintf(out, "Done.\n\n") }), + ) + defer bar.Close() + + _, err = io.Copy(io.MultiWriter(f, bar), resp.Body) + if err != nil { + return err + } + + return nil +} diff --git a/cli/internal/cmd/readconfig.go b/cli/internal/cmd/readconfig.go index ea1ee37da..3d748bc9f 100644 --- a/cli/internal/cmd/readconfig.go +++ b/cli/internal/cmd/readconfig.go @@ -16,10 +16,6 @@ import ( ) func readConfig(out io.Writer, fileHandler file.Handler, name string) (*config.Config, error) { - if name == "" { - return config.Default(), nil - } - cnf, err := config.FromFile(fileHandler, name) if err != nil { return nil, err diff --git a/cli/internal/helm/loader.go b/cli/internal/helm/loader.go index 3f8a46cda..dea6bdeab 100644 --- a/cli/internal/helm/loader.go +++ b/cli/internal/helm/loader.go @@ -31,7 +31,7 @@ var HelmFS embed.FS type ChartLoader struct{} -func (i *ChartLoader) Load(csp string, conformanceMode bool) ([]byte, error) { +func (i *ChartLoader) Load(csp cloudprovider.Provider, conformanceMode bool) ([]byte, error) { ciliumDeployment, err := i.loadCilium(csp, conformanceMode) if err != nil { return nil, err @@ -44,13 +44,13 @@ func (i *ChartLoader) Load(csp string, conformanceMode bool) ([]byte, error) { return depl, nil } -func (i *ChartLoader) loadCilium(csp string, conformanceMode bool) (helm.Deployment, error) { +func (i *ChartLoader) loadCilium(csp cloudprovider.Provider, conformanceMode bool) (helm.Deployment, error) { chart, err := loadChartsDir(HelmFS, "charts/cilium") if err != nil { return helm.Deployment{}, err } var ciliumVals map[string]interface{} - switch cloudprovider.FromString(csp) { + switch csp { case cloudprovider.GCP: ciliumVals = gcpVals case cloudprovider.Azure: diff --git a/cli/internal/libvirt/Dockerfile b/cli/internal/libvirt/Dockerfile index a1c40c6e6..f769c6bab 100644 --- a/cli/internal/libvirt/Dockerfile +++ b/cli/internal/libvirt/Dockerfile @@ -6,7 +6,6 @@ RUN dnf -y update && \ qemu-kvm \ swtpm \ swtpm-tools \ - xsltproc \ libvirt-client && \ dnf clean all diff --git a/cli/internal/libvirt/libvirt.go b/cli/internal/libvirt/libvirt.go index 516ef0e46..0ab4f545b 100644 --- a/cli/internal/libvirt/libvirt.go +++ b/cli/internal/libvirt/libvirt.go @@ -9,14 +9,21 @@ package libvirt import ( "context" "errors" + "io" "github.com/docker/docker/api/types" "github.com/docker/docker/api/types/container" + "github.com/docker/docker/api/types/filters" docker "github.com/docker/docker/client" "github.com/edgelesssys/constellation/v2/internal/file" "github.com/spf13/afero" ) +// LibvirtTCPConnectURI is the default URI to connect to containerized libvirt. +// Non standard port to avoid conflict with host libvirt. +// Changes here should also be reflected in the Dockerfile in "cli/internal/libvirt/Dockerfile". +const LibvirtTCPConnectURI = "qemu+tcp://localhost:16599/system" + // Runner handles starting and stopping of containerized libvirt instances. type Runner struct { nameFile string @@ -40,6 +47,32 @@ func (r *Runner) Start(ctx context.Context, name, imageName string) error { defer docker.Close() containerName := name + "-libvirt" + + // check if image exists locally, if not pull it + // this allows us to use a custom image without having to push it to a registry + images, err := docker.ImageList(ctx, types.ImageListOptions{ + Filters: filters.NewArgs( + filters.KeyValuePair{ + Key: "reference", + Value: imageName, + }, + ), + }) + if err != nil { + return err + } + if len(images) == 0 { + reader, err := docker.ImagePull(ctx, imageName, types.ImagePullOptions{}) + if err != nil { + return err + } + defer reader.Close() + if _, err := io.Copy(io.Discard, reader); err != nil { + return err + } + } + + // create and start the libvirt container if _, err := docker.ContainerCreate(ctx, &container.Config{ Image: imageName, @@ -61,13 +94,12 @@ func (r *Runner) Start(ctx context.Context, name, imageName string) error { return err } + // write the name of the container to a file so we can remove it later if err := r.file.Write(r.nameFile, []byte(containerName)); err != nil { _ = docker.ContainerRemove(ctx, containerName, types.ContainerRemoveOptions{Force: true}) return err } - // time.Sleep(15 * time.Second) - return nil } diff --git a/cli/internal/libvirt/start.sh b/cli/internal/libvirt/start.sh index 4e7af4d9c..c5455a967 100755 --- a/cli/internal/libvirt/start.sh +++ b/cli/internal/libvirt/start.sh @@ -1,5 +1,10 @@ #!/bin/bash +# Assign qemu the GID of the host system's 'kvm' group to avoid permission issues for environments defaulting to 660 for /dev/kvm (e.g. Debian-based distros) +KVM_HOST_GID="$(stat -c '%g' /dev/kvm)" +groupadd -o -g "$KVM_HOST_GID" host-kvm +usermod -a -G host-kvm qemu + # Start libvirt daemon libvirtd --daemon --listen virtlogd --daemon diff --git a/cli/internal/terraform/terraform.go b/cli/internal/terraform/terraform.go index f0e1283fa..af1be1cca 100644 --- a/cli/internal/terraform/terraform.go +++ b/cli/internal/terraform/terraform.go @@ -89,6 +89,7 @@ func (c *Client) CreateCluster(ctx context.Context, name string, vars Variables) return errors.New("invalid type in IP output: not a string") } c.state = state.ConstellationState{ + Name: name, CloudProvider: c.provider.String(), LoadBalancerIP: ip, } diff --git a/go.mod b/go.mod index 297eea443..971fead5c 100644 --- a/go.mod +++ b/go.mod @@ -262,7 +262,7 @@ require ( golang.org/x/net v0.0.0-20220624214902-1bab6f366d9e // indirect golang.org/x/oauth2 v0.0.0-20220622183110-fd043fe589d2 // indirect golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f // indirect - golang.org/x/sys v0.0.0-20220915200043-7b5979e65e41 // indirect + golang.org/x/sys v0.0.0-20220915200043-7b5979e65e41 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 // indirect golang.org/x/text v0.3.7 golang.org/x/time v0.0.0-20220224211638-0e9765cccd65 // indirect diff --git a/hack/go.mod b/hack/go.mod index b0ab41ef8..3fa5e9947 100644 --- a/hack/go.mod +++ b/hack/go.mod @@ -68,8 +68,12 @@ require ( github.com/hashicorp/hc-install v0.4.0 // indirect github.com/hashicorp/terraform-exec v0.17.3 // indirect github.com/hashicorp/terraform-json v0.14.0 // indirect + github.com/mattn/go-runewidth v0.0.13 // indirect + github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect github.com/opencontainers/go-digest v1.0.0 // indirect github.com/opencontainers/image-spec v1.0.3-0.20220114050600-8b9d41f48198 // indirect + github.com/rivo/uniseg v0.2.0 // indirect + github.com/schollz/progressbar/v3 v3.8.6 // indirect github.com/sirupsen/logrus v1.8.1 // indirect github.com/zclconf/go-cty v1.11.0 // indirect ) diff --git a/hack/go.sum b/hack/go.sum index ce275b780..55192918d 100644 --- a/hack/go.sum +++ b/hack/go.sum @@ -873,6 +873,7 @@ github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7V github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= github.com/julz/importas v0.0.0-20210419104244-841f0c0fe66d/go.mod h1:oSFU2R4XK/P7kNBrnL/FEQlDGN1/6WoxXEjSSXO0DV0= github.com/k0kubun/colorstring v0.0.0-20150214042306-9440f1994b88/go.mod h1:3w7q1U84EfirKl04SVQ/s7nPm1ZPhiXd34z40TNz36k= +github.com/k0kubun/go-ansi v0.0.0-20180517002512-3bf9e2903213/go.mod h1:vNUNkEQ1e29fT/6vq2aBdFsgNPmy8qMdSay1npru+Sw= github.com/kevinburke/ssh_config v0.0.0-20190725054713-01f96b0aa0cd/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM= github.com/kevinburke/ssh_config v0.0.0-20201106050909-4977a11b4351 h1:DowS9hvgyYSX4TO5NpyC606/Z4SxnNYbT+WX27or6Ck= github.com/kevinburke/ssh_config v0.0.0-20201106050909-4977a11b4351/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM= @@ -961,6 +962,8 @@ github.com/mattn/go-runewidth v0.0.4/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzp github.com/mattn/go-runewidth v0.0.6/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= github.com/mattn/go-runewidth v0.0.7/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= +github.com/mattn/go-runewidth v0.0.13 h1:lTGmDsbAYt5DmK6OnoV7EuIF1wEIFAcxld6ypU4OSgU= +github.com/mattn/go-runewidth v0.0.13/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= github.com/mattn/go-shellwords v1.0.10/go.mod h1:EZzvwXDESEeg03EKmM+RmDnNOPKG4lLtQsUlTZDWQ8Y= github.com/mattn/go-sqlite3 v1.9.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= github.com/mattn/go-zglob v0.0.1/go.mod h1:9fxibJccNxU2cnpIKLRRFA7zX7qhkJIQWBb449FYHOo= @@ -980,6 +983,8 @@ github.com/miekg/pkcs11 v1.0.2/go.mod h1:XsNlhZGX73bx86s2hdc/FuaLm2CPZJemRLMA+WT github.com/miekg/pkcs11 v1.0.3/go.mod h1:XsNlhZGX73bx86s2hdc/FuaLm2CPZJemRLMA+WTFxgs= github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= github.com/mitchellh/cli v1.1.0/go.mod h1:xcISNoH86gajksDmfB23e/pu+B+GeFRMYmoHXxx3xhI= +github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db h1:62I3jR2EmQ4l5rM/4FEfDWcRD+abF5XlKShorW5LRoQ= +github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db/go.mod h1:l0dey0ia/Uv7NcFFVbCLtqEBQbrT4OCwCSKTEv6enCw= github.com/mitchellh/copystructure v1.0.0/go.mod h1:SNtv71yrdKgLRyLFxmLdkAbkKEFWgYaq1OVrnRcwhnw= github.com/mitchellh/copystructure v1.2.0/go.mod h1:qLl+cE2AmVv+CoeAwDPye/v+N2HKCj9FbZEVFJRxO9s= github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= @@ -1160,6 +1165,8 @@ github.com/quasilyte/go-ruleguard/rules v0.0.0-20201231183845-9e62ed36efe1/go.mo github.com/quasilyte/go-ruleguard/rules v0.0.0-20210428214800-545e0d2e0bf7/go.mod h1:4cgAphtvu7Ftv7vOT2ZOYhC6CvBxZixcasr8qIOTA50= github.com/quasilyte/regex/syntax v0.0.0-20200407221936-30656e2c4a95/go.mod h1:rlzQ04UMyJXu/aOvhd8qT+hvDrFpiwqp8MRXDY9szc0= github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= +github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY= +github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/fastuuid v1.1.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= @@ -1184,6 +1191,8 @@ github.com/sagikazarmark/crypt v0.1.0/go.mod h1:B/mN0msZuINBtQ1zZLEQcegFJJf9vnYI github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da/go.mod h1:gi+0XIa01GRL2eRQVjQkKGqKF3SF9vZR/HnPullcV2E= github.com/sanposhiho/wastedassign/v2 v2.0.6/go.mod h1:KyZ0MWTwxxBmfwn33zh3k1dmsbF2ud9pAAGfoLfjhtI= github.com/sassoftware/go-rpmutils v0.0.0-20190420191620-a8f1baeba37b/go.mod h1:am+Fp8Bt506lA3Rk3QCmSqmYmLMnPDhdDUcosQCAx+I= +github.com/schollz/progressbar/v3 v3.8.6 h1:QruMUdzZ1TbEP++S1m73OqRJk20ON11m6Wqv4EoGg8c= +github.com/schollz/progressbar/v3 v3.8.6/go.mod h1:W5IEwbJecncFGBvuEh4A7HT1nZZ6WNIL2i3qbnI0WKY= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= github.com/sebdah/goldie v1.0.0/go.mod h1:jXP4hmWywNEwZzhMuv2ccnqTSFpuq8iyQhtQdkkZBH4= github.com/securego/gosec/v2 v2.9.1/go.mod h1:oDcDLcatOJxkCGaCaq8lua1jTnYf6Sou4wdiJ1n4iHc= @@ -1452,6 +1461,7 @@ golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5y golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.0.0-20220131195533-30dcbda58838/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220214200702-86341886e292/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220829220503-c86fa9a7ed90 h1:Y/gsMcFOcR+6S6f3YeMKl5g+dZMEWqcz5Czj/GWYbkM= golang.org/x/crypto v0.0.0-20220829220503-c86fa9a7ed90/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= diff --git a/hack/terraform-to-state/create-state.go b/hack/terraform-to-state/create-state.go deleted file mode 100644 index a96b6df2f..000000000 --- a/hack/terraform-to-state/create-state.go +++ /dev/null @@ -1,96 +0,0 @@ -/* -Copyright (c) Edgeless Systems GmbH - -SPDX-License-Identifier: AGPL-3.0-only -*/ - -package main - -import ( - "bytes" - "encoding/json" - "fmt" - "os" - "os/exec" - - "github.com/edgelesssys/constellation/v2/internal/cloud/cloudprovider" - "github.com/edgelesssys/constellation/v2/internal/cloud/cloudtypes" - "github.com/edgelesssys/constellation/v2/internal/state" -) - -type terraformOutput struct { - ControlPlaneIPs struct { - Value []string `json:"value"` - } `json:"control_plane_ips"` - WorkerIPs struct { - Value []string `json:"value"` - } `json:"worker_ips"` -} - -func terraformOut(workspaceDir string) (terraformOutput, error) { - cmd := exec.Command("terraform", "output", "--json") - cmd.Dir = workspaceDir - var stdout, stderr bytes.Buffer - cmd.Stdout = &stdout - cmd.Stderr = &stderr - if err := cmd.Run(); err != nil { - return terraformOutput{}, fmt.Errorf("command terraform output failed: %q: %w", stderr.String(), err) - } - var tfOut terraformOutput - if err := json.Unmarshal(stdout.Bytes(), &tfOut); err != nil { - return terraformOutput{}, fmt.Errorf("unmarshaling terraform output: %w", err) - } - return tfOut, nil -} - -func transformState(tfOut terraformOutput) state.ConstellationState { - conState := state.ConstellationState{ - Name: "qemu", - UID: "debug", - CloudProvider: cloudprovider.QEMU.String(), - LoadBalancerIP: tfOut.ControlPlaneIPs.Value[0], - QEMUWorkerInstances: cloudtypes.Instances{}, - QEMUControlPlaneInstances: cloudtypes.Instances{}, - } - for i, ip := range tfOut.ControlPlaneIPs.Value { - conState.QEMUControlPlaneInstances[fmt.Sprintf("control-plane-%d", i)] = cloudtypes.Instance{ - PublicIP: ip, - PrivateIP: ip, - } - } - for i, ip := range tfOut.WorkerIPs.Value { - conState.QEMUWorkerInstances[fmt.Sprintf("worker-%d", i)] = cloudtypes.Instance{ - PublicIP: ip, - PrivateIP: ip, - } - } - return conState -} - -func writeState(workspaceDir string, conState state.ConstellationState) error { - rawState, err := json.Marshal(conState) - if err != nil { - return fmt.Errorf("marshaling state: %w", err) - } - stateFile := fmt.Sprintf("%s/constellation-state.json", workspaceDir) - if err := os.WriteFile(stateFile, rawState, 0o644); err != nil { - return fmt.Errorf("writing state: %w", err) - } - return nil -} - -func main() { - if len(os.Args) != 3 { - fmt.Printf("Usage: %v \n", os.Args[0]) - os.Exit(1) - } - - tfOut, err := terraformOut(os.Args[1]) - if err != nil { - panic(err) - } - conState := transformState(tfOut) - if err := writeState(os.Args[2], conState); err != nil { - panic(err) - } -} diff --git a/internal/config/config.go b/internal/config/config.go index 6ee0661cb..844dd5b5f 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -242,9 +242,9 @@ func Default() *Config { ImageFormat: "qcow2", VCPUs: 2, Memory: 2048, - MetadataAPIImage: "ghcr.io/edgelesssys/constellation/qemu-metadata-api:v2.1.0-pre.0.20221004080046-26a21f00b8cb", + MetadataAPIImage: versions.QEMUMetadataImage, LibvirtURI: "", - LibvirtContainerImage: "ghcr.io/edgelesssys/constellation/libvirt:v2.1.0-pre.0.20221004080046-26a21f00b8cb", + LibvirtContainerImage: versions.LibvirtImage, Measurements: copyPCRMap(qemuPCRs), EnforcedMeasurements: []uint32{11, 12}, }, diff --git a/internal/state/state.go b/internal/state/state.go index cbf294bbe..14a67933e 100644 --- a/internal/state/state.go +++ b/internal/state/state.go @@ -28,7 +28,4 @@ type ConstellationState struct { AzureWorkerScaleSet string `json:"azureworkersscaleset,omitempty"` AzureControlPlaneScaleSet string `json:"azurecontrolplanesscaleset,omitempty"` AzureADAppObjectID string `json:"azureadappobjectid,omitempty"` - - QEMUWorkerInstances cloudtypes.Instances `json:"qemuworkers,omitempty"` - QEMUControlPlaneInstances cloudtypes.Instances `json:"qemucontrolplanes,omitempty"` } diff --git a/internal/versions/versions.go b/internal/versions/versions.go index 9be25c810..e5cdd0bc7 100644 --- a/internal/versions/versions.go +++ b/internal/versions/versions.go @@ -56,6 +56,12 @@ const ( // once https://github.com/medik8s/node-maintenance-operator/issues/49 is resolved. NodeMaintenanceOperatorCatalogImage = "ghcr.io/edgelesssys/constellation/node-maintenance-operator-catalog:v0.13.1-alpha1@sha256:d382c3aaf9bc470cde6f6c05c2c6ff5c9dcfd90540d5b11f9cf69c4e1dd1ca9d" + QEMUMetadataImage = "ghcr.io/edgelesssys/constellation/qemu-metadata-api:v2.1.0-pre.0.20221004080046-26a21f00b8cb" + LibvirtImage = "ghcr.io/edgelesssys/constellation/libvirt:v2.1.0-pre.0.20221006160540-96e6381ca9e2" + + // ConstellationQEMUImageURL is the artifact URL for QEMU qcow2 images. + ConstellationQEMUImageURL = "https://public-edgeless-constellation.s3.us-east-2.amazonaws.com/mini-constellation/mini-constellation-v2.1.0-prerelease.qcow2" + // currently supported versions. //nolint:revive V1_23 ValidK8sVersion = "1.23"