cli: install helm charts in cli instead of bootstrapper (#2136)

* init

* fixup! init

* gcp working?

* fixup! fixup! init

* azure cfg for microService installation

* fixup! azure cfg for microService installation

* fixup! azure cfg for microService installation

* cleanup bootstrapper code

* cleanup helminstall code

* fixup! cleanup helminstall code

* Update internal/deploy/helm/install.go

Co-authored-by: Daniel Weiße <66256922+daniel-weisse@users.noreply.github.com>

* daniel feedback

* TODO add provider (also to CreateCluster) so we can ensure that provider specific output

* fixup! daniel feedback

* use debugLog in helm installer

* placeholderHelmInstaller

* rename to stub

---------

Co-authored-by: Daniel Weiße <66256922+daniel-weisse@users.noreply.github.com>
This commit is contained in:
Adrian Stobbe 2023-07-31 10:53:05 +02:00 committed by GitHub
parent ef60d00a60
commit 26305e8f80
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
38 changed files with 775 additions and 970 deletions

View file

@ -34,7 +34,6 @@ go_library(
"@com_github_azure_azure_sdk_for_go//profiles/latest/attestation/attestation",
"@com_github_azure_azure_sdk_for_go_sdk_azcore//policy",
"@com_github_azure_azure_sdk_for_go_sdk_azidentity//:azidentity",
"@com_github_hashicorp_terraform_json//:terraform-json",
"@com_github_spf13_cobra//:cobra",
],
)
@ -59,7 +58,6 @@ go_test(
"//internal/cloud/cloudprovider",
"//internal/cloud/gcpshared",
"//internal/config",
"@com_github_hashicorp_terraform_json//:terraform-json",
"@com_github_stretchr_testify//assert",
"@com_github_stretchr_testify//require",
"@org_uber_go_goleak//:goleak",

View file

@ -13,7 +13,6 @@ import (
"github.com/edgelesssys/constellation/v2/cli/internal/terraform"
"github.com/edgelesssys/constellation/v2/internal/attestation/variant"
"github.com/edgelesssys/constellation/v2/internal/cloud/cloudprovider"
tfjson "github.com/hashicorp/terraform-json"
)
// imageFetcher gets an image reference from the versionsapi.
@ -24,14 +23,23 @@ type imageFetcher interface {
) (string, error)
}
type terraformClient interface {
PrepareWorkspace(path string, input terraform.Variables) error
ApplyIAMConfig(ctx context.Context, provider cloudprovider.Provider, logLevel terraform.LogLevel) (terraform.IAMOutput, error)
CreateCluster(ctx context.Context, logLevel terraform.LogLevel) (terraform.ApplyOutput, error)
Destroy(ctx context.Context, logLevel terraform.LogLevel) error
type tfCommonClient interface {
CleanUpWorkspace() error
Destroy(ctx context.Context, logLevel terraform.LogLevel) error
PrepareWorkspace(path string, input terraform.Variables) error
RemoveInstaller()
Show(ctx context.Context) (*tfjson.State, error)
}
type tfResourceClient interface {
tfCommonClient
CreateCluster(ctx context.Context, provider cloudprovider.Provider, logLevel terraform.LogLevel) (terraform.ApplyOutput, error)
ShowCluster(ctx context.Context, provider cloudprovider.Provider) (terraform.ApplyOutput, error)
}
type tfIAMClient interface {
tfCommonClient
ApplyIAMConfig(ctx context.Context, provider cloudprovider.Provider, logLevel terraform.LogLevel) (terraform.IAMOutput, error)
ShowIAM(ctx context.Context, provider cloudprovider.Provider) (terraform.IAMOutput, error)
}
type libvirtRunner interface {

View file

@ -14,7 +14,6 @@ import (
"github.com/edgelesssys/constellation/v2/cli/internal/terraform"
"github.com/edgelesssys/constellation/v2/internal/attestation/variant"
"github.com/edgelesssys/constellation/v2/internal/cloud/cloudprovider"
tfjson "github.com/hashicorp/terraform-json"
"go.uber.org/goleak"
)
@ -32,7 +31,7 @@ type stubTerraformClient struct {
iamOutput terraform.IAMOutput
uid string
attestationURL string
tfjsonState *tfjson.State
applyOutput terraform.ApplyOutput
cleanUpWorkspaceCalled bool
removeInstallerCalled bool
destroyCalled bool
@ -45,12 +44,14 @@ type stubTerraformClient struct {
showErr error
}
func (c *stubTerraformClient) CreateCluster(_ context.Context, _ terraform.LogLevel) (terraform.ApplyOutput, error) {
func (c *stubTerraformClient) CreateCluster(_ context.Context, _ cloudprovider.Provider, _ terraform.LogLevel) (terraform.ApplyOutput, error) {
return terraform.ApplyOutput{
IP: c.ip,
Secret: c.initSecret,
UID: c.uid,
AttestationURL: c.attestationURL,
IP: c.ip,
Secret: c.initSecret,
UID: c.uid,
Azure: &terraform.AzureApplyOutput{
AttestationURL: c.attestationURL,
},
}, c.createClusterErr
}
@ -76,9 +77,14 @@ func (c *stubTerraformClient) RemoveInstaller() {
c.removeInstallerCalled = true
}
func (c *stubTerraformClient) Show(_ context.Context) (*tfjson.State, error) {
func (c *stubTerraformClient) ShowCluster(_ context.Context, _ cloudprovider.Provider) (terraform.ApplyOutput, error) {
c.showCalled = true
return c.tfjsonState, c.showErr
return c.applyOutput, c.showErr
}
func (c *stubTerraformClient) ShowIAM(_ context.Context, _ cloudprovider.Provider) (terraform.IAMOutput, error) {
c.showCalled = true
return c.iamOutput, c.showErr
}
type stubLibvirtRunner struct {

View file

@ -31,7 +31,7 @@ import (
type Creator struct {
out io.Writer
image imageFetcher
newTerraformClient func(ctx context.Context) (terraformClient, error)
newTerraformClient func(ctx context.Context) (tfResourceClient, error)
newLibvirtRunner func() libvirtRunner
newRawDownloader func() rawDownloader
policyPatcher policyPatcher
@ -42,7 +42,7 @@ func NewCreator(out io.Writer) *Creator {
return &Creator{
out: out,
image: imagefetcher.New(),
newTerraformClient: func(ctx context.Context) (terraformClient, error) {
newTerraformClient: func(ctx context.Context) (tfResourceClient, error) {
return terraform.New(ctx, constants.TerraformWorkingDir)
},
newLibvirtRunner: func() libvirtRunner {
@ -115,18 +115,20 @@ func (c *Creator) Create(ctx context.Context, opts CreateOptions) (clusterid.Fil
if err != nil {
return clusterid.File{}, fmt.Errorf("creating cluster: %w", err)
}
return clusterid.File{
res := clusterid.File{
CloudProvider: opts.Provider,
IP: tfOutput.IP,
APIServerCertSANs: tfOutput.APIServerCertSANs,
InitSecret: []byte(tfOutput.Secret),
UID: tfOutput.UID,
AttestationURL: tfOutput.AttestationURL,
}, nil
}
if tfOutput.Azure != nil {
res.AttestationURL = tfOutput.Azure.AttestationURL
}
return res, nil
}
func (c *Creator) createAWS(ctx context.Context, cl terraformClient, opts CreateOptions) (tfOutput terraform.ApplyOutput, retErr error) {
func (c *Creator) createAWS(ctx context.Context, cl tfResourceClient, opts CreateOptions) (tfOutput terraform.ApplyOutput, retErr error) {
vars := awsTerraformVars(opts.Config, opts.image, &opts.ControlPlaneCount, &opts.WorkerCount)
tfOutput, err := runTerraformCreate(ctx, cl, cloudprovider.AWS, vars, c.out, opts.TFLogLevel)
@ -137,7 +139,7 @@ func (c *Creator) createAWS(ctx context.Context, cl terraformClient, opts Create
return tfOutput, nil
}
func (c *Creator) createGCP(ctx context.Context, cl terraformClient, opts CreateOptions) (tfOutput terraform.ApplyOutput, retErr error) {
func (c *Creator) createGCP(ctx context.Context, cl tfResourceClient, opts CreateOptions) (tfOutput terraform.ApplyOutput, retErr error) {
vars := gcpTerraformVars(opts.Config, opts.image, &opts.ControlPlaneCount, &opts.WorkerCount)
tfOutput, err := runTerraformCreate(ctx, cl, cloudprovider.GCP, vars, c.out, opts.TFLogLevel)
@ -148,7 +150,7 @@ func (c *Creator) createGCP(ctx context.Context, cl terraformClient, opts Create
return tfOutput, nil
}
func (c *Creator) createAzure(ctx context.Context, cl terraformClient, opts CreateOptions) (tfOutput terraform.ApplyOutput, retErr error) {
func (c *Creator) createAzure(ctx context.Context, cl tfResourceClient, opts CreateOptions) (tfOutput terraform.ApplyOutput, retErr error) {
vars := azureTerraformVars(opts.Config, opts.image, &opts.ControlPlaneCount, &opts.WorkerCount)
tfOutput, err := runTerraformCreate(ctx, cl, cloudprovider.Azure, vars, c.out, opts.TFLogLevel)
@ -158,7 +160,10 @@ func (c *Creator) createAzure(ctx context.Context, cl terraformClient, opts Crea
if vars.GetCreateMAA() {
// Patch the attestation policy to allow the cluster to boot while having secure boot disabled.
if err := c.policyPatcher.Patch(ctx, tfOutput.AttestationURL); err != nil {
if tfOutput.Azure == nil {
return terraform.ApplyOutput{}, errors.New("no Terraform Azure output found")
}
if err := c.policyPatcher.Patch(ctx, tfOutput.Azure.AttestationURL); err != nil {
return terraform.ApplyOutput{}, err
}
}
@ -197,7 +202,7 @@ func normalizeAzureURIs(vars *terraform.AzureClusterVariables) *terraform.AzureC
return vars
}
func (c *Creator) createOpenStack(ctx context.Context, cl terraformClient, opts CreateOptions) (tfOutput terraform.ApplyOutput, retErr error) {
func (c *Creator) createOpenStack(ctx context.Context, cl tfResourceClient, opts CreateOptions) (tfOutput terraform.ApplyOutput, retErr error) {
// TODO(malt3): Remove this once OpenStack is supported.
if os.Getenv("CONSTELLATION_OPENSTACK_DEV") != "1" {
return terraform.ApplyOutput{}, errors.New("OpenStack isn't supported yet")
@ -221,13 +226,13 @@ func (c *Creator) createOpenStack(ctx context.Context, cl terraformClient, opts
return tfOutput, nil
}
func runTerraformCreate(ctx context.Context, cl terraformClient, provider cloudprovider.Provider, vars terraform.Variables, outWriter io.Writer, loglevel terraform.LogLevel) (output terraform.ApplyOutput, retErr error) {
func runTerraformCreate(ctx context.Context, cl tfResourceClient, provider cloudprovider.Provider, vars terraform.Variables, outWriter io.Writer, loglevel terraform.LogLevel) (output terraform.ApplyOutput, retErr error) {
if err := cl.PrepareWorkspace(path.Join("terraform", strings.ToLower(provider.String())), vars); err != nil {
return terraform.ApplyOutput{}, err
}
defer rollbackOnError(outWriter, &retErr, &rollbackerTerraform{client: cl}, loglevel)
tfOutput, err := cl.CreateCluster(ctx, loglevel)
tfOutput, err := cl.CreateCluster(ctx, provider, loglevel)
if err != nil {
return terraform.ApplyOutput{}, err
}
@ -240,7 +245,7 @@ type qemuCreateOptions struct {
CreateOptions
}
func (c *Creator) createQEMU(ctx context.Context, cl terraformClient, lv libvirtRunner, opts qemuCreateOptions) (tfOutput terraform.ApplyOutput, retErr error) {
func (c *Creator) createQEMU(ctx context.Context, cl tfResourceClient, lv libvirtRunner, opts qemuCreateOptions) (tfOutput terraform.ApplyOutput, retErr error) {
qemuRollbacker := &rollbackerQEMU{client: cl, libvirt: lv, createdWorkspace: false}
defer rollbackOnError(c.out, &retErr, qemuRollbacker, opts.TFLogLevel)
@ -300,7 +305,7 @@ func (c *Creator) createQEMU(ctx context.Context, cl terraformClient, lv libvirt
// Allow rollback of QEMU Terraform workspace from this point on
qemuRollbacker.createdWorkspace = true
tfOutput, err = cl.CreateCluster(ctx, opts.TFLogLevel)
tfOutput, err = cl.CreateCluster(ctx, opts.Provider, opts.TFLogLevel)
if err != nil {
return terraform.ApplyOutput{}, fmt.Errorf("create cluster: %w", err)
}

View file

@ -28,7 +28,7 @@ func TestCreator(t *testing.T) {
someErr := errors.New("failed")
testCases := map[string]struct {
tfClient terraformClient
tfClient tfResourceClient
newTfClientErr error
libvirt *stubLibvirtRunner
provider cloudprovider.Provider
@ -203,7 +203,7 @@ func TestCreator(t *testing.T) {
image: &stubImageFetcher{
reference: "some-image",
},
newTerraformClient: func(ctx context.Context) (terraformClient, error) {
newTerraformClient: func(ctx context.Context) (tfResourceClient, error) {
return tc.tfClient, tc.newTfClientErr
},
newLibvirtRunner: func() libvirtRunner {

View file

@ -24,7 +24,7 @@ import (
// IAMDestroyer destroys an IAM configuration.
type IAMDestroyer struct {
client terraformClient
client tfIAMClient
}
// NewIAMDestroyer creates a new IAM Destroyer.
@ -38,35 +38,23 @@ func NewIAMDestroyer(ctx context.Context) (*IAMDestroyer, error) {
// GetTfstateServiceAccountKey returns the sa_key output from the terraform state.
func (d *IAMDestroyer) GetTfstateServiceAccountKey(ctx context.Context) (gcpshared.ServiceAccountKey, error) {
tfState, err := d.client.Show(ctx)
tfState, err := d.client.ShowIAM(ctx, cloudprovider.GCP)
if err != nil {
return gcpshared.ServiceAccountKey{}, err
return gcpshared.ServiceAccountKey{}, fmt.Errorf("getting terraform state: %w", err)
}
if saKeyString := tfState.GCP.SaKey; saKeyString != "" {
saKey, err := base64.StdEncoding.DecodeString(saKeyString)
if err != nil {
return gcpshared.ServiceAccountKey{}, err
}
var tfSaKey gcpshared.ServiceAccountKey
if err := json.Unmarshal(saKey, &tfSaKey); err != nil {
return gcpshared.ServiceAccountKey{}, err
}
if tfState.Values == nil {
return gcpshared.ServiceAccountKey{}, errors.New("no Values field in terraform state")
return tfSaKey, nil
}
saKeyJSON := tfState.Values.Outputs["sa_key"]
if saKeyJSON == nil {
return gcpshared.ServiceAccountKey{}, errors.New("no sa_key in outputs of the terraform state")
}
saKeyString, ok := saKeyJSON.Value.(string)
if !ok {
return gcpshared.ServiceAccountKey{}, errors.New("sa_key field in terraform state is not a string")
}
saKey, err := base64.StdEncoding.DecodeString(saKeyString)
if err != nil {
return gcpshared.ServiceAccountKey{}, err
}
var tfSaKey gcpshared.ServiceAccountKey
if err := json.Unmarshal(saKey, &tfSaKey); err != nil {
return gcpshared.ServiceAccountKey{}, err
}
return tfSaKey, nil
return gcpshared.ServiceAccountKey{}, errors.New("no saKey in terraform state")
}
// DestroyIAMConfiguration destroys the previously created IAM configuration and deletes the local IAM terraform files.
@ -80,7 +68,7 @@ func (d *IAMDestroyer) DestroyIAMConfiguration(ctx context.Context, logLevel ter
// IAMCreator creates the IAM configuration on the cloud provider.
type IAMCreator struct {
out io.Writer
newTerraformClient func(ctx context.Context) (terraformClient, error)
newTerraformClient func(ctx context.Context) (tfIAMClient, error)
}
// IAMConfigOptions holds the necessary values for IAM configuration.
@ -116,7 +104,7 @@ type AWSIAMConfig struct {
func NewIAMCreator(out io.Writer) *IAMCreator {
return &IAMCreator{
out: out,
newTerraformClient: func(ctx context.Context) (terraformClient, error) {
newTerraformClient: func(ctx context.Context) (tfIAMClient, error) {
return terraform.New(ctx, constants.TerraformIAMWorkingDir)
},
}
@ -152,7 +140,7 @@ func (c *IAMCreator) Create(ctx context.Context, provider cloudprovider.Provider
}
// createGCP creates the IAM configuration on GCP.
func (c *IAMCreator) createGCP(ctx context.Context, cl terraformClient, opts *IAMConfigOptions) (retFile iamid.File, retErr error) {
func (c *IAMCreator) createGCP(ctx context.Context, cl tfIAMClient, opts *IAMConfigOptions) (retFile iamid.File, retErr error) {
defer rollbackOnError(c.out, &retErr, &rollbackerTerraform{client: cl}, opts.TFLogLevel)
vars := terraform.GCPIAMVariables{
@ -180,7 +168,7 @@ func (c *IAMCreator) createGCP(ctx context.Context, cl terraformClient, opts *IA
}
// createAzure creates the IAM configuration on Azure.
func (c *IAMCreator) createAzure(ctx context.Context, cl terraformClient, opts *IAMConfigOptions) (retFile iamid.File, retErr error) {
func (c *IAMCreator) createAzure(ctx context.Context, cl tfIAMClient, opts *IAMConfigOptions) (retFile iamid.File, retErr error) {
defer rollbackOnError(c.out, &retErr, &rollbackerTerraform{client: cl}, opts.TFLogLevel)
vars := terraform.AzureIAMVariables{
@ -209,7 +197,7 @@ func (c *IAMCreator) createAzure(ctx context.Context, cl terraformClient, opts *
}
// createAWS creates the IAM configuration on AWS.
func (c *IAMCreator) createAWS(ctx context.Context, cl terraformClient, opts *IAMConfigOptions) (retFile iamid.File, retErr error) {
func (c *IAMCreator) createAWS(ctx context.Context, cl tfIAMClient, opts *IAMConfigOptions) (retFile iamid.File, retErr error) {
defer rollbackOnError(c.out, &retErr, &rollbackerTerraform{client: cl}, opts.TFLogLevel)
vars := terraform.AWSIAMVariables{

View file

@ -17,7 +17,6 @@ import (
"github.com/edgelesssys/constellation/v2/cli/internal/terraform"
"github.com/edgelesssys/constellation/v2/internal/cloud/cloudprovider"
"github.com/edgelesssys/constellation/v2/internal/cloud/gcpshared"
tfjson "github.com/hashicorp/terraform-json"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
@ -83,7 +82,7 @@ func TestIAMCreator(t *testing.T) {
}
testCases := map[string]struct {
tfClient terraformClient
tfClient tfIAMClient
newTfClientErr error
config *IAMConfigOptions
provider cloudprovider.Provider
@ -125,7 +124,7 @@ func TestIAMCreator(t *testing.T) {
creator := &IAMCreator{
out: &bytes.Buffer{},
newTerraformClient: func(ctx context.Context) (terraformClient, error) {
newTerraformClient: func(ctx context.Context) (tfIAMClient, error) {
return tc.tfClient, tc.newTfClientErr
},
}
@ -225,13 +224,9 @@ func TestGetTfstateServiceAccountKey(t *testing.T) {
}{
"valid": {
cl: &stubTerraformClient{
tfjsonState: &tfjson.State{
Values: &tfjson.StateValues{
Outputs: map[string]*tfjson.StateOutput{
"sa_key": {
Value: gcpFileB64,
},
},
iamOutput: terraform.IAMOutput{
GCP: terraform.GCPIAMOutput{
SaKey: gcpFileB64,
},
},
},
@ -247,31 +242,16 @@ func TestGetTfstateServiceAccountKey(t *testing.T) {
},
"nil tfstate values": {
cl: &stubTerraformClient{
tfjsonState: &tfjson.State{
Values: nil,
},
},
wantErr: true,
wantShowCalled: true,
},
"no key": {
cl: &stubTerraformClient{
tfjsonState: &tfjson.State{
Values: &tfjson.StateValues{},
},
iamOutput: terraform.IAMOutput{},
},
wantErr: true,
wantShowCalled: true,
},
"invalid base64": {
cl: &stubTerraformClient{
tfjsonState: &tfjson.State{
Values: &tfjson.StateValues{
Outputs: map[string]*tfjson.StateOutput{
"sa_key": {
Value: "iamnotvalid",
},
},
iamOutput: terraform.IAMOutput{
GCP: terraform.GCPIAMOutput{
SaKey: "iamnotvalid",
},
},
},
@ -280,28 +260,9 @@ func TestGetTfstateServiceAccountKey(t *testing.T) {
},
"valid base64 invalid json": {
cl: &stubTerraformClient{
tfjsonState: &tfjson.State{
Values: &tfjson.StateValues{
Outputs: map[string]*tfjson.StateOutput{
"sa_key": {
Value: base64.StdEncoding.EncodeToString([]byte("asdf")),
},
},
},
},
},
wantErr: true,
wantShowCalled: true,
},
"not string": {
cl: &stubTerraformClient{
tfjsonState: &tfjson.State{
Values: &tfjson.StateValues{
Outputs: map[string]*tfjson.StateOutput{
"sa_key": {
Value: 1,
},
},
iamOutput: terraform.IAMOutput{
GCP: terraform.GCPIAMOutput{
SaKey: base64.StdEncoding.EncodeToString([]byte("asdf")),
},
},
},

View file

@ -36,7 +36,7 @@ func rollbackOnError(w io.Writer, onErr *error, roll rollbacker, logLevel terraf
}
type rollbackerTerraform struct {
client terraformClient
client tfCommonClient
}
func (r *rollbackerTerraform) rollback(ctx context.Context, logLevel terraform.LogLevel) error {
@ -47,7 +47,7 @@ func (r *rollbackerTerraform) rollback(ctx context.Context, logLevel terraform.L
}
type rollbackerQEMU struct {
client terraformClient
client tfResourceClient
libvirt libvirtRunner
createdWorkspace bool
}

View file

@ -16,14 +16,14 @@ import (
// Terminator deletes cloud provider resources.
type Terminator struct {
newTerraformClient func(ctx context.Context) (terraformClient, error)
newTerraformClient func(ctx context.Context) (tfResourceClient, error)
newLibvirtRunner func() libvirtRunner
}
// NewTerminator create a new cloud terminator.
func NewTerminator() *Terminator {
return &Terminator{
newTerraformClient: func(ctx context.Context) (terraformClient, error) {
newTerraformClient: func(ctx context.Context) (tfResourceClient, error) {
return terraform.New(ctx, constants.TerraformWorkingDir)
},
newLibvirtRunner: func() libvirtRunner {
@ -49,7 +49,7 @@ func (t *Terminator) Terminate(ctx context.Context, logLevel terraform.LogLevel)
return t.terminateTerraform(ctx, cl, logLevel)
}
func (t *Terminator) terminateTerraform(ctx context.Context, cl terraformClient, logLevel terraform.LogLevel) error {
func (t *Terminator) terminateTerraform(ctx context.Context, cl tfResourceClient, logLevel terraform.LogLevel) error {
if err := cl.Destroy(ctx, logLevel); err != nil {
return err
}

View file

@ -19,7 +19,7 @@ func TestTerminator(t *testing.T) {
someErr := errors.New("failed")
testCases := map[string]struct {
tfClient terraformClient
tfClient tfResourceClient
newTfClientErr error
libvirt *stubLibvirtRunner
wantErr bool
@ -55,7 +55,7 @@ func TestTerminator(t *testing.T) {
assert := assert.New(t)
terminator := &Terminator{
newTerraformClient: func(ctx context.Context) (terraformClient, error) {
newTerraformClient: func(ctx context.Context) (tfResourceClient, error) {
return tc.tfClient, tc.newTfClientErr
},
newLibvirtRunner: func() libvirtRunner {