diff --git a/.github/workflows/e2e-test-provider-example.yml b/.github/workflows/e2e-test-provider-example.yml index fd24816f0..b688e3e76 100644 --- a/.github/workflows/e2e-test-provider-example.yml +++ b/.github/workflows/e2e-test-provider-example.yml @@ -328,7 +328,6 @@ jobs: CONTROLNODES: 2 run: | terraform output -raw kubeconfig > constellation-admin.conf - export KUBECONFIG=$(realpath constellation-admin.conf) if [[ -n ${MICROSERVICES} ]]; then MICROSERVICES_FLAG="--target-microservices=$MICROSERVICES" @@ -336,8 +335,10 @@ jobs: if [[ -n ${KUBERNETES} ]]; then KUBERNETES_FLAG="--target-kubernetes=$KUBERNETES" fi + + # cfg must be in same dir as KUBECONFIG ./constellation config generate ${{ inputs.cloudProvider }} - bazel run //e2e/provider-upgrade:provider-upgrade_test -- --want-worker $WORKERNODES --want-control $CONTROLNODES --target-image "$IMAGE" "$KUBERNETES_FLAG" "$MICROSERVICES_FLAG" --cli ${{ github.workspace }}/cluster/constellation + KUBECONFIG=${{ github.workspace }}/cluster/constellation-admin.conf bazel run //e2e/provider-upgrade:provider-upgrade_test -- --want-worker $WORKERNODES --want-control $CONTROLNODES --target-image "$IMAGE" "$KUBERNETES_FLAG" "$MICROSERVICES_FLAG" --cli ${{ github.workspace }}/cluster/constellation - name: Destroy Terraform Cluster # outcome is part of the steps context (https://docs.github.com/en/actions/learn-github-actions/contexts#steps-context) diff --git a/e2e/internal/upgrade/BUILD.bazel b/e2e/internal/upgrade/BUILD.bazel index 444594a1d..86e934e4c 100644 --- a/e2e/internal/upgrade/BUILD.bazel +++ b/e2e/internal/upgrade/BUILD.bazel @@ -15,6 +15,7 @@ go_library( "//internal/semver", "//internal/versions", "@com_github_stretchr_testify//require", + "@io_bazel_rules_go//go/runfiles:go_default_library", "@io_k8s_apimachinery//pkg/apis/meta/v1:meta", "@io_k8s_client_go//kubernetes", "@sh_helm_helm_v3//pkg/action", @@ -48,7 +49,6 @@ go_test( "//internal/versions", "@com_github_spf13_afero//:afero", "@com_github_stretchr_testify//require", - "@io_bazel_rules_go//go/runfiles:go_default_library", "@io_k8s_api//core/v1:core", "@io_k8s_apimachinery//pkg/apis/meta/v1:meta", "@io_k8s_client_go//kubernetes", diff --git a/e2e/internal/upgrade/upgrade.go b/e2e/internal/upgrade/upgrade.go index 30b50fa27..819ec28e9 100644 --- a/e2e/internal/upgrade/upgrade.go +++ b/e2e/internal/upgrade/upgrade.go @@ -23,15 +23,19 @@ package upgrade import ( "bufio" "context" + "errors" "fmt" "io" "log" + "os" "os/exec" + "path/filepath" "strings" "sync" "testing" "time" + "github.com/bazelbuild/rules_go/go/runfiles" "github.com/edgelesssys/constellation/v2/internal/semver" "github.com/edgelesssys/constellation/v2/internal/versions" "github.com/stretchr/testify/require" @@ -39,12 +43,14 @@ import ( "k8s.io/client-go/kubernetes" ) +// VersionContainer contains the versions that the cluster should be upgraded to. type VersionContainer struct { ImageRef string Kubernetes versions.ValidK8sVersion Microservices semver.Semver } +// AssertUpgradeSuccessful tests that the upgrade to the target version is successful. func AssertUpgradeSuccessful(t *testing.T, cli string, targetVersions VersionContainer, k *kubernetes.Clientset, wantControl, wantWorker int, timeout time.Duration) { wg := queryStatusAsync(t, cli) require.NotNil(t, k) @@ -184,3 +190,68 @@ func runCommandWithSeparateOutputs(cmd *exec.Cmd) (stdout, stderr []byte, err er return stdout, stderr, err } + +// Setup checks that the prerequisites for the test are met: +// - a workspace is set +// - a CLI path is set +// - the constellation-upgrade folder does not exist. +func Setup(workspace, cliPath string) error { + workingDir, err := workingDir(workspace) + if err != nil { + return fmt.Errorf("getting working directory: %w", err) + } + + if err := os.Chdir(workingDir); err != nil { + return fmt.Errorf("changing working directory: %w", err) + } + + if _, err := getCLIPath(cliPath); err != nil { + return fmt.Errorf("getting CLI path: %w", err) + } + return nil +} + +// workingDir returns the path to the workspace. +func workingDir(workspace string) (string, error) { + workingDir := os.Getenv("BUILD_WORKING_DIRECTORY") + switch { + case workingDir != "": + return workingDir, nil + case workspace != "": + return workspace, nil + default: + return "", errors.New("neither 'BUILD_WORKING_DIRECTORY' nor 'workspace' flag set") + } +} + +// getCLIPath returns the path to the CLI. +func getCLIPath(cliPathFlag string) (string, error) { + pathCLI := os.Getenv("PATH_CLI") + var relCLIPath string + switch { + case pathCLI != "": + relCLIPath = pathCLI + case cliPathFlag != "": + relCLIPath = cliPathFlag + default: + return "", errors.New("neither 'PATH_CLI' nor 'cli' flag set") + } + + // try to find the CLI in the working directory + // (e.g. when running via `go test` or when specifying a path manually) + workdir, err := os.Getwd() + if err != nil { + return "", fmt.Errorf("getting working directory: %w", err) + } + + absCLIPath := relCLIPath + if !filepath.IsAbs(relCLIPath) { + absCLIPath = filepath.Join(workdir, relCLIPath) + } + if _, err := os.Stat(absCLIPath); err == nil { + return absCLIPath, nil + } + + // fall back to runfiles (e.g. when running via bazel) + return runfiles.Rlocation(pathCLI) +} diff --git a/e2e/internal/upgrade/upgrade_test.go b/e2e/internal/upgrade/upgrade_test.go index f743b8ad1..7334c7a87 100644 --- a/e2e/internal/upgrade/upgrade_test.go +++ b/e2e/internal/upgrade/upgrade_test.go @@ -16,12 +16,10 @@ import ( "log" "os" "os/exec" - "path/filepath" "strings" "testing" "time" - "github.com/bazelbuild/rules_go/go/runfiles" "github.com/edgelesssys/constellation/v2/e2e/internal/kubectl" "github.com/edgelesssys/constellation/v2/internal/api/attestationconfigapi" "github.com/edgelesssys/constellation/v2/internal/config" @@ -60,7 +58,7 @@ var ( func TestUpgrade(t *testing.T) { require := require.New(t) - err := setup() + err := Setup(*workspace, *cliPath) require.NoError(err) k, err := kubectl.New() @@ -97,71 +95,6 @@ func TestUpgrade(t *testing.T) { AssertUpgradeSuccessful(t, cli, targetVersions, k, *wantControl, *wantWorker, *timeout) } -// setup checks that the prerequisites for the test are met: -// - a workspace is set -// - a CLI path is set -// - the constellation-upgrade folder does not exist. -func setup() error { - workingDir, err := workingDir(*workspace) - if err != nil { - return fmt.Errorf("getting working directory: %w", err) - } - - if err := os.Chdir(workingDir); err != nil { - return fmt.Errorf("changing working directory: %w", err) - } - - if _, err := getCLIPath(*cliPath); err != nil { - return fmt.Errorf("getting CLI path: %w", err) - } - return nil -} - -// workingDir returns the path to the workspace. -func workingDir(workspace string) (string, error) { - workingDir := os.Getenv("BUILD_WORKING_DIRECTORY") - switch { - case workingDir != "": - return workingDir, nil - case workspace != "": - return workspace, nil - default: - return "", errors.New("neither 'BUILD_WORKING_DIRECTORY' nor 'workspace' flag set") - } -} - -// getCLIPath returns the path to the CLI. -func getCLIPath(cliPathFlag string) (string, error) { - pathCLI := os.Getenv("PATH_CLI") - var relCLIPath string - switch { - case pathCLI != "": - relCLIPath = pathCLI - case cliPathFlag != "": - relCLIPath = cliPathFlag - default: - return "", errors.New("neither 'PATH_CLI' nor 'cli' flag set") - } - - // try to find the CLI in the working directory - // (e.g. when running via `go test` or when specifying a path manually) - workdir, err := os.Getwd() - if err != nil { - return "", fmt.Errorf("getting working directory: %w", err) - } - - absCLIPath := relCLIPath - if !filepath.IsAbs(relCLIPath) { - absCLIPath = filepath.Join(workdir, relCLIPath) - } - if _, err := os.Stat(absCLIPath); err == nil { - return absCLIPath, nil - } - - // fall back to runfiles (e.g. when running via bazel) - return runfiles.Rlocation(pathCLI) -} - // testPodsEventuallyReady checks that: // 1) all pods are running. // 2) all pods have good status conditions. diff --git a/e2e/provider-upgrade/upgrade_test.go b/e2e/provider-upgrade/upgrade_test.go index 54608ead5..1ad908bbb 100644 --- a/e2e/provider-upgrade/upgrade_test.go +++ b/e2e/provider-upgrade/upgrade_test.go @@ -26,6 +26,8 @@ var ( targetMicroservices = flag.String("target-microservices", "", "Microservice version (MAJOR.MINOR.PATCH) to upgrade to. Defaults to default version of target CLI.") // When executing the test as a bazel target the CLI path is supplied through an env variable that bazel sets. // When executing via `go test` extra care should be taken that the supplied CLI is built on the same commit as this test. + // When executing the test as a bazel target the workspace path is supplied through an env variable that bazel sets. + workspace = flag.String("workspace", "", "Constellation workspace in which to run the tests.") cliPath = flag.String("cli", "", "Constellation CLI to run the tests.") wantWorker = flag.Int("want-worker", 0, "Number of wanted worker nodes.") wantControl = flag.Int("want-control", 0, "Number of wanted control nodes.") @@ -56,5 +58,8 @@ func TestUpgradeSuccessful(t *testing.T) { } k, err := kubectl.New() require.NoError(t, err) + + err = upgrade.Setup(*workspace, *cliPath) + require.NoError(t, err) upgrade.AssertUpgradeSuccessful(t, *cliPath, v, k, *wantControl, *wantWorker, *timeout) }