From 0a28cdecb2e4476a0fb5299a3b0f4975092448be Mon Sep 17 00:00:00 2001 From: Moritz Sanft <58110325+msanft@users.noreply.github.com> Date: Fri, 15 Sep 2023 17:21:42 +0200 Subject: [PATCH] ci: add malicious join test (#2304) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * malicious node join test Signed-off-by: Moritz Sanft <58110325+msanft@users.noreply.github.com> * add e2e build tag Signed-off-by: Moritz Sanft <58110325+msanft@users.noreply.github.com> * add namespaces to job apply Signed-off-by: Moritz Sanft <58110325+msanft@users.noreply.github.com> * fix image and workflow Signed-off-by: Moritz Sanft <58110325+msanft@users.noreply.github.com> * fix linter checks Signed-off-by: Moritz Sanft <58110325+msanft@users.noreply.github.com> * build instructions in Dockerfile Signed-off-by: Moritz Sanft <58110325+msanft@users.noreply.github.com> * only print important flags Signed-off-by: Moritz Sanft <58110325+msanft@users.noreply.github.com> * use `malicious-join` namespace Signed-off-by: Moritz Sanft <58110325+msanft@users.noreply.github.com> * build with bazel Signed-off-by: Moritz Sanft <58110325+msanft@users.noreply.github.com> * order imports Signed-off-by: Moritz Sanft <58110325+msanft@users.noreply.github.com> * test cases Signed-off-by: Moritz Sanft <58110325+msanft@users.noreply.github.com> * various fixes Signed-off-by: Moritz Sanft <58110325+msanft@users.noreply.github.com> * add missing quotes Signed-off-by: Moritz Sanft <58110325+msanft@users.noreply.github.com> * fix typo Signed-off-by: Moritz Sanft <58110325+msanft@users.noreply.github.com> * Update e2e/malicious-join/malicious-join.go Co-authored-by: Daniel Weiße <66256922+daniel-weisse@users.noreply.github.com> * Update e2e/malicious-join/malicious-join.go Co-authored-by: Daniel Weiße <66256922+daniel-weisse@users.noreply.github.com> * use switch case Signed-off-by: Moritz Sanft <58110325+msanft@users.noreply.github.com> * update image version Signed-off-by: Moritz Sanft <58110325+msanft@users.noreply.github.com> * fix linter checks Signed-off-by: Moritz Sanft <58110325+msanft@users.noreply.github.com> * wip Signed-off-by: Moritz Sanft <58110325+msanft@users.noreply.github.com> * various fixes Signed-off-by: Moritz Sanft <58110325+msanft@users.noreply.github.com> * update buildfiles Signed-off-by: Moritz Sanft <58110325+msanft@users.noreply.github.com> * use workdir Signed-off-by: Moritz Sanft <58110325+msanft@users.noreply.github.com> * fix linter Signed-off-by: Moritz Sanft <58110325+msanft@users.noreply.github.com> * add required permissions Signed-off-by: Moritz Sanft <58110325+msanft@users.noreply.github.com> * remove permissions Signed-off-by: Moritz Sanft <58110325+msanft@users.noreply.github.com> * remove packages: write permission at step Signed-off-by: Moritz Sanft <58110325+msanft@users.noreply.github.com> * login to registry Signed-off-by: Moritz Sanft <58110325+msanft@users.noreply.github.com> * fix typo Signed-off-by: Moritz Sanft <58110325+msanft@users.noreply.github.com> * fix log Signed-off-by: Moritz Sanft <58110325+msanft@users.noreply.github.com> * source base lib Signed-off-by: Moritz Sanft <58110325+msanft@users.noreply.github.com> * fix sourcing order Signed-off-by: Moritz Sanft <58110325+msanft@users.noreply.github.com> * export after definition Signed-off-by: Moritz Sanft <58110325+msanft@users.noreply.github.com> * fix script header Signed-off-by: Moritz Sanft <58110325+msanft@users.noreply.github.com> * dont exit after -e flag has been set Co-authored-by: Paul Meyer <49727155+katexochen@users.noreply.github.com> --------- Signed-off-by: Moritz Sanft <58110325+msanft@users.noreply.github.com> Co-authored-by: Daniel Weiße <66256922+daniel-weisse@users.noreply.github.com> Co-authored-by: Paul Meyer <49727155+katexochen@users.noreply.github.com> --- .github/actions/e2e_malicious_join/action.yml | 48 ++++ .github/actions/e2e_test/action.yml | 14 +- .github/workflows/e2e-test-manual.yml | 1 + .github/workflows/e2e-test-weekly.yml | 14 ++ e2e/malicious-join/BUILD.bazel | 88 ++++++++ e2e/malicious-join/job.yaml | 12 + e2e/malicious-join/job_template.sh.in | 26 +++ e2e/malicious-join/malicious-join.go | 208 ++++++++++++++++++ 8 files changed, 408 insertions(+), 3 deletions(-) create mode 100644 .github/actions/e2e_malicious_join/action.yml create mode 100644 e2e/malicious-join/BUILD.bazel create mode 100644 e2e/malicious-join/job.yaml create mode 100644 e2e/malicious-join/job_template.sh.in create mode 100644 e2e/malicious-join/malicious-join.go diff --git a/.github/actions/e2e_malicious_join/action.yml b/.github/actions/e2e_malicious_join/action.yml new file mode 100644 index 000000000..3862cbfd2 --- /dev/null +++ b/.github/actions/e2e_malicious_join/action.yml @@ -0,0 +1,48 @@ +name: Malicious join +description: "Verify that a malicious node cannot join a Constellation cluster." + +inputs: + cloudProvider: + description: "The cloud provider the test runs on." + required: true + kubeconfig: + description: "The kubeconfig file for the cluster." + required: true + githubToken: + description: "GitHub authorization token" + required: true + +runs: + using: "composite" + steps: + - name: Log in to the Container registry + id: docker-login + uses: ./.github/actions/container_registry_login + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ inputs.githubToken }} + + - name: Run malicious join + shell: bash + env: + KUBECONFIG: ${{ inputs.kubeconfig }} + working-directory: e2e/malicious-join + run: | + bazel run //e2e/malicious-join:stamp_and_push + yq eval -i "(.spec.template.spec.containers[0].command) = \ + [ \"/malicious-join_bin\", \ + \"--js-endpoint=join-service.kube-system:9090\", \ + \"--csp=${{ inputs.cloudProvider }}\", \ + \"--variant=default\" ]" job.yaml + kubectl create ns malicious-join + kubectl apply -n malicious-join -f job.yaml + kubectl wait -n malicious-join --for=condition=complete --timeout=10m job/malicious-join + kubectl logs -n malicious-join job/malicious-join | tail -n 1 | jq '.' + ALL_TESTS_PASSED=$(kubectl logs -n malicious-join job/malicious-join | tail -n 1 | jq -r '.allPassed') + if [[ "$ALL_TESTS_PASSED" != "true" ]]; then + kubectl logs -n malicious-join job/malicious-join + kubectl logs -n kube-system svc/join-service + exit 1 + fi + kubectl delete ns malicious-join diff --git a/.github/actions/e2e_test/action.yml b/.github/actions/e2e_test/action.yml index 9c393dc98..2c0b14761 100644 --- a/.github/actions/e2e_test/action.yml +++ b/.github/actions/e2e_test/action.yml @@ -51,7 +51,7 @@ inputs: description: "Azure credentials authorized to create an IAM configuration." required: true test: - description: "The test to run. Can currently be one of [sonobuoy full, sonobuoy quick, autoscaling, lb, perf-bench, verify, recover, nop]." + description: "The test to run. Can currently be one of [sonobuoy full, sonobuoy quick, autoscaling, lb, perf-bench, verify, recover, malicious join, nop]." required: true sonobuoyTestSuiteCmd: description: "The sonobuoy test suite to run." @@ -85,7 +85,7 @@ runs: using: "composite" steps: - name: Check input - if: (!contains(fromJson('["sonobuoy full", "sonobuoy quick", "autoscaling", "perf-bench", "verify", "lb", "recover", "nop"]'), inputs.test)) + if: (!contains(fromJson('["sonobuoy full", "sonobuoy quick", "autoscaling", "perf-bench", "verify", "lb", "recover", "malicious join", "nop"]'), inputs.test)) shell: bash run: | echo "::error::Invalid input for test field: ${{ inputs.test }}" @@ -261,10 +261,10 @@ runs: test: ${{ inputs.test }} provider: ${{ inputs.cloudProvider }} isDebugImage: ${{ inputs.isDebugImage }} + # # Test payloads # - - name: Nop test payload if: inputs.test == 'nop' shell: bash @@ -326,3 +326,11 @@ runs: controlNodesCount: ${{ inputs.controlNodesCount }} kubeconfig: ${{ steps.constellation-create.outputs.kubeconfig }} masterSecret: ${{ steps.constellation-create.outputs.masterSecret }} + + - name: Run malicious join test + if: inputs.test == 'malicious join' + uses: ./.github/actions/e2e_malicious_join + with: + cloudProvider: ${{ inputs.cloudProvider }} + kubeconfig: ${{ steps.constellation-create.outputs.kubeconfig }} + githubToken: ${{ inputs.githubToken }} diff --git a/.github/workflows/e2e-test-manual.yml b/.github/workflows/e2e-test-manual.yml index cbd8eaa9f..cc07032db 100644 --- a/.github/workflows/e2e-test-manual.yml +++ b/.github/workflows/e2e-test-manual.yml @@ -34,6 +34,7 @@ on: - "perf-bench" - "verify" - "recover" + - "malicious join" - "nop" required: true kubernetesVersion: diff --git a/.github/workflows/e2e-test-weekly.yml b/.github/workflows/e2e-test-weekly.yml index c405fe421..a7a88d944 100644 --- a/.github/workflows/e2e-test-weekly.yml +++ b/.github/workflows/e2e-test-weekly.yml @@ -157,6 +157,20 @@ jobs: provider: "azure" kubernetes-version: "v1.28" + # malicious join test on latest k8s version + - test: "malicious join" + refStream: "ref/main/stream/debug/?" + provider: "gcp" + kubernetes-version: "v1.28" + - test: "malicious join" + refStream: "ref/main/stream/debug/?" + provider: "azure" + kubernetes-version: "v1.28" + - test: "malicious join" + refStream: "ref/main/stream/debug/?" + provider: "aws" + kubernetes-version: "v1.28" + # # Tests on release-stable refStream # diff --git a/e2e/malicious-join/BUILD.bazel b/e2e/malicious-join/BUILD.bazel new file mode 100644 index 000000000..62c047033 --- /dev/null +++ b/e2e/malicious-join/BUILD.bazel @@ -0,0 +1,88 @@ +load("@com_github_ash2k_bazel_tools//multirun:def.bzl", "multirun") +load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library") +load("@rules_oci//oci:defs.bzl", "oci_image", "oci_push") +load("@rules_pkg//:pkg.bzl", "pkg_tar") +load("//bazel/sh:def.bzl", "sh_template") + +go_library( + name = "malicious-join_lib", + srcs = ["malicious-join.go"], + importpath = "github.com/edgelesssys/constellation/v2/e2e/malicious-join", + visibility = ["//visibility:public"], + deps = [ + "//internal/attestation/variant", + "//internal/cloud/cloudprovider", + "//internal/grpc/dialer", + "//internal/logger", + "//joinservice/joinproto", + "@org_uber_go_zap//zapcore", + ], +) + +go_binary( + name = "malicious-join_bin", + embed = [":malicious-join_lib"], + pure = "on", + race = "off", + visibility = ["//visibility:public"], +) + +pkg_tar( + name = "layer", + srcs = [ + ":malicious-join_bin", + ], + mode = "0755", + remap_paths = {"/malicious-join_bin": "/malicious-join_bin"}, +) + +oci_image( + name = "malicious-join_image", + base = "@distroless_static_linux_amd64", + entrypoint = ["/malicious-join_bin"], + tars = [ + ":layer", + ], + visibility = ["//visibility:public"], +) + +genrule( + name = "malicious-join-test_repotag", + srcs = [ + "//bazel/settings:tag", + ], + outs = ["repotag.txt"], + cmd = "echo -n 'ghcr.io/edgelesssys/malicious-join-test:' | cat - $(location //bazel/settings:tag) > $@", + visibility = ["//visibility:public"], +) + +oci_push( + name = "malicious-join_push", + image = ":malicious-join_image", + repotags = ":repotag.txt", +) + +sh_template( + name = "template_job", + data = [ + "job.yaml", + ":repotag.txt", + "@yq_toolchains//:resolved_toolchain", + ], + substitutions = { + "@@REPO_TAG@@": "$(rootpath :repotag.txt)", + "@@TEMPLATE@@": "$(rootpath :job.yaml)", + "@@YQ_BIN@@": "$(rootpath @yq_toolchains//:resolved_toolchain)", + }, + template = "job_template.sh.in", + visibility = ["//visibility:public"], +) + +multirun( + name = "stamp_and_push", + commands = [ + ":template_job", + ":malicious-join_push", + ], + visibility = ["//visibility:public"], +) diff --git a/e2e/malicious-join/job.yaml b/e2e/malicious-join/job.yaml new file mode 100644 index 000000000..7a67968fd --- /dev/null +++ b/e2e/malicious-join/job.yaml @@ -0,0 +1,12 @@ +apiVersion: batch/v1 +kind: Job +metadata: + name: malicious-join +spec: + template: + spec: + containers: + - name: malicious-join + image: ghcr.io/edgelesssys/malicious-join-test:latest@sha256:f36fe306d50a6731ecdae3920682606967eb339fdd1a1e978b0ce39c2ab744bd + restartPolicy: Never + backoffLimit: 0 # Do not retry diff --git a/e2e/malicious-join/job_template.sh.in b/e2e/malicious-join/job_template.sh.in new file mode 100644 index 000000000..cd1fed0bd --- /dev/null +++ b/e2e/malicious-join/job_template.sh.in @@ -0,0 +1,26 @@ +#!/usr/bin/env bash + +lib=$(realpath @@BASE_LIB@@) || exit 1 +stat "${lib}" >> /dev/null || exit 1 + +# shellcheck source=../../bazel/sh/lib.bash +if ! source "${lib}"; then + echo "Error: could not find import" + exit 1 +fi + +yq=$(realpath @@YQ_BIN@@) +template=$(realpath @@TEMPLATE@@) +REPO_TAG=$(realpath @@REPO_TAG@@) +export REPO_TAG + +cd "${BUILD_WORKING_DIRECTORY}" + +if [[ $# -eq 0 ]]; then + workdir="." +else + workdir="$1" +fi + +echo "Stamping job deployment with $REPO_TAG" +$yq eval '.spec.template.spec.containers[0].image |= "ghcr.io/edgelesssys/malicious-join-test:" + load_str(strenv(REPO_TAG))' "$template" > "$workdir/stamped_job.yaml" diff --git a/e2e/malicious-join/malicious-join.go b/e2e/malicious-join/malicious-join.go new file mode 100644 index 000000000..ebdbe6ba8 --- /dev/null +++ b/e2e/malicious-join/malicious-join.go @@ -0,0 +1,208 @@ +/* +Copyright (c) Edgeless Systems GmbH + +SPDX-License-Identifier: AGPL-3.0-only +*/ + +// End-to-end test that issues various types of malicious join requests to a cluster. +package main + +import ( + "context" + "encoding/json" + "flag" + "fmt" + "net" + "strings" + + "github.com/edgelesssys/constellation/v2/internal/attestation/variant" + "github.com/edgelesssys/constellation/v2/internal/cloud/cloudprovider" + "github.com/edgelesssys/constellation/v2/internal/grpc/dialer" + "github.com/edgelesssys/constellation/v2/internal/logger" + "github.com/edgelesssys/constellation/v2/joinservice/joinproto" + "go.uber.org/zap/zapcore" +) + +func main() { + jsEndpoint := flag.String("js-endpoint", "", "Join service endpoint to use.") + csp := flag.String("csp", "", "Cloud service provider to use.") + attVariant := flag.String( + "variant", + "", + fmt.Sprintf("Attestation variant to use. Set to \"default\" to use the default attestation variant for the CSP,"+ + "or one of: %s", variant.GetAvailableAttestationVariants()), + ) + flag.Parse() + fmt.Println(formatFlags(*attVariant, *csp, *jsEndpoint)) + + testCases := map[string]struct { + fn func(attVariant, csp, jsEndpoint string) error + wantErr bool + }{ + "JoinFromUnattestedNode": { + fn: JoinFromUnattestedNode, + wantErr: true, + }, + } + + allPassed := true + testOutput := &testOutput{ + TestCases: make(map[string]testCaseOutput), + } + for name, tc := range testCases { + fmt.Printf("Running testcase %s\n", name) + + err := tc.fn(*attVariant, *csp, *jsEndpoint) + + switch { + case err == nil && tc.wantErr: + fmt.Printf("Test case %s failed: Expected error but got none\n", name) + testOutput.TestCases[name] = testCaseOutput{ + Passed: false, + Message: "Expected error but got none", + } + allPassed = false + case !tc.wantErr && err != nil: + fmt.Printf("Test case %s failed: Got unexpected error: %s\n", name, err) + testOutput.TestCases[name] = testCaseOutput{ + Passed: false, + Message: fmt.Sprintf("Got unexpected error: %s", err), + } + allPassed = false + case tc.wantErr && err != nil: + fmt.Printf("Test case %s succeeded\n", name) + testOutput.TestCases[name] = testCaseOutput{ + Passed: true, + Message: fmt.Sprintf("Got expected error: %s", err), + } + case !tc.wantErr && err == nil: + fmt.Printf("Test case %s succeeded\n", name) + testOutput.TestCases[name] = testCaseOutput{ + Passed: true, + Message: "No error, as expected", + } + default: + panic("invalid result") + } + } + + testOutput.AllPassed = allPassed + out, err := json.Marshal(testOutput) + if err != nil { + panic(fmt.Sprintf("marshalling test output: %s", err)) + } + fmt.Println(string(out)) +} + +type testOutput struct { + AllPassed bool `json:"allPassed"` + TestCases map[string]testCaseOutput `json:"testCases"` +} + +type testCaseOutput struct { + Passed bool `json:"passed"` + Message string `json:"message"` +} + +func formatFlags(attVariant, csp, jsEndpoint string) string { + var sb strings.Builder + sb.WriteString("Using Flags:\n") + sb.WriteString(fmt.Sprintf("\tjs-endpoint: %s\n", jsEndpoint)) + sb.WriteString(fmt.Sprintf("\tcsp: %s\n", csp)) + sb.WriteString(fmt.Sprintf("\tvariant: %s\n", attVariant)) + return sb.String() +} + +// JoinFromUnattestedNode simulates a join request from a Node that uses a stub issuer +// and thus cannot be attested correctly. +func JoinFromUnattestedNode(attVariant, csp, jsEndpoint string) error { + log := logger.New(logger.JSONLog, zapcore.DebugLevel) + joiner, err := newMaliciousJoiner(attVariant, csp, jsEndpoint, log) + if err != nil { + return fmt.Errorf("creating malicious joiner: %w", err) + } + + _, err = joiner.join(context.Background()) + if err != nil { + return fmt.Errorf("joining cluster: %w", err) + } + return nil +} + +// newMaliciousJoiner creates a new malicious joiner, i.e. a simulated node that issues +// an invalid join request. +func newMaliciousJoiner(attVariant, csp, endpoint string, log *logger.Logger) (*maliciousJoiner, error) { + var attVariantOid variant.Variant + var err error + if strings.EqualFold(attVariant, "default") { + attVariantOid = variant.GetDefaultAttestation(cloudprovider.FromString(csp)) + } else { + attVariantOid, err = variant.FromString(attVariant) + if err != nil { + return nil, fmt.Errorf("parsing attestation variant: %w", err) + } + } + + issuer := newFakeIssuer(attVariantOid) + + return &maliciousJoiner{ + endpoint: endpoint, + logger: log, + dialer: dialer.New(issuer, nil, &net.Dialer{}), + }, nil +} + +// maliciousJoiner simulates a malicious node joining a cluster. +type maliciousJoiner struct { + endpoint string + logger *logger.Logger + dialer *dialer.Dialer +} + +// join issues a join request to the join service endpoint. +func (j *maliciousJoiner) join(ctx context.Context) (*joinproto.IssueJoinTicketResponse, error) { + j.logger.Debugf("Dialing join service endpoint %s", j.endpoint) + conn, err := j.dialer.Dial(ctx, j.endpoint) + if err != nil { + return nil, fmt.Errorf("dialing join service endpoint: %w", err) + } + defer conn.Close() + j.logger.Debugf("Successfully dialed join service endpoint %s", j.endpoint) + + protoClient := joinproto.NewAPIClient(conn) + + j.logger.Debugf("Issuing join ticket") + req := &joinproto.IssueJoinTicketRequest{ + DiskUuid: "", + CertificateRequest: []byte{}, + IsControlPlane: false, + } + res, err := protoClient.IssueJoinTicket(ctx, req) + j.logger.Debugf("Got join ticket response: %+v", res) + if err != nil { + return nil, fmt.Errorf("issuing join ticket: %w", err) + } + + return res, nil +} + +// newFakeIssuer creates a new fake issuer for a given attestation variant. +func newFakeIssuer(oid variant.Getter) *fakeIssuer { + return &fakeIssuer{oid} +} + +// fakeIssuer simulates an issuer that issues a fake / invalid attestation document. +type fakeIssuer struct { + variant.Getter +} + +// Issue issues a fake attestation document. +func (i *fakeIssuer) Issue(_ context.Context, userData, nonce []byte) ([]byte, error) { + return json.Marshal(fakeAttestationDoc{UserData: userData, Nonce: nonce}) +} + +// fakeAttestationDoc is a fake attestation document. +type fakeAttestationDoc struct { + UserData []byte + Nonce []byte +}