ci: add malicious join test (#2304)

* malicious node join test

Signed-off-by: Moritz Sanft <58110325+msanft@users.noreply.github.com>

* add e2e build tag

Signed-off-by: Moritz Sanft <58110325+msanft@users.noreply.github.com>

* add namespaces to job apply

Signed-off-by: Moritz Sanft <58110325+msanft@users.noreply.github.com>

* fix image and workflow

Signed-off-by: Moritz Sanft <58110325+msanft@users.noreply.github.com>

* fix linter checks

Signed-off-by: Moritz Sanft <58110325+msanft@users.noreply.github.com>

* build instructions in Dockerfile

Signed-off-by: Moritz Sanft <58110325+msanft@users.noreply.github.com>

* only print important flags

Signed-off-by: Moritz Sanft <58110325+msanft@users.noreply.github.com>

* use `malicious-join` namespace

Signed-off-by: Moritz Sanft <58110325+msanft@users.noreply.github.com>

* build with bazel

Signed-off-by: Moritz Sanft <58110325+msanft@users.noreply.github.com>

* order imports

Signed-off-by: Moritz Sanft <58110325+msanft@users.noreply.github.com>

* test cases

Signed-off-by: Moritz Sanft <58110325+msanft@users.noreply.github.com>

* various fixes

Signed-off-by: Moritz Sanft <58110325+msanft@users.noreply.github.com>

* add missing quotes

Signed-off-by: Moritz Sanft <58110325+msanft@users.noreply.github.com>

* fix typo

Signed-off-by: Moritz Sanft <58110325+msanft@users.noreply.github.com>

* Update e2e/malicious-join/malicious-join.go

Co-authored-by: Daniel Weiße <66256922+daniel-weisse@users.noreply.github.com>

* Update e2e/malicious-join/malicious-join.go

Co-authored-by: Daniel Weiße <66256922+daniel-weisse@users.noreply.github.com>

* use switch case

Signed-off-by: Moritz Sanft <58110325+msanft@users.noreply.github.com>

* update image version

Signed-off-by: Moritz Sanft <58110325+msanft@users.noreply.github.com>

* fix linter checks

Signed-off-by: Moritz Sanft <58110325+msanft@users.noreply.github.com>

* wip

Signed-off-by: Moritz Sanft <58110325+msanft@users.noreply.github.com>

* various fixes

Signed-off-by: Moritz Sanft <58110325+msanft@users.noreply.github.com>

* update buildfiles

Signed-off-by: Moritz Sanft <58110325+msanft@users.noreply.github.com>

* use workdir

Signed-off-by: Moritz Sanft <58110325+msanft@users.noreply.github.com>

* fix linter

Signed-off-by: Moritz Sanft <58110325+msanft@users.noreply.github.com>

* add required permissions

Signed-off-by: Moritz Sanft <58110325+msanft@users.noreply.github.com>

* remove permissions

Signed-off-by: Moritz Sanft <58110325+msanft@users.noreply.github.com>

* remove packages: write permission at step

Signed-off-by: Moritz Sanft <58110325+msanft@users.noreply.github.com>

* login to registry

Signed-off-by: Moritz Sanft <58110325+msanft@users.noreply.github.com>

* fix typo

Signed-off-by: Moritz Sanft <58110325+msanft@users.noreply.github.com>

* fix log

Signed-off-by: Moritz Sanft <58110325+msanft@users.noreply.github.com>

* source base lib

Signed-off-by: Moritz Sanft <58110325+msanft@users.noreply.github.com>

* fix sourcing order

Signed-off-by: Moritz Sanft <58110325+msanft@users.noreply.github.com>

* export after definition

Signed-off-by: Moritz Sanft <58110325+msanft@users.noreply.github.com>

* fix script header

Signed-off-by: Moritz Sanft <58110325+msanft@users.noreply.github.com>

* dont exit after -e flag has been set

Co-authored-by: Paul Meyer <49727155+katexochen@users.noreply.github.com>

---------

Signed-off-by: Moritz Sanft <58110325+msanft@users.noreply.github.com>
Co-authored-by: Daniel Weiße <66256922+daniel-weisse@users.noreply.github.com>
Co-authored-by: Paul Meyer <49727155+katexochen@users.noreply.github.com>

.github/actions/e2e_malicious_join/action.yml

@@ -0,0 +1,48 @@
+name: Malicious join
+description: "Verify that a malicious node cannot join a Constellation cluster."
+
+inputs:
+  cloudProvider:
+    description: "The cloud provider the test runs on."
+    required: true
+  kubeconfig:
+    description: "The kubeconfig file for the cluster."
+    required: true
+  githubToken:
+    description: "GitHub authorization token"
+    required: true
+
+runs:
+  using: "composite"
+  steps:
+    - name: Log in to the Container registry
+      id: docker-login
+      uses: ./.github/actions/container_registry_login
+      with:
+        registry: ghcr.io
+        username: ${{ github.actor }}
+        password: ${{ inputs.githubToken }}
+
+    - name: Run malicious join
+      shell: bash
+      env:
+        KUBECONFIG: ${{ inputs.kubeconfig }}
+      working-directory: e2e/malicious-join
+      run: |
+        bazel run //e2e/malicious-join:stamp_and_push
+        yq eval -i "(.spec.template.spec.containers[0].command) = \
+          [ \"/malicious-join_bin\", \
+          \"--js-endpoint=join-service.kube-system:9090\", \
+          \"--csp=${{ inputs.cloudProvider }}\", \
+          \"--variant=default\" ]" job.yaml
+        kubectl create ns malicious-join
+        kubectl apply -n malicious-join -f job.yaml
+        kubectl wait -n malicious-join --for=condition=complete --timeout=10m job/malicious-join
+        kubectl logs -n malicious-join job/malicious-join | tail -n 1 | jq '.'
+        ALL_TESTS_PASSED=$(kubectl logs -n malicious-join job/malicious-join | tail -n 1 | jq -r '.allPassed')
+        if [[ "$ALL_TESTS_PASSED" != "true" ]]; then
+          kubectl logs -n malicious-join job/malicious-join
+          kubectl logs -n kube-system svc/join-service
+          exit 1
+        fi
+        kubectl delete ns malicious-join

.github/actions/e2e_test/action.yml

@@ -51,7 +51,7 @@ inputs:
     description: "Azure credentials authorized to create an IAM configuration."
     required: true
   test:
-    description: "The test to run. Can currently be one of [sonobuoy full, sonobuoy quick, autoscaling, lb, perf-bench, verify, recover, nop]."
+    description: "The test to run. Can currently be one of [sonobuoy full, sonobuoy quick, autoscaling, lb, perf-bench, verify, recover, malicious join, nop]."
     required: true
   sonobuoyTestSuiteCmd:
     description: "The sonobuoy test suite to run."
@@ -85,7 +85,7 @@ runs:
   using: "composite"
   steps:
     - name: Check input
-      if: (!contains(fromJson('["sonobuoy full", "sonobuoy quick", "autoscaling", "perf-bench", "verify", "lb", "recover", "nop"]'), inputs.test))
+      if: (!contains(fromJson('["sonobuoy full", "sonobuoy quick", "autoscaling", "perf-bench", "verify", "lb", "recover", "malicious join", "nop"]'), inputs.test))
       shell: bash
       run: |
         echo "::error::Invalid input for test field: ${{ inputs.test }}"
@@ -261,10 +261,10 @@ runs:
         test: ${{ inputs.test }}
         provider: ${{ inputs.cloudProvider }}
         isDebugImage: ${{ inputs.isDebugImage }}
+
     #
     # Test payloads
     #
-
     - name: Nop test payload
       if: inputs.test == 'nop'
       shell: bash
@@ -326,3 +326,11 @@ runs:
         controlNodesCount: ${{ inputs.controlNodesCount }}
         kubeconfig: ${{ steps.constellation-create.outputs.kubeconfig }}
         masterSecret: ${{ steps.constellation-create.outputs.masterSecret }}
+
+    - name: Run malicious join test
+      if: inputs.test == 'malicious join'
+      uses: ./.github/actions/e2e_malicious_join
+      with:
+        cloudProvider: ${{ inputs.cloudProvider }}
+        kubeconfig: ${{ steps.constellation-create.outputs.kubeconfig }}
+        githubToken: ${{ inputs.githubToken }}

.github/workflows/e2e-test-manual.yml

@@ -34,6 +34,7 @@ on:
           - "perf-bench"
           - "verify"
           - "recover"
+          - "malicious join"
           - "nop"
         required: true
       kubernetesVersion:

.github/workflows/e2e-test-weekly.yml

@@ -157,6 +157,20 @@ jobs:
             provider: "azure"
             kubernetes-version: "v1.28"
 
+          # malicious join test on latest k8s version
+          - test: "malicious join"
+            refStream: "ref/main/stream/debug/?"
+            provider: "gcp"
+            kubernetes-version: "v1.28"
+          - test: "malicious join"
+            refStream: "ref/main/stream/debug/?"
+            provider: "azure"
+            kubernetes-version: "v1.28"
+          - test: "malicious join"
+            refStream: "ref/main/stream/debug/?"
+            provider: "aws"
+            kubernetes-version: "v1.28"
+
           #
           # Tests on release-stable refStream
           #

e2e/malicious-join/BUILD.bazel

@@ -0,0 +1,88 @@
+load("@com_github_ash2k_bazel_tools//multirun:def.bzl", "multirun")
+load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library")
+load("@rules_oci//oci:defs.bzl", "oci_image", "oci_push")
+load("@rules_pkg//:pkg.bzl", "pkg_tar")
+load("//bazel/sh:def.bzl", "sh_template")
+
+go_library(
+    name = "malicious-join_lib",
+    srcs = ["malicious-join.go"],
+    importpath = "github.com/edgelesssys/constellation/v2/e2e/malicious-join",
+    visibility = ["//visibility:public"],
+    deps = [
+        "//internal/attestation/variant",
+        "//internal/cloud/cloudprovider",
+        "//internal/grpc/dialer",
+        "//internal/logger",
+        "//joinservice/joinproto",
+        "@org_uber_go_zap//zapcore",
+    ],
+)
+
+go_binary(
+    name = "malicious-join_bin",
+    embed = [":malicious-join_lib"],
+    pure = "on",
+    race = "off",
+    visibility = ["//visibility:public"],
+)
+
+pkg_tar(
+    name = "layer",
+    srcs = [
+        ":malicious-join_bin",
+    ],
+    mode = "0755",
+    remap_paths = {"/malicious-join_bin": "/malicious-join_bin"},
+)
+
+oci_image(
+    name = "malicious-join_image",
+    base = "@distroless_static_linux_amd64",
+    entrypoint = ["/malicious-join_bin"],
+    tars = [
+        ":layer",
+    ],
+    visibility = ["//visibility:public"],
+)
+
+genrule(
+    name = "malicious-join-test_repotag",
+    srcs = [
+        "//bazel/settings:tag",
+    ],
+    outs = ["repotag.txt"],
+    cmd = "echo -n 'ghcr.io/edgelesssys/malicious-join-test:' | cat - $(location //bazel/settings:tag) > $@",
+    visibility = ["//visibility:public"],
+)
+
+oci_push(
+    name = "malicious-join_push",
+    image = ":malicious-join_image",
+    repotags = ":repotag.txt",
+)
+
+sh_template(
+    name = "template_job",
+    data = [
+        "job.yaml",
+        ":repotag.txt",
+        "@yq_toolchains//:resolved_toolchain",
+    ],
+    substitutions = {
+        "@@REPO_TAG@@": "$(rootpath :repotag.txt)",
+        "@@TEMPLATE@@": "$(rootpath :job.yaml)",
+        "@@YQ_BIN@@": "$(rootpath @yq_toolchains//:resolved_toolchain)",
+    },
+    template = "job_template.sh.in",
+    visibility = ["//visibility:public"],
+)
+
+multirun(
+    name = "stamp_and_push",
+    commands = [
+        ":template_job",
+        ":malicious-join_push",
+    ],
+    visibility = ["//visibility:public"],
+)

e2e/malicious-join/job.yaml

@@ -0,0 +1,12 @@
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: malicious-join
+spec:
+  template:
+    spec:
+      containers:
+        - name: malicious-join
+          image: ghcr.io/edgelesssys/malicious-join-test:latest@sha256:f36fe306d50a6731ecdae3920682606967eb339fdd1a1e978b0ce39c2ab744bd
+      restartPolicy: Never
+  backoffLimit: 0 # Do not retry

e2e/malicious-join/job_template.sh.in

@@ -0,0 +1,26 @@
+#!/usr/bin/env bash
+
+lib=$(realpath @@BASE_LIB@@) || exit 1
+stat "${lib}" >> /dev/null || exit 1
+
+# shellcheck source=../../bazel/sh/lib.bash
+if ! source "${lib}"; then
+  echo "Error: could not find import"
+  exit 1
+fi
+
+yq=$(realpath @@YQ_BIN@@)
+template=$(realpath @@TEMPLATE@@)
+REPO_TAG=$(realpath @@REPO_TAG@@)
+export REPO_TAG
+
+cd "${BUILD_WORKING_DIRECTORY}"
+
+if [[ $# -eq 0 ]]; then
+  workdir="."
+else
+  workdir="$1"
+fi
+
+echo "Stamping job deployment with $REPO_TAG"
+$yq eval '.spec.template.spec.containers[0].image |= "ghcr.io/edgelesssys/malicious-join-test:" + load_str(strenv(REPO_TAG))' "$template" > "$workdir/stamped_job.yaml"

e2e/malicious-join/malicious-join.go

@@ -0,0 +1,208 @@
+/*
+Copyright (c) Edgeless Systems GmbH
+
+SPDX-License-Identifier: AGPL-3.0-only
+*/
+
+// End-to-end test that issues various types of malicious join requests to a cluster.
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"flag"
+	"fmt"
+	"net"
+	"strings"
+
+	"github.com/edgelesssys/constellation/v2/internal/attestation/variant"
+	"github.com/edgelesssys/constellation/v2/internal/cloud/cloudprovider"
+	"github.com/edgelesssys/constellation/v2/internal/grpc/dialer"
+	"github.com/edgelesssys/constellation/v2/internal/logger"
+	"github.com/edgelesssys/constellation/v2/joinservice/joinproto"
+	"go.uber.org/zap/zapcore"
+)
+
+func main() {
+	jsEndpoint := flag.String("js-endpoint", "", "Join service endpoint to use.")
+	csp := flag.String("csp", "", "Cloud service provider to use.")
+	attVariant := flag.String(
+		"variant",
+		"",
+		fmt.Sprintf("Attestation variant to use. Set to \"default\" to use the default attestation variant for the CSP,"+
+			"or one of: %s", variant.GetAvailableAttestationVariants()),
+	)
+	flag.Parse()
+	fmt.Println(formatFlags(*attVariant, *csp, *jsEndpoint))
+
+	testCases := map[string]struct {
+		fn      func(attVariant, csp, jsEndpoint string) error
+		wantErr bool
+	}{
+		"JoinFromUnattestedNode": {
+			fn:      JoinFromUnattestedNode,
+			wantErr: true,
+		},
+	}
+
+	allPassed := true
+	testOutput := &testOutput{
+		TestCases: make(map[string]testCaseOutput),
+	}
+	for name, tc := range testCases {
+		fmt.Printf("Running testcase %s\n", name)
+
+		err := tc.fn(*attVariant, *csp, *jsEndpoint)
+
+		switch {
+		case err == nil && tc.wantErr:
+			fmt.Printf("Test case %s failed: Expected error but got none\n", name)
+			testOutput.TestCases[name] = testCaseOutput{
+				Passed:  false,
+				Message: "Expected error but got none",
+			}
+			allPassed = false
+		case !tc.wantErr && err != nil:
+			fmt.Printf("Test case %s failed: Got unexpected error: %s\n", name, err)
+			testOutput.TestCases[name] = testCaseOutput{
+				Passed:  false,
+				Message: fmt.Sprintf("Got unexpected error: %s", err),
+			}
+			allPassed = false
+		case tc.wantErr && err != nil:
+			fmt.Printf("Test case %s succeeded\n", name)
+			testOutput.TestCases[name] = testCaseOutput{
+				Passed:  true,
+				Message: fmt.Sprintf("Got expected error: %s", err),
+			}
+		case !tc.wantErr && err == nil:
+			fmt.Printf("Test case %s succeeded\n", name)
+			testOutput.TestCases[name] = testCaseOutput{
+				Passed:  true,
+				Message: "No error, as expected",
+			}
+		default:
+			panic("invalid result")
+		}
+	}
+
+	testOutput.AllPassed = allPassed
+	out, err := json.Marshal(testOutput)
+	if err != nil {
+		panic(fmt.Sprintf("marshalling test output: %s", err))
+	}
+	fmt.Println(string(out))
+}
+
+type testOutput struct {
+	AllPassed bool                      `json:"allPassed"`
+	TestCases map[string]testCaseOutput `json:"testCases"`
+}
+
+type testCaseOutput struct {
+	Passed  bool   `json:"passed"`
+	Message string `json:"message"`
+}
+
+func formatFlags(attVariant, csp, jsEndpoint string) string {
+	var sb strings.Builder
+	sb.WriteString("Using Flags:\n")
+	sb.WriteString(fmt.Sprintf("\tjs-endpoint: %s\n", jsEndpoint))
+	sb.WriteString(fmt.Sprintf("\tcsp: %s\n", csp))
+	sb.WriteString(fmt.Sprintf("\tvariant: %s\n", attVariant))
+	return sb.String()
+}
+
+// JoinFromUnattestedNode simulates a join request from a Node that uses a stub issuer
+// and thus cannot be attested correctly.
+func JoinFromUnattestedNode(attVariant, csp, jsEndpoint string) error {
+	log := logger.New(logger.JSONLog, zapcore.DebugLevel)
+	joiner, err := newMaliciousJoiner(attVariant, csp, jsEndpoint, log)
+	if err != nil {
+		return fmt.Errorf("creating malicious joiner: %w", err)
+	}
+
+	_, err = joiner.join(context.Background())
+	if err != nil {
+		return fmt.Errorf("joining cluster: %w", err)
+	}
+	return nil
+}
+
+// newMaliciousJoiner creates a new malicious joiner, i.e. a simulated node that issues
+// an invalid join request.
+func newMaliciousJoiner(attVariant, csp, endpoint string, log *logger.Logger) (*maliciousJoiner, error) {
+	var attVariantOid variant.Variant
+	var err error
+	if strings.EqualFold(attVariant, "default") {
+		attVariantOid = variant.GetDefaultAttestation(cloudprovider.FromString(csp))
+	} else {
+		attVariantOid, err = variant.FromString(attVariant)
+		if err != nil {
+			return nil, fmt.Errorf("parsing attestation variant: %w", err)
+		}
+	}
+
+	issuer := newFakeIssuer(attVariantOid)
+
+	return &maliciousJoiner{
+		endpoint: endpoint,
+		logger:   log,
+		dialer:   dialer.New(issuer, nil, &net.Dialer{}),
+	}, nil
+}
+
+// maliciousJoiner simulates a malicious node joining a cluster.
+type maliciousJoiner struct {
+	endpoint string
+	logger   *logger.Logger
+	dialer   *dialer.Dialer
+}
+
+// join issues a join request to the join service endpoint.
+func (j *maliciousJoiner) join(ctx context.Context) (*joinproto.IssueJoinTicketResponse, error) {
+	j.logger.Debugf("Dialing join service endpoint %s", j.endpoint)
+	conn, err := j.dialer.Dial(ctx, j.endpoint)
+	if err != nil {
+		return nil, fmt.Errorf("dialing join service endpoint: %w", err)
+	}
+	defer conn.Close()
+	j.logger.Debugf("Successfully dialed join service endpoint %s", j.endpoint)
+
+	protoClient := joinproto.NewAPIClient(conn)
+
+	j.logger.Debugf("Issuing join ticket")
+	req := &joinproto.IssueJoinTicketRequest{
+		DiskUuid:           "",
+		CertificateRequest: []byte{},
+		IsControlPlane:     false,
+	}
+	res, err := protoClient.IssueJoinTicket(ctx, req)
+	j.logger.Debugf("Got join ticket response: %+v", res)
+	if err != nil {
+		return nil, fmt.Errorf("issuing join ticket: %w", err)
+	}
+
+	return res, nil
+}
+
+// newFakeIssuer creates a new fake issuer for a given attestation variant.
+func newFakeIssuer(oid variant.Getter) *fakeIssuer {
+	return &fakeIssuer{oid}
+}
+
+// fakeIssuer simulates an issuer that issues a fake / invalid attestation document.
+type fakeIssuer struct {
+	variant.Getter
+}
+
+// Issue issues a fake attestation document.
+func (i *fakeIssuer) Issue(_ context.Context, userData, nonce []byte) ([]byte, error) {
+	return json.Marshal(fakeAttestationDoc{UserData: userData, Nonce: nonce})
+}
+
+// fakeAttestationDoc is a fake attestation document.
+type fakeAttestationDoc struct {
+	UserData []byte
+	Nonce    []byte
+}