From 45479b307e107b61e93862f6e177ad4a7c0a5e96 Mon Sep 17 00:00:00 2001
From: 3u13r <lc@edgeless.systems>
Date: Tue, 2 Jan 2024 14:33:03 +0100
Subject: [PATCH] helm: masq traffic to the mini-qemu-metadata container so
 that the join-service can retrieve it's metadata (#2782)

* helm: masq traffic to the mini-qemu-metadata container

* ci: fix waiting for nodes in miniconstellation e2e test
---
 e2e/miniconstellation/test-remote.sh     | 35 +++++++++++++++---------
 internal/constellation/helm/overrides.go | 11 +++++++-
 2 files changed, 32 insertions(+), 14 deletions(-)

diff --git a/e2e/miniconstellation/test-remote.sh b/e2e/miniconstellation/test-remote.sh
index 7166dec1c..3cef1fd96 100755
--- a/e2e/miniconstellation/test-remote.sh
+++ b/e2e/miniconstellation/test-remote.sh
@@ -49,20 +49,29 @@ echo "Done waiting."
 
 export KUBECONFIG="${PWD}/constellation-admin.conf"
 
-# Wait for nodes to actually show up in K8s
-count=0
-until kubectl wait --for=condition=Ready --timeout=2s nodes control-plane-0 2> /dev/null || [[ ${count} -eq 30 ]]; do
-  echo "Control-planes are not registered in Kubernetes yet. Waiting..."
-  sleep 10
-  count=$((count + 1))
-done
-
-count=0
-until kubectl wait --for=condition=Ready --timeout=2s nodes worker-0 2> /dev/null || [[ ${count} -eq 30 ]]; do
-  echo "Worker nodes are not registered in Kubernetes yet. Waiting..."
-  sleep 10
-  count=$((count + 1))
+# Wait for nodes to actually show up in K8s (taken from .github/actions/constellation_create/action.yml)
+echo "::group::Wait for nodes"
+NODES_COUNT=2
+JOINWAIT=0
+JOINTIMEOUT="600" # 10 minutes timeout for all nodes to join
+until [[ "$(kubectl get nodes -o json | jq '.items | length')" == "${NODES_COUNT}" ]] || [[ $JOINWAIT -gt $JOINTIMEOUT ]]; do
+  echo "$(kubectl get nodes -o json | jq '.items | length')/${NODES_COUNT} nodes have joined.. waiting.."
+  JOINWAIT=$((JOINWAIT + 30))
+  sleep 30
 done
+if [[ $JOINWAIT -gt $JOINTIMEOUT ]]; then
+  echo "Timed out waiting for nodes to join"
+  exit 1
+fi
+echo "$(kubectl get nodes -o json | jq '.items | length')/${NODES_COUNT} nodes have joined"
+if ! kubectl wait --for=condition=ready --all nodes --timeout=20m; then
+  kubectl get pods -n kube-system
+  kubectl get events -n kube-system
+  echo "::error::kubectl wait timed out before all nodes became ready"
+  echo "::endgroup::"
+  exit 1
+fi
+echo "::endgroup::"
 
 # Wait for deployments
 kubectl -n kube-system wait --for=condition=Available=True --timeout=180s deployment coredns
diff --git a/internal/constellation/helm/overrides.go b/internal/constellation/helm/overrides.go
index 60c8be7da..4672685e1 100644
--- a/internal/constellation/helm/overrides.go
+++ b/internal/constellation/helm/overrides.go
@@ -59,9 +59,18 @@ func extraCiliumValues(provider cloudprovider.Provider, conformanceMode bool, ou
 	extraVals["encryption"] = map[string]any{
 		"strictMode": strictMode,
 	}
+
+	// On QEMU e.g. the join-service must talk to our mini-qemu-metadata docker container
+	// This container runs inside the node CIDR, so we need to masq any pod traffic to it
+	// with the node's IP address. To archive that, we override Cilium's default masq ranges
+	// with an empty list.
+	masqCIDRs := []string{}
+	if provider != cloudprovider.QEMU {
+		masqCIDRs = append(masqCIDRs, output.IPCidrNode)
+	}
 	extraVals["ipMasqAgent"] = map[string]any{
 		"config": map[string]any{
-			"nonMasqueradeCIDRs": []string{output.IPCidrNode},
+			"nonMasqueradeCIDRs": masqCIDRs,
 		},
 	}