fix cilium WireGuard Pod2Pod connectivity

coordinator/kubernetes/k8sapi/util.go

@@ -4,9 +4,11 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"net/http"
 	"os"
 	"os/exec"
 	"regexp"
+	"strings"
 	"time"
 
 	"github.com/edgelesssys/constellation/coordinator/kubernetes/k8sapi/resources"
@@ -160,21 +162,57 @@ func (k *KubernetesUtil) setupGCPPodNetwork(ctx context.Context, nodeName, nodeP
 		return err
 	}
 
-	err = exec.CommandContext(ctx, kubectlPath, "--kubeconfig", kubeConfig, "-n", "kube-system", "patch", "deployment", "coredns", "--type", "json", "-p", "[{\"op\":\"add\",\"path\":\"/spec/template/spec/tolerations/-\",\"value\":{\"key\":\"node.kubernetes.io/network-unavailable\",\"value\":\"\",\"effect\":\"NoSchedule\"}}]").Run()
-	if err != nil {
-		return err
-	}
-
-	ciliumInstall := exec.CommandContext(ctx, "cilium", "install", "--ipam", "kubernetes", "--ipv4-native-routing-cidr", subnetworkPodCIDR, "--helm-set", "endpointRoutes.enabled=true,tunnel=disabled,encryption.enabled=true,encryption.type=wireguard,l7Proxy=false")
+	ciliumInstall := exec.CommandContext(ctx, "cilium", "install", "--ipam", "kubernetes", "--ipv4-native-routing-cidr", subnetworkPodCIDR,
+		"--helm-set", "endpointRoutes.enabled=true,tunnel=disabled,encryption.enabled=true,encryption.type=wireguard,l7Proxy=false")
 	ciliumInstall.Env = append(os.Environ(), "KUBECONFIG="+kubeConfig)
 	out, err = ciliumInstall.CombinedOutput()
 	if err != nil {
 		err = errors.New(string(out))
 		return err
 	}
+
 	return nil
 }
 
+// FixCilium fixes https://github.com/cilium/cilium/issues/19958 but instead of a rollout restart of
+// the cilium daemonset, it only restarts the local cilium pod.
+func (k *KubernetesUtil) FixCilium(nodeNameK8s string) {
+	// wait for cilium pod to be healthy
+	for {
+		time.Sleep(5 * time.Second)
+		resp, err := http.Get("http://127.0.0.1:9876/healthz")
+		if err != nil {
+			fmt.Printf("waiting for local cilium daemonset pod not healthy: %v\n", err)
+			continue
+		}
+		resp.Body.Close()
+		if resp.StatusCode == 200 {
+			break
+		}
+	}
+
+	// get cilium pod name
+	out, err := exec.CommandContext(context.Background(), "/bin/bash", "-c", "/run/state/bin/crictl ps -o json | jq -r '.containers[] | select(.metadata.name == \"cilium-agent\") | .podSandboxId'").CombinedOutput()
+	if err != nil {
+		fmt.Printf("getting pod id failed: %v: %v\n", err, string(out))
+		return
+	}
+	outLines := strings.Split(string(out), "\n")
+	fmt.Println(outLines)
+	podID := outLines[len(outLines)-2]
+
+	// stop and delete pod
+	out, err = exec.CommandContext(context.Background(), "/run/state/bin/crictl", "stopp", podID).CombinedOutput()
+	if err != nil {
+		fmt.Printf("stopping cilium agent pod failed: %v: %v\n", err, string(out))
+		return
+	}
+	out, err = exec.CommandContext(context.Background(), "/run/state/bin/crictl", "rmp", podID).CombinedOutput()
+	if err != nil {
+		fmt.Printf("removing cilium agent pod failed: %v: %v\n", err, string(out))
+	}
+}
+
 func (k *KubernetesUtil) setupQemuPodNetwork(ctx context.Context) error {
 	ciliumInstall := exec.CommandContext(ctx, "cilium", "install", "--encryption", "wireguard", "--helm-set", "ipam.operator.clusterPoolIPv4PodCIDRList=10.244.0.0/16,endpointRoutes.enabled=true")
 	ciliumInstall.Env = append(os.Environ(), "KUBECONFIG="+kubeConfig)
@@ -248,6 +286,7 @@ func (k *KubernetesUtil) JoinCluster(ctx context.Context, joinConfig []byte) err
 		}
 		return fmt.Errorf("kubeadm join failed: %w", err)
 	}
+
 	return nil
 }
 

coordinator/kubernetes/k8sutil.go

@@ -23,4 +23,5 @@ type clusterUtil interface {
 	RestartKubelet() error
 	GetControlPlaneJoinCertificateKey(ctx context.Context) (string, error)
 	CreateJoinToken(ctx context.Context, ttl time.Duration) (*kubeadm.BootstrapTokenDiscovery, error)
+	FixCilium(nodeNameK8s string)
 }

coordinator/kubernetes/kubernetes.go

@@ -157,6 +157,8 @@ func (k *KubeWrapper) InitCluster(ctx context.Context, autoscalingNodeGroups []s
 		return fmt.Errorf("failed to setup access-manager: %w", err)
 	}
 
+	go k.clusterUtil.FixCilium(nodeName)
+
 	return nil
 }
 
@@ -210,6 +212,8 @@ func (k *KubeWrapper) JoinCluster(ctx context.Context, args *kubeadm.BootstrapTo
 		return fmt.Errorf("joining cluster failed: %v %w ", string(joinConfigYAML), err)
 	}
 
+	go k.clusterUtil.FixCilium(nodeName)
+
 	return nil
 }
 

coordinator/kubernetes/kubernetes_test.go

@@ -566,6 +566,9 @@ func (s *stubClusterUtil) CreateJoinToken(ctx context.Context, ttl time.Duration
 	return s.createJoinTokenResponse, s.createJoinTokenErr
 }
 
+func (s *stubClusterUtil) FixCilium(nodeName string) {
+}
+
 type stubConfigProvider struct {
 	InitConfig k8sapi.KubeadmInitYAML
 	JoinConfig k8sapi.KubeadmJoinYAML