Add pod disruption budgets so the cluster-autoscaler is able to move kube-admin namespaced resources (#2781)

* helm: refactor cilium helm values

* helm: add pod disruption budgets

internal/constellation/helm/BUILD.bazel

@@ -450,6 +450,7 @@ go_library(
         "charts/cert-manager/templates/controller-config.yaml",
         "charts/cert-manager/templates/poddisruptionbudget.yaml",
         "charts/cert-manager/templates/webhook-poddisruptionbudget.yaml",
+        "charts/edgeless/constellation-services/charts/autoscaler/templates/coredns-pdb.yaml",
     ],
     importpath = "github.com/edgelesssys/constellation/v2/internal/constellation/helm",
     visibility = ["//:__subpackages__"],

internal/constellation/helm/charts/edgeless/constellation-services/charts/autoscaler/templates/coredns-pdb.yaml

@@ -0,0 +1,10 @@
+apiVersion: policy/v1
+kind: PodDisruptionBudget
+metadata:
+  name: coredns-pdb
+  namespace: "kube-system"
+spec:
+  maxUnavailable: 1
+  selector:
+    matchLabels:
+      k8s-app: kube-dns

internal/constellation/helm/loader.go

@@ -185,10 +185,9 @@ func (i *chartLoader) loadRelease(info chartInfo, helmWaitMode WaitMode) (releas
 
 	switch info.releaseName {
 	case ciliumInfo.releaseName:
-		var ok bool
+		values, err = i.loadCiliumValues(i.csp)
-		values, ok = ciliumVals[i.csp.String()]
+		if err != nil {
-		if !ok {
+			return release{}, fmt.Errorf("loading cilium values: %w", err)
-			return release{}, fmt.Errorf("cilium values for csp %q not found", i.csp.String())
 		}
 	case certManagerInfo.releaseName:
 		values = i.loadCertManagerValues()
@@ -230,9 +229,17 @@ func (i *chartLoader) loadCertManagerValues() map[string]any {
 		"tolerations": controlPlaneTolerations,
 		"webhook": map[string]any{
 			"tolerations": controlPlaneTolerations,
+			"podDisruptionBudget": map[string]any{
+				"enabled": true,
+			},
+			"replicaCount": 2,
 		},
 		"cainjector": map[string]any{
 			"tolerations": controlPlaneTolerations,
+			"podDisruptionBudget": map[string]any{
+				"enabled": true,
+			},
+			"replicaCount": 2,
 		},
 		"startupapicheck": map[string]any{
 			"timeout": "5m",
@@ -241,6 +248,10 @@ func (i *chartLoader) loadCertManagerValues() map[string]any {
 			},
 			"tolerations": controlPlaneTolerations,
 		},
+		"podDisruptionBudget": map[string]any{
+			"enabled": true,
+		},
+		"replicaCount": 2,
 	}
 }
 
@@ -321,6 +332,89 @@ func (i *chartLoader) cspTags() map[string]any {
 	}
 }
 
+func (i *chartLoader) loadCiliumValues(cloudprovider.Provider) (map[string]any, error) {
+	sharedConfig := map[string]any{
+		"extraArgs": []string{"--node-encryption-opt-out-labels=invalid.label"},
+		"endpointRoutes": map[string]any{
+			"enabled": true,
+		},
+		"l7Proxy": false,
+		"image": map[string]any{
+			"repository": "ghcr.io/3u13r/cilium",
+			"suffix":     "",
+			"tag":        "v1.15.0-pre.2-edg.1",
+			"digest":     "sha256:eebf631fd0f27e1f28f1fdeb2e049f2c83b887381466245c4b3e26440daefa27",
+			"useDigest":  true,
+		},
+		"operator": map[string]any{
+			"image": map[string]any{
+				"repository":    "ghcr.io/3u13r/operator",
+				"tag":           "v1.15.0-pre.2-edg.1",
+				"suffix":        "",
+				"genericDigest": "sha256:bfaeac2e05e8c38f439b0fbc36558fd8d11602997f2641423e8d86bd7ac6a88c",
+				"useDigest":     true,
+			},
+			"podDisruptionBudget": map[string]any{
+				"enabled": true,
+			},
+		},
+		"encryption": map[string]any{
+			"enabled":        true,
+			"type":           "wireguard",
+			"nodeEncryption": true,
+			"strictMode": map[string]any{
+				"enabled":                   true,
+				"podCIDRList":               []string{"10.244.0.0/16"},
+				"allowRemoteNodeIdentities": false,
+			},
+		},
+		"ipam": map[string]any{
+			"operator": map[string]any{
+				"clusterPoolIPv4PodCIDRList": []string{
+					"10.244.0.0/16",
+				},
+			},
+		},
+		"bpf": map[string]any{
+			"masquerade": true,
+		},
+		"ipMasqAgent": map[string]any{
+			"enabled": true,
+			"config": map[string]any{
+				"masqLinkLocal": true,
+			},
+		},
+		"kubeProxyReplacement":                "strict",
+		"enableCiliumEndpointSlice":           true,
+		"kubeProxyReplacementHealthzBindAddr": "0.0.0.0:10256",
+	}
+	cspOverrideConfigs := map[string]map[string]any{
+		cloudprovider.AWS.String():   {},
+		cloudprovider.Azure.String(): {},
+		cloudprovider.GCP.String(): {
+			"tunnel": "disabled",
+			"encryption": map[string]any{
+				"strictMode": map[string]any{
+					"podCIDRList": []string{""},
+				},
+			},
+			"ipam": map[string]any{
+				"mode": "kubernetes",
+			},
+		},
+		cloudprovider.OpenStack.String(): {},
+		cloudprovider.QEMU.String(): {
+			"extraArgs": []string{""},
+		},
+	}
+
+	cspValues, ok := cspOverrideConfigs[i.csp.String()]
+	if !ok {
+		return nil, fmt.Errorf("cilium values for csp %q not found", i.csp.String())
+	}
+	return mergeMaps(sharedConfig, cspValues), nil
+}
+
 // updateVersions changes all versions of direct dependencies that are set to "0.0.0" to newVersion.
 func updateVersions(chart *chart.Chart, newVersion semver.Semver) {
 	chart.Metadata.Version = newVersion.String()

internal/constellation/helm/testdata/AWS/constellation-services/charts/autoscaler/templates/coredns-pdb.yaml

@@ -0,0 +1,10 @@
+apiVersion: policy/v1
+kind: PodDisruptionBudget
+metadata:
+  name: coredns-pdb
+  namespace: kube-system
+spec:
+  maxUnavailable: 1
+  selector:
+    matchLabels:
+      k8s-app: kube-dns

internal/constellation/helm/testdata/Azure/constellation-services/charts/autoscaler/templates/coredns-pdb.yaml

@@ -0,0 +1,10 @@
+apiVersion: policy/v1
+kind: PodDisruptionBudget
+metadata:
+  name: coredns-pdb
+  namespace: kube-system
+spec:
+  maxUnavailable: 1
+  selector:
+    matchLabels:
+      k8s-app: kube-dns

internal/constellation/helm/testdata/GCP/constellation-services/charts/autoscaler/templates/coredns-pdb.yaml

@@ -0,0 +1,10 @@
+apiVersion: policy/v1
+kind: PodDisruptionBudget
+metadata:
+  name: coredns-pdb
+  namespace: kube-system
+spec:
+  maxUnavailable: 1
+  selector:
+    matchLabels:
+      k8s-app: kube-dns

internal/constellation/helm/values.go

@@ -6,265 +6,6 @@ SPDX-License-Identifier: AGPL-3.0-only
 
 package helm
 
-import "github.com/edgelesssys/constellation/v2/internal/cloud/cloudprovider"
-
-// Values for the Cilium Helm releases for AWS.
-var ciliumVals = map[string]map[string]any{
-	cloudprovider.AWS.String(): {
-		"endpointRoutes": map[string]any{
-			"enabled": true,
-		},
-		"extraArgs": []string{"--node-encryption-opt-out-labels=invalid.label"},
-		"encryption": map[string]any{
-			"enabled":        true,
-			"type":           "wireguard",
-			"nodeEncryption": true,
-			"strictMode": map[string]any{
-				"enabled":                   true,
-				"allowRemoteNodeIdentities": false,
-				"podCIDRList":               []string{"10.244.0.0/16"},
-			},
-		},
-		"l7Proxy": false,
-		"ipam": map[string]any{
-			"operator": map[string]any{
-				"clusterPoolIPv4PodCIDRList": []string{
-					"10.244.0.0/16",
-				},
-			},
-		},
-		"image": map[string]any{
-			"repository": "ghcr.io/3u13r/cilium",
-			"suffix":     "",
-			"tag":        "v1.15.0-pre.2-edg.1",
-			"digest":     "sha256:eebf631fd0f27e1f28f1fdeb2e049f2c83b887381466245c4b3e26440daefa27",
-			"useDigest":  true,
-		},
-		"operator": map[string]any{
-			"image": map[string]any{
-				"repository":    "ghcr.io/3u13r/operator",
-				"tag":           "v1.15.0-pre.2-edg.1",
-				"suffix":        "",
-				"genericDigest": "sha256:bfaeac2e05e8c38f439b0fbc36558fd8d11602997f2641423e8d86bd7ac6a88c",
-				"useDigest":     true,
-			},
-		},
-		"bpf": map[string]any{
-			"masquerade": true,
-		},
-		"ipMasqAgent": map[string]any{
-			"enabled": true,
-			"config": map[string]any{
-				"masqLinkLocal": true,
-			},
-		},
-		"kubeProxyReplacement":                "strict",
-		"enableCiliumEndpointSlice":           true,
-		"kubeProxyReplacementHealthzBindAddr": "0.0.0.0:10256",
-	},
-	cloudprovider.Azure.String(): {
-		"endpointRoutes": map[string]any{
-			"enabled": true,
-		},
-		"extraArgs": []string{"--node-encryption-opt-out-labels=invalid.label"},
-		"encryption": map[string]any{
-			"enabled":        true,
-			"type":           "wireguard",
-			"nodeEncryption": true,
-			"strictMode": map[string]any{
-				"enabled":                   true,
-				"allowRemoteNodeIdentities": false,
-				"podCIDRList":               []string{"10.244.0.0/16"},
-			},
-		},
-		"l7Proxy": false,
-		"ipam": map[string]any{
-			"operator": map[string]any{
-				"clusterPoolIPv4PodCIDRList": []string{
-					"10.244.0.0/16",
-				},
-			},
-		},
-		"image": map[string]any{
-			"repository": "ghcr.io/3u13r/cilium",
-			"suffix":     "",
-			"tag":        "v1.15.0-pre.2-edg.1",
-			"digest":     "sha256:eebf631fd0f27e1f28f1fdeb2e049f2c83b887381466245c4b3e26440daefa27",
-			"useDigest":  true,
-		},
-		"operator": map[string]any{
-			"image": map[string]any{
-				"repository":    "ghcr.io/3u13r/operator",
-				"tag":           "v1.15.0-pre.2-edg.1",
-				"suffix":        "",
-				"genericDigest": "sha256:bfaeac2e05e8c38f439b0fbc36558fd8d11602997f2641423e8d86bd7ac6a88c",
-				"useDigest":     true,
-			},
-		},
-		"bpf": map[string]any{
-			"masquerade": true,
-		},
-		"ipMasqAgent": map[string]any{
-			"enabled": true,
-			"config": map[string]any{
-				"masqLinkLocal": true,
-			},
-		},
-		"kubeProxyReplacement":                "strict",
-		"enableCiliumEndpointSlice":           true,
-		"kubeProxyReplacementHealthzBindAddr": "0.0.0.0:10256",
-	},
-	cloudprovider.GCP.String(): {
-		"endpointRoutes": map[string]any{
-			"enabled": true,
-		},
-		"extraArgs": []string{"--node-encryption-opt-out-labels=invalid.label"},
-		"tunnel":    "disabled",
-		"encryption": map[string]any{
-			"enabled":        true,
-			"type":           "wireguard",
-			"nodeEncryption": true,
-			"strictMode": map[string]any{
-				"enabled":                   true,
-				"allowRemoteNodeIdentities": false,
-			},
-		},
-		"image": map[string]any{
-			"repository": "ghcr.io/3u13r/cilium",
-			"suffix":     "",
-			"tag":        "v1.15.0-pre.2-edg.1",
-			"digest":     "sha256:eebf631fd0f27e1f28f1fdeb2e049f2c83b887381466245c4b3e26440daefa27",
-			"useDigest":  true,
-		},
-		"operator": map[string]any{
-			"image": map[string]any{
-				"repository":    "ghcr.io/3u13r/operator",
-				"suffix":        "",
-				"tag":           "v1.15.0-pre.2-edg.1",
-				"genericDigest": "sha256:bfaeac2e05e8c38f439b0fbc36558fd8d11602997f2641423e8d86bd7ac6a88c",
-				"useDigest":     true,
-			},
-		},
-		"l7Proxy": false,
-		"ipam": map[string]any{
-			"mode": "kubernetes",
-		},
-		"bpf": map[string]any{
-			"masquerade": true,
-		},
-		"ipMasqAgent": map[string]any{
-			"enabled": true,
-			"config": map[string]any{
-				"masqLinkLocal": true,
-			},
-		},
-		"kubeProxyReplacement":                "strict",
-		"enableCiliumEndpointSlice":           true,
-		"kubeProxyReplacementHealthzBindAddr": "0.0.0.0:10256",
-	},
-	cloudprovider.OpenStack.String(): {
-		"endpointRoutes": map[string]any{
-			"enabled": true,
-		},
-		"extraArgs": []string{"--node-encryption-opt-out-labels=invalid.label"},
-		"encryption": map[string]any{
-			"enabled":        true,
-			"type":           "wireguard",
-			"nodeEncryption": true,
-			"strictMode": map[string]any{
-				"enabled":     true,
-				"podCIDRList": []string{"10.244.0.0/16"},
-			},
-		},
-		"l7Proxy": false,
-		"ipam": map[string]any{
-			"operator": map[string]any{
-				"clusterPoolIPv4PodCIDRList": []string{
-					"10.244.0.0/16",
-				},
-			},
-		},
-		"image": map[string]any{
-			"repository": "ghcr.io/3u13r/cilium",
-			"suffix":     "",
-			"tag":        "v1.15.0-pre.2-edg.1",
-			"digest":     "sha256:eebf631fd0f27e1f28f1fdeb2e049f2c83b887381466245c4b3e26440daefa27",
-			"useDigest":  true,
-		},
-		"operator": map[string]any{
-			"image": map[string]any{
-				"repository":    "ghcr.io/3u13r/operator",
-				"tag":           "v1.15.0-pre.2-edg.1",
-				"suffix":        "",
-				"genericDigest": "sha256:bfaeac2e05e8c38f439b0fbc36558fd8d11602997f2641423e8d86bd7ac6a88c",
-				"useDigest":     true,
-			},
-		},
-		"bpf": map[string]any{
-			"masquerade": true,
-		},
-		"ipMasqAgent": map[string]any{
-			"enabled": true,
-			"config": map[string]any{
-				"masqLinkLocal": true,
-			},
-		},
-		"kubeProxyReplacement":                "strict",
-		"enableCiliumEndpointSlice":           true,
-		"kubeProxyReplacementHealthzBindAddr": "0.0.0.0:10256",
-	},
-	cloudprovider.QEMU.String(): {
-		"endpointRoutes": map[string]any{
-			"enabled": true,
-		},
-		"encryption": map[string]any{
-			"enabled":        true,
-			"type":           "wireguard",
-			"nodeEncryption": true,
-			"strictMode": map[string]any{
-				"enabled":     true,
-				"podCIDRList": []string{"10.244.0.0/16"},
-			},
-		},
-		"image": map[string]any{
-			"repository": "ghcr.io/3u13r/cilium",
-			"suffix":     "",
-			"tag":        "v1.15.0-pre.2-edg.1",
-			"digest":     "sha256:eebf631fd0f27e1f28f1fdeb2e049f2c83b887381466245c4b3e26440daefa27",
-			"useDigest":  true,
-		},
-		"operator": map[string]any{
-			"image": map[string]any{
-				"repository":    "ghcr.io/3u13r/operator",
-				"suffix":        "",
-				"tag":           "v1.15.0-pre.2-edg.1",
-				"genericDigest": "sha256:bfaeac2e05e8c38f439b0fbc36558fd8d11602997f2641423e8d86bd7ac6a88c",
-				"useDigest":     true,
-			},
-		},
-		"ipam": map[string]any{
-			"operator": map[string]any{
-				"clusterPoolIPv4PodCIDRList": []string{
-					"10.244.0.0/16",
-				},
-			},
-		},
-		"bpf": map[string]any{
-			"masquerade": true,
-		},
-		"ipMasqAgent": map[string]any{
-			"enabled": true,
-			"config": map[string]any{
-				"masqLinkLocal": true,
-			},
-		},
-		"kubeProxyReplacement":                "strict",
-		"enableCiliumEndpointSlice":           true,
-		"kubeProxyReplacementHealthzBindAddr": "0.0.0.0:10256",
-		"l7Proxy":                             false,
-	},
-}
-
 var controlPlaneNodeSelector = map[string]any{"node-role.kubernetes.io/control-plane": ""}
 
 var controlPlaneTolerations = []map[string]any{