helm: fix kubeadm bugs caused by CoreDNS installation (#3353)

* helm: rename CoreDNS configmap

* upgrade-agent: ignore CoreDNS preflight errors

* fixup! helm: rename CoreDNS configmap

internal/constellation/helm.go

@@ -60,6 +60,21 @@ func (a *Applier) AnnotateCoreDNSResources(ctx context.Context) error {
 	return nil
 }
 
+// CleanupCoreDNSResources removes CoreDNS resources that are not managed by Helm.
+//
+// This is only required when CoreDNS was installed by kubeadm directly.
+// TODO(burgerdev): remove after v2.19 is released.
+func (a *Applier) CleanupCoreDNSResources(ctx context.Context) error {
+	err := a.dynamicClient.
+		Resource(schema.GroupVersionResource{Group: "", Version: "v1", Resource: "configmaps"}).
+		Namespace("kube-system").
+		Delete(ctx, "coredns", v1.DeleteOptions{})
+	if !k8serrors.IsNotFound(err) {
+		return err
+	}
+	return nil
+}
+
 // PrepareHelmCharts loads Helm charts for Constellation and returns an executor to apply them.
 func (a *Applier) PrepareHelmCharts(
 	flags helm.Options, state *state.State, serviceAccURI string, masterSecret uri.MasterSecret,

internal/constellation/helm/charts/coredns/templates/configmap.yaml

@@ -1,9 +1,4 @@
-
 apiVersion: v1
-kind: ConfigMap
-metadata:
-  name: coredns
-  namespace: kube-system
 data:
   Corefile: |
     .:53 {
@@ -26,3 +21,8 @@ data:
         reload
         loadbalance
     }
+kind: ConfigMap
+metadata:
+  creationTimestamp: null
+  name: edg-coredns
+  namespace: kube-system

internal/constellation/helm/charts/coredns/templates/deployment.yaml

@@ -104,6 +104,6 @@ spec:
           items:
           - key: Corefile
             path: Corefile
-          name: coredns
+          name: edg-coredns
         name: config-volume
 status: {}

internal/constellation/helm/corednsgen/corednsgen.go

@@ -29,6 +29,8 @@ import (
 	"sigs.k8s.io/yaml"
 )
 
+const configMapName = "edg-coredns"
+
 var chartDir = flag.String("charts", "./charts", "target directory to create charts in")
 
 func main() {
@@ -44,9 +46,9 @@ func main() {
 	writeTemplate(kubedns.CoreDNSServiceAccount, "serviceaccount.yaml")
 	writeTemplate(kubedns.CoreDNSClusterRole, "clusterrole.yaml")
 	writeTemplate(kubedns.CoreDNSClusterRoleBinding, "clusterrolebinding.yaml")
-	writeTemplate(kubedns.CoreDNSConfigMap, "configmap.yaml")
 	writeTemplate(kubedns.CoreDNSService, "service.yaml")
 
+	writeFileRelativeToChartDir(patchedConfigMap(), "templates", "configmap.yaml")
 	writeFileRelativeToChartDir(patchedDeployment(), "templates", "deployment.yaml")
 }
 
@@ -92,7 +94,25 @@ func valuesYAML() []byte {
 	return data
 }
 
-// patchedDeployment extracts the CoreDNS deployment from kubeadm and adds necessary tolerations.
+// patchedConfigMap renames the CoreDNS ConfigMap such that kubeadm does not find it.
+//
+// See https://github.com/kubernetes/kubeadm/issues/2846#issuecomment-1899942683.
+func patchedConfigMap() []byte {
+	var cm corev1.ConfigMap
+	if err := yaml.Unmarshal(parseTemplate(kubedns.CoreDNSConfigMap), &cm); err != nil {
+		log.Fatalf("Could not parse configmap: %v", err)
+	}
+
+	cm.Name = configMapName
+
+	out, err := yaml.Marshal(cm)
+	if err != nil {
+		log.Fatalf("Could not marshal patched deployment: %v", err)
+	}
+	return out
+}
+
+// patchedDeployment extracts the CoreDNS Deployment from kubeadm, adds necessary tolerations and updates the ConfigMap reference.
 func patchedDeployment() []byte {
 	var d appsv1.Deployment
 	if err := yaml.Unmarshal(parseTemplate(kubedns.CoreDNSDeployment), &d); err != nil {
@@ -104,6 +124,14 @@ func patchedDeployment() []byte {
 		{Key: "node.kubernetes.io/unreachable", Operator: corev1.TolerationOpExists, Effect: corev1.TaintEffectNoExecute, TolerationSeconds: toPtr(int64(10))},
 	}
 	d.Spec.Template.Spec.Tolerations = append(d.Spec.Template.Spec.Tolerations, tolerations...)
+
+	for i, vol := range d.Spec.Template.Spec.Volumes {
+		if vol.ConfigMap != nil {
+			vol.ConfigMap.Name = configMapName
+		}
+		d.Spec.Template.Spec.Volumes[i] = vol
+	}
+
 	out, err := yaml.Marshal(d)
 	if err != nil {
 		log.Fatalf("Could not marshal patched deployment: %v", err)