diff --git a/internal/cloud/azure/BUILD.bazel b/internal/cloud/azure/BUILD.bazel index ff8fbea92..d157af807 100644 --- a/internal/cloud/azure/BUILD.bazel +++ b/internal/cloud/azure/BUILD.bazel @@ -7,6 +7,8 @@ go_library( "azure.go", "imds.go", "interface.go", + "iptables_cross.go", + "iptables_linux.go", ], importpath = "github.com/edgelesssys/constellation/v2/internal/cloud/azure", visibility = ["//:__subpackages__"], @@ -20,9 +22,17 @@ go_library( "@com_github_azure_azure_sdk_for_go_sdk_azidentity//:azidentity", "@com_github_azure_azure_sdk_for_go_sdk_resourcemanager_compute_armcompute_v5//:armcompute", "@com_github_azure_azure_sdk_for_go_sdk_resourcemanager_network_armnetwork_v5//:armnetwork", - "@io_k8s_kubernetes//pkg/util/iptables", - "@io_k8s_utils//exec", - ], + ] + select({ + "@io_bazel_rules_go//go/platform:android": [ + "@io_k8s_kubernetes//pkg/util/iptables", + "@io_k8s_utils//exec", + ], + "@io_bazel_rules_go//go/platform:linux": [ + "@io_k8s_kubernetes//pkg/util/iptables", + "@io_k8s_utils//exec", + ], + "//conditions:default": [], + }), ) go_test( diff --git a/internal/cloud/azure/azure.go b/internal/cloud/azure/azure.go index 52a468471..85f718026 100644 --- a/internal/cloud/azure/azure.go +++ b/internal/cloud/azure/azure.go @@ -19,7 +19,6 @@ import ( "context" "errors" "fmt" - "log/slog" "path" "strconv" @@ -31,8 +30,6 @@ import ( "github.com/edgelesssys/constellation/v2/internal/cloud/metadata" "github.com/edgelesssys/constellation/v2/internal/constants" "github.com/edgelesssys/constellation/v2/internal/role" - "k8s.io/kubernetes/pkg/util/iptables" - "k8s.io/utils/exec" ) // Cloud provides Azure metadata and API access. @@ -439,68 +436,6 @@ func (c *Cloud) getLoadBalancerDNSName(ctx context.Context) (string, error) { } */ -// PrepareControlPlaneNode sets up iptables for the control plane node only -// if an internal load balancer is used. -// -// This is needed since during `kubeadm init` the API server must talk to the -// kubeAPIEndpoint, which is the load balancer IP address. During that time, the -// only healthy VM is the VM itself. Therefore, traffic is sent to the load balancer -// and the 5-tuple is (VM IP, , LB IP, 6443, TCP). -// Now the load balancer does not re-write the source IP address only the destination (DNAT). -// Therefore the 5-tuple is (VM IP, , VM IP, 6443, TCP). -// Now the VM responds to the SYN packet with a SYN-ACK packet, but the outgoing -// connection waits on a response from the load balancer and not the VM therefore -// dropping the packet. -// -// OpenShift also uses the same mechanism to redirect traffic to the API server: -// https://github.com/openshift/machine-config-operator/blob/e453bd20bac0e48afa74e9a27665abaf454d93cd/templates/master/00-master/azure/files/opt-libexec-openshift-azure-routes-sh.yaml -func (c *Cloud) PrepareControlPlaneNode(ctx context.Context, log *slog.Logger) error { - selfMetadata, err := c.Self(ctx) - if err != nil { - return fmt.Errorf("failed to get self metadata: %w", err) - } - - // skipping iptables setup for worker nodes - if selfMetadata.Role != role.ControlPlane { - log.Info("not a control plane node, skipping iptables setup") - return nil - } - - // skipping iptables setup if no internal LB exists e.g. - // for public LB architectures - loadbalancerIP, err := c.getLoadBalancerPrivateIP(ctx) - if err != nil { - log.With(slog.Any("error", err)).Warn("skipping iptables setup, failed to get load balancer private IP") - return nil - } - - log.Info(fmt.Sprintf("Setting up iptables for control plane node with load balancer IP %s", loadbalancerIP)) - - iptablesExec := iptables.New(exec.New(), iptables.ProtocolIPv4) - if err != nil { - return fmt.Errorf("failed to create iptables client: %w", err) - } - - const chainName = "azure-lb-nat" - if _, err := iptablesExec.EnsureChain(iptables.TableNAT, chainName); err != nil { - return fmt.Errorf("failed to create iptables chain: %w", err) - } - - if _, err := iptablesExec.EnsureRule(iptables.Append, iptables.TableNAT, "PREROUTING", "-j", chainName); err != nil { - return fmt.Errorf("failed to add rule to iptables chain: %w", err) - } - - if _, err := iptablesExec.EnsureRule(iptables.Append, iptables.TableNAT, "OUTPUT", "-j", chainName); err != nil { - return fmt.Errorf("failed to add rule to iptables chain: %w", err) - } - - if _, err := iptablesExec.EnsureRule(iptables.Append, iptables.TableNAT, chainName, "--dst", loadbalancerIP, "-p", "tcp", "--dport", "6443", "-j", "REDIRECT"); err != nil { - return fmt.Errorf("failed to add rule to iptables chain: %w", err) - } - - return nil -} - // convertToInstanceMetadata converts a armcomputev2.VirtualMachineScaleSetVM to a metadata.InstanceMetadata. func convertToInstanceMetadata(vm armcompute.VirtualMachineScaleSetVM, networkInterfaces []armnetwork.Interface, ) (metadata.InstanceMetadata, error) { diff --git a/internal/cloud/azure/iptables_cross.go b/internal/cloud/azure/iptables_cross.go new file mode 100644 index 000000000..f1caf3321 --- /dev/null +++ b/internal/cloud/azure/iptables_cross.go @@ -0,0 +1,18 @@ +//go:build !linux + +/* +Copyright (c) Edgeless Systems GmbH + +SPDX-License-Identifier: AGPL-3.0-only +*/ + +package azure + +import ( + "context" + "log/slog" +) + +func (c *Cloud) PrepareControlPlaneNode(_ context.Context, _ *slog.Logger) error { + panic("azure.*Cloud.PrepareControlPlaneNode is only supported on Linux") +} diff --git a/internal/cloud/azure/iptables_linux.go b/internal/cloud/azure/iptables_linux.go new file mode 100644 index 000000000..24ada6de0 --- /dev/null +++ b/internal/cloud/azure/iptables_linux.go @@ -0,0 +1,77 @@ +//go:build linux + +/* +Copyright (c) Edgeless Systems GmbH + +SPDX-License-Identifier: AGPL-3.0-only +*/ + +package azure + +import ( + "context" + "fmt" + "log/slog" + + "github.com/edgelesssys/constellation/v2/internal/role" + "k8s.io/kubernetes/pkg/util/iptables" + "k8s.io/utils/exec" +) + +// PrepareControlPlaneNode sets up iptables for the control plane node only +// if an internal load balancer is used. +// +// This is needed since during `kubeadm init` the API server must talk to the +// kubeAPIEndpoint, which is the load balancer IP address. During that time, the +// only healthy VM is the VM itself. Therefore, traffic is sent to the load balancer +// and the 5-tuple is (VM IP, , LB IP, 6443, TCP). +// Now the load balancer does not re-write the source IP address only the destination (DNAT). +// Therefore the 5-tuple is (VM IP, , VM IP, 6443, TCP). +// Now the VM responds to the SYN packet with a SYN-ACK packet, but the outgoing +// connection waits on a response from the load balancer and not the VM therefore +// dropping the packet. +// +// OpenShift also uses the same mechanism to redirect traffic to the API server: +// https://github.com/openshift/machine-config-operator/blob/e453bd20bac0e48afa74e9a27665abaf454d93cd/templates/master/00-master/azure/files/opt-libexec-openshift-azure-routes-sh.yaml +func (c *Cloud) PrepareControlPlaneNode(ctx context.Context, log *slog.Logger) error { + selfMetadata, err := c.Self(ctx) + if err != nil { + return fmt.Errorf("failed to get self metadata: %w", err) + } + + // skipping iptables setup for worker nodes + if selfMetadata.Role != role.ControlPlane { + log.Info("not a control plane node, skipping iptables setup") + return nil + } + + // skipping iptables setup if no internal LB exists e.g. + // for public LB architectures + loadbalancerIP, err := c.getLoadBalancerPrivateIP(ctx) + if err != nil { + log.With(slog.Any("error", err)).Warn("skipping iptables setup, failed to get load balancer private IP") + return nil + } + + log.Info(fmt.Sprintf("Setting up iptables for control plane node with load balancer IP %s", loadbalancerIP)) + iptablesExec := iptables.New(exec.New(), iptables.ProtocolIPv4) + + const chainName = "azure-lb-nat" + if _, err := iptablesExec.EnsureChain(iptables.TableNAT, chainName); err != nil { + return fmt.Errorf("failed to create iptables chain: %w", err) + } + + if _, err := iptablesExec.EnsureRule(iptables.Append, iptables.TableNAT, "PREROUTING", "-j", chainName); err != nil { + return fmt.Errorf("failed to add rule to iptables chain: %w", err) + } + + if _, err := iptablesExec.EnsureRule(iptables.Append, iptables.TableNAT, "OUTPUT", "-j", chainName); err != nil { + return fmt.Errorf("failed to add rule to iptables chain: %w", err) + } + + if _, err := iptablesExec.EnsureRule(iptables.Append, iptables.TableNAT, chainName, "--dst", loadbalancerIP, "-p", "tcp", "--dport", "6443", "-j", "REDIRECT"); err != nil { + return fmt.Errorf("failed to add rule to iptables chain: %w", err) + } + + return nil +}