Support internal load balancers (#2388)

* arch: support internal lb on Azure

* arch: support internal lb on GCP

* helm: remove lb svc from verify deployment

* arch: support internal lb on AWS

* terraform: add jump hosts for internal lb

* cli: expose internalLoadBalancer in config

* ci: add e2e-manual-internal

* add in-cluster endpoint to terraform output

.github/actions/constellation_create/action.yml

@@ -47,6 +47,9 @@ inputs:
   refStream:
     description: "Reference and stream of the image in use"
     required: false
+  internalLoadBalancer:
+    description: "Whether to use an internal load balancer for the control plane"
+    required: false
 
 outputs:
   kubeconfig:
@@ -115,6 +118,12 @@ runs:
       run: |
         yq eval -i '(.debugCluster) = true' constellation-conf.yaml
 
+    - name: Enable internalLoadBalancer flag
+      if: inputs.internalLoadBalancer == 'true'
+      shell: bash
+      run: |
+        yq eval -i '(.internalLoadBalancer) = true' constellation-conf.yaml
+
     # Uses --force flag since the CLI currently does not have a pre-release version and is always on the latest released version.
     # However, many of our pipelines work on prerelease images. Thus the used images are newer than the CLI's version.
     # This makes the version validation in the CLI fail.

.github/actions/e2e_test/action.yml

@@ -74,6 +74,8 @@ inputs:
     default: "false"
   azureSNPEnforcementPolicy:
     description: "Enable security policy for the cluster."
+  internalLoadBalancer:
+    description: "Enable internal load balancer for the cluster."
 
 outputs:
   kubeconfig:
@@ -253,6 +255,7 @@ runs:
         azureClusterCreateCredentials: ${{ inputs.azureClusterCreateCredentials }}
         kubernetesVersion: ${{ inputs.kubernetesVersion }}
         refStream: ${{ inputs.refStream }}
+        internalLoadBalancer: ${{ inputs.internalLoadBalancer }}
 
     - name: Deploy log- and metrics-collection (Kubernetes)
       id: deploy-logcollection

.github/workflows/e2e-test-manual-internal.yml

@@ -0,0 +1,227 @@
+name: e2e test manual internal LB
+
+on:
+  workflow_dispatch:
+    inputs:
+      nodeCount:
+        description: "Number of nodes to use in the cluster. Given in format `<control-plane nodes>:<worker nodes>`."
+        default: "3:2"
+        type: string
+      cloudProvider:
+        description: "Which cloud provider to use."
+        type: choice
+        options:
+          - "gcp"
+          - "azure"
+          - "aws"
+        default: "azure"
+        required: true
+      test:
+        description: "The test to run."
+        type: choice
+        options:
+          - "sonobuoy quick"
+          - "sonobuoy full"
+          - "autoscaling"
+          - "lb"
+          - "perf-bench"
+          - "verify"
+          - "recover"
+          - "malicious join"
+          - "nop"
+        required: true
+      kubernetesVersion:
+        description: "Kubernetes version to create the cluster from."
+        default: "1.27"
+        required: true
+      cliVersion:
+        description: "Version of a released CLI to download. Leave empty to build the CLI from the checked out ref."
+        type: string
+        default: ""
+        required: false
+      imageVersion:
+        description: "Full name of OS image (CSP independent image version UID). Leave empty for latest debug image on main."
+        type: string
+        default: ""
+        required: false
+  workflow_call:
+    inputs:
+      nodeCount:
+        description: "Number of nodes to use in the cluster. Given in format `<control-plane nodes>:<worker nodes>`."
+        default: "3:2"
+        type: string
+      cloudProvider:
+        description: "Which cloud provider to use."
+        type: string
+        required: true
+      test:
+        description: "The test to run."
+        type: string
+        required: true
+      kubernetesVersion:
+        description: "Kubernetes version to create the cluster from."
+        type: string
+        required: true
+      cliVersion:
+        description: "Version of a released CLI to download. Leave empty to build the CLI from the checked out ref."
+        type: string
+        default: ""
+        required: false
+      imageVersion:
+        description: "Full name of OS image (CSP independent image version UID). Leave empty for latest debug image on main."
+        type: string
+        default: ""
+        required: false
+
+jobs:
+  split-nodeCount:
+    name: Split nodeCount
+    runs-on: ubuntu-22.04
+    permissions:
+      id-token: write
+      contents: read
+    outputs:
+      workerNodes: ${{ steps.split-nodeCount.outputs.workerNodes }}
+      controlPlaneNodes: ${{ steps.split-nodeCount.outputs.controlPlaneNodes }}
+    steps:
+      - name: Split nodeCount
+        id: split-nodeCount
+        shell: bash
+        run: |
+          nodeCount="${{ inputs.nodeCount }}"
+          workerNodes="${nodeCount##*:}"
+          controlPlaneNodes="${nodeCount%%:*}"
+
+          if [[ -z "${workerNodes}" ]] || [[ -z "{controlPlaneNodes}" ]]; then
+            echo "Invalid nodeCount input: '${nodeCount}'."
+            exit 1
+          fi
+
+          echo "workerNodes=${workerNodes}" | tee -a "$GITHUB_OUTPUT"
+          echo "controlPlaneNodes=${controlPlaneNodes}" | tee -a "$GITHUB_OUTPUT"
+
+  find-latest-image:
+    name: Select image
+    runs-on: ubuntu-22.04
+    permissions:
+      id-token: write
+      contents: read
+    outputs:
+      image: ${{ steps.find-latest-image.outputs.output }}${{ steps.check-input.outputs.image }}
+    steps:
+      - name: Check input
+        id: check-input
+        shell: bash
+        run: |
+          if [[ -z "${{ inputs.imageVersion }}" ]]; then
+            echo "Using latest debug image from main."
+            exit 0
+          else
+            echo "image=${{ inputs.imageVersion }}" | tee -a "$GITHUB_OUTPUT"
+          fi
+
+      - name: Checkout head
+        if: inputs.imageVersion == ''
+        uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3
+        with:
+          ref: ${{ !github.event.pull_request.head.repo.fork && github.head_ref || '' }}
+
+      - name: Login to AWS
+        if: inputs.imageVersion == ''
+        uses: aws-actions/configure-aws-credentials@5fd3084fc36e372ff1fff382a39b10d03659f355 # v2.2.0
+        with:
+          role-to-assume: arn:aws:iam::795746500882:role/GithubConstellationVersionsAPIRead
+          aws-region: eu-central-1
+
+      - name: Find latest image
+        id: find-latest-image
+        if: inputs.imageVersion == ''
+        uses: ./.github/actions/versionsapi
+        with:
+          command: latest
+          ref: main
+          stream: debug
+
+      - name: Is debug image?
+        id: isDebugImage
+        shell: bash
+        run: |
+          case "${{ inputs.imageVersion }}" in
+            "")
+              ;;
+            *"/stream/debug/"*)
+              ;;
+            *)
+              echo "Only debug images are supported for internal LB tests."
+              exit 1
+              ;;
+          esac
+
+  e2e-test-manual:
+    runs-on: ubuntu-22.04
+    permissions:
+      id-token: write
+      checks: write
+      contents: read
+      packages: write
+    needs: [find-latest-image, split-nodeCount]
+    if: always() && !cancelled()
+    steps:
+      - name: Install basic tools (macOS)
+        if: runner.os == 'macOS'
+        shell: bash
+        run: brew install coreutils kubectl bash terraform
+
+      - name: Checkout head
+        uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3
+        with:
+          ref: ${{ !github.event.pull_request.head.repo.fork && github.head_ref || '' }}
+
+      - name: Run manual E2E test
+        id: e2e_test
+        uses: ./.github/actions/e2e_test
+        with:
+          workerNodesCount: ${{ needs.split-nodeCount.outputs.workerNodes }}
+          controlNodesCount: ${{ needs.split-nodeCount.outputs.controlPlaneNodes }}
+          cloudProvider: ${{ inputs.cloudProvider }}
+          gcpProject: ${{ secrets.GCP_E2E_PROJECT }}
+          gcpClusterCreateServiceAccount: "constellation-e2e-cluster@constellation-331613.iam.gserviceaccount.com"
+          gcpIAMCreateServiceAccount: "constellation-iam-e2e@constellation-331613.iam.gserviceaccount.com"
+          gcpInClusterServiceAccountKey: ${{ secrets.GCP_CLUSTER_SERVICE_ACCOUNT }}
+          test: ${{ inputs.test }}
+          kubernetesVersion: ${{ inputs.kubernetesVersion }}
+          awsOpenSearchDomain: ${{ secrets.AWS_OPENSEARCH_DOMAIN }}
+          awsOpenSearchUsers: ${{ secrets.AWS_OPENSEARCH_USER }}
+          awsOpenSearchPwd: ${{ secrets.AWS_OPENSEARCH_PWD }}
+          osImage: ${{ needs.find-latest-image.outputs.image }}
+          cliVersion: ${{ inputs.cliVersion }}
+          isDebugImage: true
+          buildBuddyApiKey: ${{ secrets.BUILDBUDDY_ORG_API_KEY }}
+          azureClusterCreateCredentials: ${{ secrets.AZURE_E2E_CLUSTER_CREDENTIALS }}
+          azureIAMCreateCredentials: ${{ secrets.AZURE_E2E_IAM_CREDENTIALS }}
+          registry: ghcr.io
+          githubToken: ${{ secrets.GITHUB_TOKEN }}
+          cosignPassword: ${{ secrets.COSIGN_PASSWORD }}
+          cosignPrivateKey: ${{ secrets.COSIGN_PRIVATE_KEY }}
+          fetchMeasurements: ${{ contains(needs.find-latest-image.outputs.image, '/stream/stable/') }}
+          internalLoadBalancer: true
+
+      - name: Always terminate cluster
+        if: always()
+        uses: ./.github/actions/constellation_destroy
+        with:
+          kubeconfig: ${{ steps.e2e_test.outputs.kubeconfig }}
+
+      - name: Always delete IAM configuration
+        if: always()
+        uses: ./.github/actions/constellation_iam_destroy
+        with:
+          cloudProvider: ${{ inputs.cloudProvider }}
+          azureCredentials: ${{ secrets.AZURE_E2E_IAM_CREDENTIALS }}
+          gcpServiceAccount: "constellation-iam-e2e@constellation-331613.iam.gserviceaccount.com"
+
+      - name: Always upload Terraform logs
+        if: always()
+        uses: ./.github/actions/upload_terraform_logs
+        with:
+          artifactNameSuffix: ${{ steps.e2e_test.outputs.namePrefix }}

bootstrapper/cmd/bootstrapper/main.go

@@ -125,6 +125,10 @@ func main() {
 		if err != nil {
 			log.With(zap.Error(err)).Fatalf("Failed to set up cloud logger")
 		}
+		if err := metadata.PrepareControlPlaneNode(ctx, log); err != nil {
+			log.With(zap.Error(err)).Fatalf("Failed to prepare Azure control plane node")
+		}
+
 		metadataAPI = metadata
 		clusterInitJoiner = kubernetes.New(
 			"azure", k8sapi.NewKubernetesUtil(), &k8sapi.KubdeadmConfiguration{}, kubectl.NewUninitialized(),

cli/internal/cloudcmd/tfvars.go

@@ -103,6 +103,7 @@ func awsTerraformVars(conf *config.Config, imageRef string) *terraform.AWSCluste
 		Debug:                  conf.IsDebugCluster(),
 		EnableSNP:              conf.GetAttestationConfig().GetVariant().Equal(variant.AWSSEVSNP{}),
 		CustomEndpoint:         conf.CustomEndpoint,
+		InternalLoadBalancer:   conf.InternalLoadBalancer,
 	}
 }
 
@@ -143,6 +144,7 @@ func azureTerraformVars(conf *config.Config, imageRef string) *terraform.AzureCl
 		UserAssignedIdentity: conf.Provider.Azure.UserAssignedIdentity,
 		ResourceGroup:        conf.Provider.Azure.ResourceGroup,
 		CustomEndpoint:       conf.CustomEndpoint,
+		InternalLoadBalancer: conf.InternalLoadBalancer,
 	}
 
 	vars = normalizeAzureURIs(vars)
@@ -180,6 +182,7 @@ func gcpTerraformVars(conf *config.Config, imageRef string) *terraform.GCPCluste
 		ImageID:              imageRef,
 		Debug:                conf.IsDebugCluster(),
 		CustomEndpoint:       conf.CustomEndpoint,
+		InternalLoadBalancer: conf.InternalLoadBalancer,
 	}
 }
 
@@ -218,6 +221,7 @@ func openStackTerraformVars(conf *config.Config, imageRef string) *terraform.Ope
 		Debug:                   conf.IsDebugCluster(),
 		NodeGroups:              nodeGroups,
 		CustomEndpoint:          conf.CustomEndpoint,
+		InternalLoadBalancer:    conf.InternalLoadBalancer,
 	}
 }
 

cli/internal/helm/BUILD.bazel

@@ -252,7 +252,6 @@ go_library(
         "charts/edgeless/constellation-services/charts/verification-service/.helmignore",
         "charts/edgeless/constellation-services/charts/verification-service/Chart.yaml",
         "charts/edgeless/constellation-services/charts/verification-service/templates/daemonset.yaml",
-        "charts/edgeless/constellation-services/charts/verification-service/templates/loadbalancer-service.yaml",
         "charts/edgeless/constellation-services/charts/verification-service/templates/nodeport-service.yaml",
         "charts/edgeless/constellation-services/charts/verification-service/values.schema.json",
         "charts/edgeless/constellation-services/charts/verification-service/values.yaml",

cli/internal/helm/charts/edgeless/constellation-services/charts/verification-service/templates/loadbalancer-service.yaml

@@ -1,18 +0,0 @@
-apiVersion: v1
-kind: Service
-metadata:
-  name: verify
-  namespace: {{ .Release.Namespace }}
-spec:
-  allocateLoadBalancerNodePorts: false
-  externalIPs:
-  - {{ .Values.loadBalancerIP | quote }}
-  loadBalancerClass: constellation
-  ports:
-  - name: grpc
-    port: {{ .Values.grpcNodePort }}
-    protocol: TCP
-    targetPort: {{ .Values.grpcContainerPort }}
-  selector:
-    k8s-app: verification-service
-  type: LoadBalancer

cli/internal/helm/charts/edgeless/constellation-services/charts/verification-service/values.schema.json

@@ -4,21 +4,22 @@
         "image": {
             "description": "Container image to use for the spawned pods.",
             "type": "string",
-            "examples": ["ghcr.io/edgelesssys/constellation/join-service:latest"]
-        },
-        "loadBalancerIP": {
-            "description": "IP of the k8s LB service",
-            "type": "string"
+            "examples": [
+                "ghcr.io/edgelesssys/constellation/join-service:latest"
+            ]
         },
         "attestationVariant": {
             "description": "Attestation variant to use for aTLS connections.",
             "type": "string",
-            "examples": ["azure-sev-snp", "azure-trusted-launch", "gcp-sev-es"]
+            "examples": [
+                "azure-sev-snp",
+                "azure-trusted-launch",
+                "gcp-sev-es"
+            ]
         }
     },
     "required": [
         "image",
-        "loadBalancerIP",
         "attestationVariant"
     ],
     "title": "Values",

cli/internal/helm/overrides.go

@@ -42,7 +42,7 @@ func extraCiliumValues(provider cloudprovider.Provider, conformanceMode bool, ou
 		}
 	}
 
-	extraVals["k8sServiceHost"] = output.ClusterEndpoint
+	extraVals["k8sServiceHost"] = output.InClusterEndpoint
 	extraVals["k8sServicePort"] = constants.KubernetesPort
 	if provider == cloudprovider.GCP {
 		extraVals["ipv4NativeRoutingCIDR"] = output.GCP.IPCidrPod
@@ -62,7 +62,6 @@ func extraConstellationServicesValues(
 	}
 	extraVals["verification-service"] = map[string]any{
 		"attestationVariant": cfg.GetAttestationConfig().GetVariant().String(),
-		"loadBalancerIP":     output.ClusterEndpoint,
 	}
 	extraVals["konnectivity"] = map[string]any{
 		"loadBalancerIP": output.ClusterEndpoint,

cli/internal/state/state.go

@@ -89,9 +89,13 @@ type Infrastructure struct {
 	//   Unique identifier the cluster's cloud resources are tagged with.
 	UID string `yaml:"uid"`
 	// description: |
-	//   Endpoint the cluster can be reached at.
+	//   Endpoint the cluster can be reached at. This is the endpoint that is being used by the CLI.
 	ClusterEndpoint string `yaml:"clusterEndpoint"`
 	// description: |
+	//   The Cluster uses to reach itself. This might differ from the ClusterEndpoint in case e.g.,
+	//   an internal load balancer is used.
+	InClusterEndpoint string `yaml:"inClusterEndpoint"`
+	// description: |
 	//   Secret used to authenticate the bootstrapping node.
 	InitSecret HexBytes `yaml:"initSecret"`
 	// description: |

cli/internal/state/state_doc.go

@@ -74,7 +74,7 @@ func init() {
 			FieldName: "infrastructure",
 		},
 	}
-	InfrastructureDoc.Fields = make([]encoder.Doc, 7)
+	InfrastructureDoc.Fields = make([]encoder.Doc, 8)
 	InfrastructureDoc.Fields[0].Name = "uid"
 	InfrastructureDoc.Fields[0].Type = "string"
 	InfrastructureDoc.Fields[0].Note = ""
@@ -83,33 +83,38 @@ func init() {
 	InfrastructureDoc.Fields[1].Name = "clusterEndpoint"
 	InfrastructureDoc.Fields[1].Type = "string"
 	InfrastructureDoc.Fields[1].Note = ""
-	InfrastructureDoc.Fields[1].Description = "Endpoint the cluster can be reached at."
-	InfrastructureDoc.Fields[1].Comments[encoder.LineComment] = "Endpoint the cluster can be reached at."
-	InfrastructureDoc.Fields[2].Name = "initSecret"
-	InfrastructureDoc.Fields[2].Type = "HexBytes"
+	InfrastructureDoc.Fields[1].Description = "Endpoint the cluster can be reached at. This is the endpoint that is being used by the CLI."
+	InfrastructureDoc.Fields[1].Comments[encoder.LineComment] = "Endpoint the cluster can be reached at. This is the endpoint that is being used by the CLI."
+	InfrastructureDoc.Fields[2].Name = "inClusterEndpoint"
+	InfrastructureDoc.Fields[2].Type = "string"
 	InfrastructureDoc.Fields[2].Note = ""
-	InfrastructureDoc.Fields[2].Description = "Secret used to authenticate the bootstrapping node."
-	InfrastructureDoc.Fields[2].Comments[encoder.LineComment] = "Secret used to authenticate the bootstrapping node."
-	InfrastructureDoc.Fields[3].Name = "apiServerCertSANs"
-	InfrastructureDoc.Fields[3].Type = "[]string"
+	InfrastructureDoc.Fields[2].Description = "The Cluster uses to reach itself. This might differ from the ClusterEndpoint in case e.g.,\nan internal load balancer is used."
+	InfrastructureDoc.Fields[2].Comments[encoder.LineComment] = "The Cluster uses to reach itself. This might differ from the ClusterEndpoint in case e.g.,"
+	InfrastructureDoc.Fields[3].Name = "initSecret"
+	InfrastructureDoc.Fields[3].Type = "HexBytes"
 	InfrastructureDoc.Fields[3].Note = ""
-	InfrastructureDoc.Fields[3].Description = "description: |\n  List of Subject Alternative Names (SANs) to add to the Kubernetes API server certificate.\n If no SANs should be added, this field can be left empty.\n"
-	InfrastructureDoc.Fields[3].Comments[encoder.LineComment] = "description: |"
-	InfrastructureDoc.Fields[4].Name = "name"
-	InfrastructureDoc.Fields[4].Type = "string"
+	InfrastructureDoc.Fields[3].Description = "Secret used to authenticate the bootstrapping node."
+	InfrastructureDoc.Fields[3].Comments[encoder.LineComment] = "Secret used to authenticate the bootstrapping node."
+	InfrastructureDoc.Fields[4].Name = "apiServerCertSANs"
+	InfrastructureDoc.Fields[4].Type = "[]string"
 	InfrastructureDoc.Fields[4].Note = ""
-	InfrastructureDoc.Fields[4].Description = "Name used in the cluster's named resources."
-	InfrastructureDoc.Fields[4].Comments[encoder.LineComment] = "Name used in the cluster's named resources."
-	InfrastructureDoc.Fields[5].Name = "azure"
-	InfrastructureDoc.Fields[5].Type = "Azure"
+	InfrastructureDoc.Fields[4].Description = "description: |\n  List of Subject Alternative Names (SANs) to add to the Kubernetes API server certificate.\n If no SANs should be added, this field can be left empty.\n"
+	InfrastructureDoc.Fields[4].Comments[encoder.LineComment] = "description: |"
+	InfrastructureDoc.Fields[5].Name = "name"
+	InfrastructureDoc.Fields[5].Type = "string"
 	InfrastructureDoc.Fields[5].Note = ""
-	InfrastructureDoc.Fields[5].Description = "Values specific to a Constellation cluster running on Azure."
-	InfrastructureDoc.Fields[5].Comments[encoder.LineComment] = "Values specific to a Constellation cluster running on Azure."
-	InfrastructureDoc.Fields[6].Name = "gcp"
-	InfrastructureDoc.Fields[6].Type = "GCP"
+	InfrastructureDoc.Fields[5].Description = "Name used in the cluster's named resources."
+	InfrastructureDoc.Fields[5].Comments[encoder.LineComment] = "Name used in the cluster's named resources."
+	InfrastructureDoc.Fields[6].Name = "azure"
+	InfrastructureDoc.Fields[6].Type = "Azure"
 	InfrastructureDoc.Fields[6].Note = ""
-	InfrastructureDoc.Fields[6].Description = "Values specific to a Constellation cluster running on GCP."
-	InfrastructureDoc.Fields[6].Comments[encoder.LineComment] = "Values specific to a Constellation cluster running on GCP."
+	InfrastructureDoc.Fields[6].Description = "Values specific to a Constellation cluster running on Azure."
+	InfrastructureDoc.Fields[6].Comments[encoder.LineComment] = "Values specific to a Constellation cluster running on Azure."
+	InfrastructureDoc.Fields[7].Name = "gcp"
+	InfrastructureDoc.Fields[7].Type = "GCP"
+	InfrastructureDoc.Fields[7].Note = ""
+	InfrastructureDoc.Fields[7].Description = "Values specific to a Constellation cluster running on GCP."
+	InfrastructureDoc.Fields[7].Comments[encoder.LineComment] = "Values specific to a Constellation cluster running on GCP."
 
 	GCPDoc.Type = "GCP"
 	GCPDoc.Comments[encoder.LineComment] = "GCP describes the infra state related to GCP."

cli/internal/terraform/BUILD.bazel

@@ -73,6 +73,17 @@ go_library(
         "terraform/iam/aws/.terraform.lock.hcl",
         "terraform/iam/azure/.terraform.lock.hcl",
         "terraform/iam/gcp/.terraform.lock.hcl",
+        "terraform/gcp/modules/internal_load_balancer/main.tf",
+        "terraform/gcp/modules/internal_load_balancer/variables.tf",
+        "terraform/gcp/modules/jump_host/main.tf",
+        "terraform/gcp/modules/jump_host/outputs.tf",
+        "terraform/gcp/modules/jump_host/variables.tf",
+        "terraform/aws/modules/jump_host/main.tf",
+        "terraform/aws/modules/jump_host/output.tf",
+        "terraform/aws/modules/jump_host/variables.tf",
+        "terraform/azure/modules/jump_host/main.tf",
+        "terraform/azure/modules/jump_host/variables.tf",
+        "terraform/azure/modules/jump_host/outputs.tf",
     ],
     importpath = "github.com/edgelesssys/constellation/v2/cli/internal/terraform",
     visibility = ["//cli:__subpackages__"],

cli/internal/terraform/terraform.go

@@ -181,11 +181,20 @@ func (c *Client) ShowInfrastructure(ctx context.Context, provider cloudprovider.
 		return state.Infrastructure{}, errors.New("terraform show: no values returned")
 	}
 
-	ipOutput, ok := tfState.Values.Outputs["ip"]
+	outOfClusterEndpointOutput, ok := tfState.Values.Outputs["out_of_cluster_endpoint"]
 	if !ok {
-		return state.Infrastructure{}, errors.New("no IP output found")
+		return state.Infrastructure{}, errors.New("no out_of_cluster_endpoint output found")
 	}
-	ip, ok := ipOutput.Value.(string)
+	outOfClusterEndpoint, ok := outOfClusterEndpointOutput.Value.(string)
+	if !ok {
+		return state.Infrastructure{}, errors.New("invalid type in IP output: not a string")
+	}
+
+	inClusterEndpointOutput, ok := tfState.Values.Outputs["in_cluster_endpoint"]
+	if !ok {
+		return state.Infrastructure{}, errors.New("no in_cluster_endpoint output found")
+	}
+	inClusterEndpoint, ok := inClusterEndpointOutput.Value.(string)
 	if !ok {
 		return state.Infrastructure{}, errors.New("invalid type in IP output: not a string")
 	}
@@ -231,7 +240,8 @@ func (c *Client) ShowInfrastructure(ctx context.Context, provider cloudprovider.
 	}
 
 	res := state.Infrastructure{
-		ClusterEndpoint:   ip,
+		ClusterEndpoint:   outOfClusterEndpoint,
+		InClusterEndpoint: inClusterEndpoint,
 		APIServerCertSANs: apiServerCertSANs,
 		InitSecret:        []byte(secret),
 		UID:               uid,

cli/internal/terraform/terraform/aws/main.tf

@@ -51,6 +51,9 @@ locals {
   tags = {
     constellation-uid = local.uid,
   }
+
+  in_cluster_endpoint     = aws_lb.front_end.dns_name
+  out_of_cluster_endpoint = var.internal_load_balancer && var.debug ? module.jump_host[0].ip : local.in_cluster_endpoint
 }
 
 resource "random_id" "uid" {
@@ -84,14 +87,14 @@ resource "aws_eip" "lb" {
   # in a future version to support all availability zones in the chosen region
   # This should only be done after we migrated to DNS-based addressing for the
   # control-plane.
-  for_each = toset([var.zone])
+  for_each = var.internal_load_balancer ? [] : toset([var.zone])
   domain   = "vpc"
   tags     = merge(local.tags, { "constellation-ip-endpoint" = each.key == var.zone ? "legacy-primary-zone" : "additional-zone" })
 }
 
 resource "aws_lb" "front_end" {
   name               = "${local.name}-loadbalancer"
-  internal           = false
+  internal           = var.internal_load_balancer
   load_balancer_type = "network"
   tags               = local.tags
   security_groups    = [aws_security_group.security_group.id]
@@ -106,7 +109,7 @@ resource "aws_lb" "front_end" {
     for_each = toset([var.zone])
     content {
       subnet_id     = module.public_private_subnet.public_subnet_id[subnet_mapping.key]
-      allocation_id = aws_eip.lb[subnet_mapping.key].id
+      allocation_id = var.internal_load_balancer ? "" : aws_eip.lb[subnet_mapping.key].id
     }
   }
   enable_cross_zone_load_balancing = true
@@ -206,6 +209,17 @@ module "instance_group" {
   )
 }
 
+module "jump_host" {
+  count                = var.internal_load_balancer && var.debug ? 1 : 0
+  source               = "./modules/jump_host"
+  base_name            = local.name
+  subnet_id            = module.public_private_subnet.public_subnet_id[var.zone]
+  lb_internal_ip       = aws_lb.front_end.dns_name
+  ports                = [for port in local.load_balancer_ports : port.port]
+  iam_instance_profile = var.iam_instance_profile_worker_nodes
+  security_group_id    = aws_security_group.security_group.id
+}
+
 # TODO(31u3r): Remove once 2.12 is released
 moved {
   from = module.load_balancer_target_konnectivity
@@ -241,3 +255,4 @@ moved {
   from = module.load_balancer_target_bootstrapper
   to   = module.load_balancer_targets["bootstrapper"]
 }
+

cli/internal/terraform/terraform/aws/modules/jump_host/main.tf

@@ -0,0 +1,59 @@
+terraform {
+  required_providers {
+    aws = {
+      source  = "hashicorp/aws"
+      version = "5.17.0"
+    }
+  }
+}
+
+
+data "aws_ami" "ubuntu" {
+  most_recent = true
+  owners      = ["099720109477"] # Canonical
+
+  filter {
+    name   = "name"
+    values = ["ubuntu/images/hvm-ssd/ubuntu-jammy-22.04-amd64-server-*"]
+  }
+}
+
+resource "aws_instance" "jump_host" {
+  ami                         = data.aws_ami.ubuntu.id
+  instance_type               = "c5a.large"
+  associate_public_ip_address = true
+
+  iam_instance_profile = var.iam_instance_profile
+  subnet_id            = var.subnet_id
+  security_groups      = [var.security_group_id]
+
+  tags = {
+    "Name" = "${var.base_name}-jump-host"
+  }
+
+  user_data = <<EOF
+#!/bin/bash
+set -x
+
+# Uncomment to create user with password
+# useradd -m user
+# usermod -aG sudo user
+# usermod --shell /bin/bash user
+# sh -c "echo \"user:pass\" | chpasswd"
+
+sysctl -w net.ipv4.ip_forward=1
+sysctl -p
+
+internal_ip=$(ip route get 8.8.8.8 | grep -oP 'src \K[^ ]+')
+
+lb_ip=${var.lb_internal_ip}
+if [[ ! $${lb_ip} =~ ^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
+  lb_ip=$(dig +short ${var.lb_internal_ip})
+fi
+%{for port in var.ports~}
+iptables -t nat -A PREROUTING -p tcp --dport ${port} -j DNAT --to-destination $${lb_ip}:${port}
+iptables -t nat -A POSTROUTING -p tcp -d $${lb_ip} --dport ${port} -j SNAT --to-source $${internal_ip}
+%{endfor~}
+EOF
+
+}

cli/internal/terraform/terraform/aws/modules/jump_host/output.tf

@@ -0,0 +1,3 @@
+output "ip" {
+  value = aws_instance.jump_host.public_ip
+}

cli/internal/terraform/terraform/aws/modules/jump_host/variables.tf

@@ -0,0 +1,29 @@
+variable "base_name" {
+  description = "Base name of the jump host"
+  type        = string
+}
+
+variable "subnet_id" {
+  description = "Subnet ID to deploy the jump host into"
+  type        = string
+}
+
+variable "lb_internal_ip" {
+  description = "Internal IP of the load balancer"
+  type        = string
+}
+
+variable "iam_instance_profile" {
+  description = "IAM instance profile to attach to the jump host"
+  type        = string
+}
+
+variable "ports" {
+  description = "Ports to forward to the load balancer"
+  type        = list(number)
+}
+
+variable "security_group_id" {
+  description = "Security group to attach to the jump host"
+  type        = string
+}

cli/internal/terraform/terraform/aws/modules/load_balancer_target/main.tf

@@ -13,6 +13,7 @@ resource "aws_lb_target_group" "front_end" {
   protocol           = "TCP"
   vpc_id             = var.vpc_id
   tags               = var.tags
+  preserve_client_ip = "false"
 
   health_check {
     port                = var.port

cli/internal/terraform/terraform/aws/outputs.tf

@@ -1,9 +1,22 @@
-output "ip" {
-  value = aws_eip.lb[var.zone].public_ip
+output "out_of_cluster_endpoint" {
+  value = local.out_of_cluster_endpoint
 }
 
+output "in_cluster_endpoint" {
+  value = local.in_cluster_endpoint
+}
 output "api_server_cert_sans" {
-  value = sort(concat([aws_eip.lb[var.zone].public_ip, local.wildcard_lb_dns_name], var.custom_endpoint == "" ? [] : [var.custom_endpoint]))
+  value = sort(
+    distinct(
+      concat(
+        [
+          local.in_cluster_endpoint,
+          local.out_of_cluster_endpoint,
+        ],
+        var.custom_endpoint == "" ? [] : [var.custom_endpoint],
+      )
+    )
+  )
 }
 
 output "uid" {

cli/internal/terraform/terraform/aws/variables.tf

@@ -69,3 +69,9 @@ variable "custom_endpoint" {
   default     = ""
   description = "Custom endpoint to use for the Kubernetes apiserver. If not set, the default endpoint will be used."
 }
+
+variable "internal_load_balancer" {
+  type        = bool
+  default     = false
+  description = "Use an internal load balancer."
+}

cli/internal/terraform/terraform/azure/.terraform.lock.hcl

@@ -60,3 +60,26 @@ provider "registry.terraform.io/hashicorp/random" {
     "zh:eab0d0495e7e711cca367f7d4df6e322e6c562fc52151ec931176115b83ed014",
   ]
 }
+
+provider "registry.terraform.io/hashicorp/tls" {
+  version = "4.0.4"
+  hashes = [
+    "h1:GZcFizg5ZT2VrpwvxGBHQ/hO9r6g0vYdQqx3bFD3anY=",
+    "h1:Wd3RqmQW60k2QWPN4sK5CtjGuO1d+CRNXgC+D4rKtXc=",
+    "h1:bNsvpX5EGuVxgGRXBQVLXlmq40PdoLp8Rfuh1ZmV7yY=",
+    "h1:pe9vq86dZZKCm+8k1RhzARwENslF3SXb9ErHbQfgjXU=",
+    "h1:rKKMyIEBZwR+8j6Tx3PwqBrStuH+J+pxcbCR5XN8WAw=",
+    "zh:23671ed83e1fcf79745534841e10291bbf34046b27d6e68a5d0aab77206f4a55",
+    "zh:45292421211ffd9e8e3eb3655677700e3c5047f71d8f7650d2ce30242335f848",
+    "zh:59fedb519f4433c0fdb1d58b27c210b27415fddd0cd73c5312530b4309c088be",
+    "zh:5a8eec2409a9ff7cd0758a9d818c74bcba92a240e6c5e54b99df68fff312bbd5",
+    "zh:5e6a4b39f3171f53292ab88058a59e64825f2b842760a4869e64dc1dc093d1fe",
+    "zh:810547d0bf9311d21c81cc306126d3547e7bd3f194fc295836acf164b9f8424e",
+    "zh:824a5f3617624243bed0259d7dd37d76017097dc3193dac669be342b90b2ab48",
+    "zh:9361ccc7048be5dcbc2fafe2d8216939765b3160bd52734f7a9fd917a39ecbd8",
+    "zh:aa02ea625aaf672e649296bce7580f62d724268189fe9ad7c1b36bb0fa12fa60",
+    "zh:c71b4cd40d6ec7815dfeefd57d88bc592c0c42f5e5858dcc88245d371b4b8b1e",
+    "zh:dabcd52f36b43d250a3d71ad7abfa07b5622c69068d989e60b79b2bb4f220316",
+    "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c",
+  ]
+}

cli/internal/terraform/terraform/azure/main.tf

@@ -40,12 +40,15 @@ locals {
   ])
   // wildcard_lb_dns_name is the DNS name of the load balancer with a wildcard for the name.
   // example: given "name-1234567890.location.cloudapp.azure.com" it will return "*.location.cloudapp.azure.com"
-  wildcard_lb_dns_name = replace(data.azurerm_public_ip.loadbalancer_ip.fqdn, "/^[^.]*\\./", "*.")
+  wildcard_lb_dns_name = var.internal_load_balancer ? "" : replace(data.azurerm_public_ip.loadbalancer_ip[0].fqdn, "/^[^.]*\\./", "*.")
   // deduce from format (subscriptions)/$ID/resourceGroups/$RG/providers/Microsoft.ManagedIdentity/userAssignedIdentities/$NAME"
   // move from the right as to ignore the optional prefixes
   uai_resource_group = element(split("/", var.user_assigned_identity), length(split("/", var.user_assigned_identity)) - 5)
   // deduce as above
   uai_name = element(split("/", var.user_assigned_identity), length(split("/", var.user_assigned_identity)) - 1)
+
+  in_cluster_endpoint     = var.internal_load_balancer ? azurerm_lb.loadbalancer.frontend_ip_configuration[0].private_ip_address : azurerm_public_ip.loadbalancer_ip[0].ip_address
+  out_of_cluster_endpoint = var.debug && var.internal_load_balancer ? module.jump_host[0].ip : local.in_cluster_endpoint
 }
 
 resource "random_id" "uid" {
@@ -84,6 +87,7 @@ resource "azurerm_application_insights" "insights" {
 }
 
 resource "azurerm_public_ip" "loadbalancer_ip" {
+  count               = var.internal_load_balancer ? 0 : 1
   name                = "${local.name}-lb"
   domain_name_label   = local.name
   resource_group_name = var.resource_group
@@ -104,6 +108,7 @@ resource "azurerm_public_ip" "loadbalancer_ip" {
 // resources for clusters created before 2.9. In those cases we need to wait until loadbalancer_ip has
 // been updated before reading from it.
 data "azurerm_public_ip" "loadbalancer_ip" {
+  count               = var.internal_load_balancer ? 0 : 1
   name                = "${local.name}-lb"
   resource_group_name = var.resource_group
   depends_on          = [azurerm_public_ip.loadbalancer_ip]
@@ -143,9 +148,21 @@ resource "azurerm_lb" "loadbalancer" {
   sku                 = "Standard"
   tags                = local.tags
 
-  frontend_ip_configuration {
+  dynamic "frontend_ip_configuration" {
+    for_each = var.internal_load_balancer ? [] : [1]
+    content {
       name                 = "PublicIPAddress"
-    public_ip_address_id = azurerm_public_ip.loadbalancer_ip.id
+      public_ip_address_id = azurerm_public_ip.loadbalancer_ip[0].id
+    }
+  }
+
+  dynamic "frontend_ip_configuration" {
+    for_each = var.internal_load_balancer ? [1] : []
+    content {
+      name                          = "PrivateIPAddress"
+      private_ip_address_allocation = "Dynamic"
+      subnet_id                     = azurerm_subnet.loadbalancer_subnet[0].id
+    }
   }
 }
 
@@ -180,6 +197,14 @@ resource "azurerm_virtual_network" "network" {
   tags                = local.tags
 }
 
+resource "azurerm_subnet" "loadbalancer_subnet" {
+  count                = var.internal_load_balancer ? 1 : 0
+  name                 = "${local.name}-lb"
+  resource_group_name  = var.resource_group
+  virtual_network_name = azurerm_virtual_network.network.name
+  address_prefixes     = ["10.10.0.0/16"]
+}
+
 resource "azurerm_subnet" "node_subnet" {
   name                 = "${local.name}-node"
   resource_group_name  = var.resource_group
@@ -246,6 +271,17 @@ module "scale_set_group" {
   ]
 }
 
+module "jump_host" {
+  count          = var.internal_load_balancer && var.debug ? 1 : 0
+  source         = "./modules/jump_host"
+  base_name      = local.name
+  resource_group = var.resource_group
+  location       = var.location
+  subnet_id      = azurerm_subnet.loadbalancer_subnet[0].id
+  ports          = [for port in local.ports : port.port]
+  lb_internal_ip = azurerm_lb.loadbalancer.frontend_ip_configuration[0].private_ip_address
+}
+
 data "azurerm_subscription" "current" {
 }
 

cli/internal/terraform/terraform/azure/modules/jump_host/main.tf

@@ -0,0 +1,85 @@
+resource "azurerm_linux_virtual_machine" "jump_host" {
+  name                = "${var.base_name}-jump-host"
+  resource_group_name = var.resource_group
+  location            = var.location
+  size                = "Standard_D2as_v5"
+
+  network_interface_ids = [
+    azurerm_network_interface.jump_host.id,
+  ]
+
+  admin_username = "adminuser"
+
+  admin_ssh_key {
+    username   = "adminuser"
+    public_key = tls_private_key.ssh_key.public_key_openssh
+  }
+
+  os_disk {
+    caching              = "ReadWrite"
+    storage_account_type = "Standard_LRS"
+  }
+
+  source_image_reference {
+    publisher = "Canonical"
+    offer     = "0001-com-ubuntu-server-jammy"
+    sku       = "22_04-lts-gen2"
+    version   = "latest"
+  }
+
+  boot_diagnostics {
+
+  }
+
+  user_data = base64encode(<<EOF
+#!/bin/bash
+set -x
+
+# Uncomment to create user with password
+# useradd -m user
+# usermod -aG sudo user
+# usermod --shell /bin/bash user
+# sh -c "echo \"user:pass\" | chpasswd"
+
+sysctl -w net.ipv4.ip_forward=1
+sysctl -p
+
+internal_ip=$(ip route get 8.8.8.8 | grep -oP 'src \K[^ ]+')
+
+lb_ip=${var.lb_internal_ip}
+if [[ ! $${lb_ip} =~ ^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
+  lb_ip=$(dig +short ${var.lb_internal_ip})
+fi
+
+%{for port in var.ports~}
+iptables -t nat -A PREROUTING -p tcp --dport ${port} -j DNAT --to-destination $${lb_ip}:${port}
+iptables -t nat -A POSTROUTING -p tcp -d $${lb_ip} --dport ${port} -j SNAT --to-source $${internal_ip}
+%{endfor~}
+EOF
+  )
+}
+
+resource "azurerm_network_interface" "jump_host" {
+  name                = "${var.base_name}-jump-host"
+  resource_group_name = var.resource_group
+  location            = var.location
+
+  ip_configuration {
+    name                          = "public"
+    subnet_id                     = var.subnet_id
+    private_ip_address_allocation = "Dynamic"
+    public_ip_address_id          = azurerm_public_ip.jump_host.id
+  }
+}
+
+resource "azurerm_public_ip" "jump_host" {
+  name                = "${var.base_name}-jump-host"
+  resource_group_name = var.resource_group
+  location            = var.location
+  allocation_method   = "Dynamic"
+}
+
+resource "tls_private_key" "ssh_key" {
+  algorithm = "RSA"
+  rsa_bits  = 4096
+}

cli/internal/terraform/terraform/azure/modules/jump_host/outputs.tf

@@ -0,0 +1,3 @@
+output "ip" {
+  value = azurerm_linux_virtual_machine.jump_host.public_ip_address
+}

cli/internal/terraform/terraform/azure/modules/jump_host/variables.tf

@@ -0,0 +1,29 @@
+variable "base_name" {
+  description = "Base name of the jump host"
+  type        = string
+}
+
+variable "ports" {
+  description = "Ports to forward to the load balancer"
+  type        = list(number)
+}
+
+variable "resource_group" {
+  description = "Resource group name to deploy the jump host into"
+  type        = string
+}
+
+variable "location" {
+  description = "Location to deploy the jump host into"
+  type        = string
+}
+
+variable "subnet_id" {
+  description = "Subnet ID to deploy the jump host into"
+  type        = string
+}
+
+variable "lb_internal_ip" {
+  description = "Internal IP of the load balancer"
+  type        = string
+}

cli/internal/terraform/terraform/azure/outputs.tf

@@ -1,9 +1,24 @@
-output "ip" {
-  value = azurerm_public_ip.loadbalancer_ip.ip_address
+output "out_of_cluster_endpoint" {
+  value = local.out_of_cluster_endpoint
+}
+
+output "in_cluster_endpoint" {
+  value = local.in_cluster_endpoint
 }
 
 output "api_server_cert_sans" {
-  value = sort(concat([azurerm_public_ip.loadbalancer_ip.ip_address, local.wildcard_lb_dns_name], var.custom_endpoint == "" ? [] : [var.custom_endpoint]))
+  value = sort(
+    distinct(
+      concat(
+        [
+          local.in_cluster_endpoint,
+          local.out_of_cluster_endpoint,
+        ],
+        var.custom_endpoint == "" ? [] : [var.custom_endpoint],
+        var.internal_load_balancer ? [] : [local.wildcard_lb_dns_name],
+      )
+    )
+  )
 }
 
 output "uid" {

cli/internal/terraform/terraform/azure/variables.tf

@@ -67,3 +67,9 @@ variable "custom_endpoint" {
   default     = ""
   description = "Custom endpoint to use for the Kubernetes apiserver. If not set, the default endpoint will be used."
 }
+
+variable "internal_load_balancer" {
+  type        = bool
+  default     = false
+  description = "Whether to use an internal load balancer for the Constellation."
+}

cli/internal/terraform/terraform/gcp/main.tf

@@ -57,6 +57,8 @@ locals {
   control_plane_instance_groups = [
     for control_plane in local.node_groups_by_role["control-plane"] : module.instance_group[control_plane].instance_group
   ]
+  in_cluster_endpoint     = var.internal_load_balancer ? google_compute_address.loadbalancer_ip_internal[0].address : google_compute_global_address.loadbalancer_ip[0].address
+  out_of_cluster_endpoint = var.debug && var.internal_load_balancer ? module.jump_host[0].ip : local.in_cluster_endpoint
 }
 
 resource "random_id" "uid" {
@@ -89,6 +91,26 @@ resource "google_compute_subnetwork" "vpc_subnetwork" {
   ]
 }
 
+
+resource "google_compute_subnetwork" "proxy_subnet" {
+  count         = var.internal_load_balancer ? 1 : 0
+  name          = "${local.name}-proxy"
+  ip_cidr_range = local.cidr_vpc_subnet_proxy
+  region        = var.region
+  purpose       = "REGIONAL_MANAGED_PROXY"
+  role          = "ACTIVE"
+  network       = google_compute_network.vpc_network.id
+}
+
+resource "google_compute_subnetwork" "ilb_subnet" {
+  count         = var.internal_load_balancer ? 1 : 0
+  name          = "${local.name}-ilb"
+  ip_cidr_range = local.cidr_vpc_subnet_ilb
+  region        = var.region
+  network       = google_compute_network.vpc_network.id
+  depends_on    = [google_compute_subnetwork.proxy_subnet]
+}
+
 resource "google_compute_router" "vpc_router" {
   name        = local.name
   description = "Constellation VPC router"
@@ -114,6 +136,7 @@ resource "google_compute_firewall" "firewall_external" {
     ports = flatten([
       [for port in local.control_plane_named_ports : port.port],
       [local.ports_node_range],
+      var.internal_load_balancer ? [22] : [],
     ])
   }
 
@@ -168,23 +191,59 @@ module "instance_group" {
   custom_endpoint     = var.custom_endpoint
 }
 
+resource "google_compute_address" "loadbalancer_ip_internal" {
+  count        = var.internal_load_balancer ? 1 : 0
+  name         = local.name
+  region       = var.region
+  subnetwork   = google_compute_subnetwork.ilb_subnet[0].id
+  purpose      = "SHARED_LOADBALANCER_VIP"
+  address_type = "INTERNAL"
+}
+
 resource "google_compute_global_address" "loadbalancer_ip" {
+  count = var.internal_load_balancer ? 0 : 1
   name  = local.name
 }
 
 module "loadbalancer_public" {
   // for every port in control_plane_named_ports if internal lb is disabled
-  for_each                = { for port in local.control_plane_named_ports : port.name => port }
+  for_each                = var.internal_load_balancer ? {} : { for port in local.control_plane_named_ports : port.name => port }
   source                  = "./modules/loadbalancer"
   name                    = local.name
   backend_port_name       = each.value.name
   port                    = each.value.port
   health_check            = each.value.health_check
   backend_instance_groups = local.control_plane_instance_groups
-  ip_address              = google_compute_global_address.loadbalancer_ip.self_link
+  ip_address              = google_compute_global_address.loadbalancer_ip[0].self_link
   frontend_labels         = merge(local.labels, { constellation-use = each.value.name })
 }
 
+module "loadbalancer_internal" {
+  for_each               = var.internal_load_balancer ? { for port in local.control_plane_named_ports : port.name => port } : {}
+  source                 = "./modules/internal_load_balancer"
+  name                   = local.name
+  backend_port_name      = each.value.name
+  port                   = each.value.port
+  health_check           = each.value.health_check
+  backend_instance_group = local.control_plane_instance_groups[0]
+  ip_address             = google_compute_address.loadbalancer_ip_internal[0].self_link
+  frontend_labels        = merge(local.labels, { constellation-use = each.value.name })
+
+  region         = var.region
+  network        = google_compute_network.vpc_network.id
+  backend_subnet = google_compute_subnetwork.ilb_subnet[0].id
+}
+
+module "jump_host" {
+  count          = var.internal_load_balancer && var.debug ? 1 : 0
+  source         = "./modules/jump_host"
+  base_name      = local.name
+  zone           = var.zone
+  subnetwork     = google_compute_subnetwork.vpc_subnetwork.id
+  labels         = local.labels
+  lb_internal_ip = google_compute_address.loadbalancer_ip_internal[0].address
+  ports          = [for port in local.control_plane_named_ports : port.port]
+}
 moved {
   from = module.loadbalancer_boot
   to   = module.loadbalancer_public["bootstrapper"]
@@ -210,11 +269,6 @@ moved {
   to   = module.loadbalancer_public["recovery"]
 }
 
-moved {
-  from = module.loadbalancer_join
-  to   = module.loadbalancer_public["join"]
-}
-
 moved {
   from = module.loadbalancer_debugd[0]
   to   = module.loadbalancer_public["debugd"]

cli/internal/terraform/terraform/gcp/modules/internal_load_balancer/main.tf

@@ -0,0 +1,72 @@
+terraform {
+  required_providers {
+    google = {
+      source  = "hashicorp/google"
+      version = "4.83.0"
+    }
+  }
+}
+
+locals {
+  name = "${var.name}-${var.backend_port_name}"
+}
+
+resource "google_compute_region_health_check" "health" {
+  name               = local.name
+  region             = var.region
+  check_interval_sec = 1
+  timeout_sec        = 1
+
+  dynamic "tcp_health_check" {
+    for_each = var.health_check == "TCP" ? [1] : []
+    content {
+      port = var.port
+    }
+  }
+
+  dynamic "https_health_check" {
+    for_each = var.health_check == "HTTPS" ? [1] : []
+    content {
+      host         = ""
+      port         = var.port
+      request_path = "/readyz"
+    }
+  }
+}
+
+resource "google_compute_region_backend_service" "backend" {
+  name                  = local.name
+  protocol              = "TCP"
+  load_balancing_scheme = "INTERNAL_MANAGED"
+  health_checks         = [google_compute_region_health_check.health.id]
+  port_name             = var.backend_port_name
+  timeout_sec           = 240
+  region                = var.region
+
+  backend {
+    group           = var.backend_instance_group
+    balancing_mode  = "UTILIZATION"
+    capacity_scaler = 1.0
+  }
+}
+
+resource "google_compute_region_target_tcp_proxy" "proxy" {
+  name            = local.name
+  region          = var.region
+  backend_service = google_compute_region_backend_service.backend.id
+}
+
+# forwarding rule
+resource "google_compute_forwarding_rule" "forwarding" {
+  name                  = local.name
+  network               = var.network
+  subnetwork            = var.backend_subnet
+  region                = var.region
+  ip_address            = var.ip_address
+  ip_protocol           = "TCP"
+  load_balancing_scheme = "INTERNAL_MANAGED"
+  port_range            = var.port
+  allow_global_access   = true
+  target                = google_compute_region_target_tcp_proxy.proxy.id
+  labels                = var.frontend_labels
+}

cli/internal/terraform/terraform/gcp/modules/internal_load_balancer/variables.tf

@@ -0,0 +1,54 @@
+variable "name" {
+  type        = string
+  description = "Base name of the load balancer."
+}
+
+variable "region" {
+  type        = string
+  description = "The region where the load balancer will be created."
+}
+
+variable "network" {
+  type        = string
+  description = "The network to which all network resources will be attached."
+}
+
+variable "backend_subnet" {
+  type        = string
+  description = "The subnet to which all backend network resources will be attached."
+}
+
+variable "health_check" {
+  type        = string
+  description = "The type of the health check. 'HTTPS' or 'TCP'."
+  validation {
+    condition     = contains(["HTTPS", "TCP"], var.health_check)
+    error_message = "Health check must be either 'HTTPS' or 'TCP'."
+  }
+}
+
+variable "port" {
+  type        = string
+  description = "The port on which to listen for incoming traffic."
+}
+
+variable "backend_port_name" {
+  type        = string
+  description = "Name of backend port. The same name should appear in the instance groups referenced by this service."
+}
+
+variable "backend_instance_group" {
+  type        = string
+  description = "The URL of the instance group resource from which the load balancer will direct traffic."
+}
+
+variable "ip_address" {
+  type        = string
+  description = "The IP address that this forwarding rule serves."
+}
+
+variable "frontend_labels" {
+  type        = map(string)
+  default     = {}
+  description = "Labels to apply to the forwarding rule."
+}

cli/internal/terraform/terraform/gcp/modules/jump_host/main.tf

@@ -0,0 +1,73 @@
+terraform {
+  required_providers {
+    google = {
+      source  = "hashicorp/google"
+      version = "4.83.0"
+    }
+
+    google-beta = {
+      source  = "hashicorp/google-beta"
+      version = "4.83.0"
+    }
+  }
+}
+
+
+data "google_compute_image" "image_ubuntu" {
+  family  = "ubuntu-2204-lts"
+  project = "ubuntu-os-cloud"
+}
+
+resource "google_compute_instance" "vm_instance" {
+  name         = "${var.base_name}-jumphost"
+  machine_type = "n2d-standard-4"
+  zone         = var.zone
+
+  boot_disk {
+    initialize_params {
+      image = data.google_compute_image.image_ubuntu.self_link
+    }
+  }
+
+  network_interface {
+    subnetwork = var.subnetwork
+    access_config {
+    }
+  }
+
+  service_account {
+    scopes = ["compute-ro"]
+  }
+
+  labels = var.labels
+
+  metadata = {
+    serial-port-enable = "TRUE"
+  }
+
+  metadata_startup_script = <<EOF
+#!/bin/bash
+set -x 
+
+# Uncomment to create user with password
+# useradd -m user
+# usermod -aG sudo user
+# usermod --shell /bin/bash user
+# sh -c "echo \"user:pass\" | chpasswd"
+
+sysctl -w net.ipv4.ip_forward=1
+sysctl -p
+
+internal_ip=$(ip route get 8.8.8.8 | grep -oP 'src \K[^ ]+')
+
+lb_ip=${var.lb_internal_ip}
+if [[ ! $${lb_ip} =~ ^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
+  lb_ip=$(dig +short ${var.lb_internal_ip})
+fi
+%{for port in var.ports~}
+iptables -t nat -A PREROUTING -p tcp --dport ${port} -j DNAT --to-destination ${var.lb_internal_ip}:${port}
+iptables -t nat -A POSTROUTING -p tcp -d ${var.lb_internal_ip} --dport ${port} -j SNAT --to-source $${internal_ip}
+%{endfor~}
+EOF
+
+}

cli/internal/terraform/terraform/gcp/modules/jump_host/outputs.tf

@@ -0,0 +1,3 @@
+output "ip" {
+  value = google_compute_instance.vm_instance.network_interface[0].access_config[0].nat_ip
+}

cli/internal/terraform/terraform/gcp/modules/jump_host/variables.tf

@@ -0,0 +1,30 @@
+variable "base_name" {
+  type        = string
+  description = "Base name of the instance group."
+}
+
+variable "labels" {
+  type        = map(string)
+  default     = {}
+  description = "Labels to apply to the instance group."
+}
+
+variable "subnetwork" {
+  type        = string
+  description = "Name of the subnetwork to use."
+}
+
+variable "zone" {
+  type        = string
+  description = "Zone to deploy the instance group in."
+}
+
+variable "lb_internal_ip" {
+  type        = string
+  description = "Internal IP of the load balancer."
+}
+
+variable "ports" {
+  type        = list(number)
+  description = "Ports to forward to the load balancer."
+}

cli/internal/terraform/terraform/gcp/outputs.tf

@@ -1,13 +1,22 @@
-output "ip" {
-  value = google_compute_global_address.loadbalancer_ip.address
+output "out_of_cluster_endpoint" {
+  value = local.out_of_cluster_endpoint
 }
 
+output "in_cluster_endpoint" {
+  value = local.in_cluster_endpoint
+}
 output "api_server_cert_sans" {
-  value = sort(concat([google_compute_global_address.loadbalancer_ip.address], var.custom_endpoint == "" ? [] : [var.custom_endpoint]))
-}
-
-output "fallback_endpoint" {
-  value = google_compute_global_address.loadbalancer_ip.address
+  value = sort(
+    distinct(
+      concat(
+        [
+          local.in_cluster_endpoint,
+          local.out_of_cluster_endpoint,
+        ],
+        var.custom_endpoint == "" ? [] : [var.custom_endpoint],
+      )
+    )
+  )
 }
 
 output "uid" {

cli/internal/terraform/terraform/gcp/variables.tf

@@ -51,3 +51,9 @@ variable "custom_endpoint" {
   default     = ""
   description = "Custom endpoint to use for the Kubernetes apiserver. If not set, the default endpoint will be used."
 }
+
+variable "internal_load_balancer" {
+  type        = bool
+  default     = false
+  description = "Enable internal load balancer. This can only be enabled if the control-plane is deployed in one zone."
+}

cli/internal/terraform/terraform_test.go

@@ -212,9 +212,12 @@ func TestCreateCluster(t *testing.T) {
 		workingState := tfjson.State{
 			Values: &tfjson.StateValues{
 				Outputs: map[string]*tfjson.StateOutput{
-					"ip": {
+					"out_of_cluster_endpoint": {
 						Value: "192.0.2.100",
 					},
+					"in_cluster_endpoint": {
+						Value: "192.0.2.101",
+					},
 					"initSecret": {
 						Value: "initSecret",
 					},
@@ -236,9 +239,12 @@ func TestCreateCluster(t *testing.T) {
 		workingState := tfjson.State{
 			Values: &tfjson.StateValues{
 				Outputs: map[string]*tfjson.StateOutput{
-					"ip": {
+					"out_of_cluster_endpoint": {
 						Value: "192.0.2.100",
 					},
+					"in_cluster_endpoint": {
+						Value: "192.0.2.101",
+					},
 					"initSecret": {
 						Value: "initSecret",
 					},
@@ -480,6 +486,7 @@ func TestCreateCluster(t *testing.T) {
 			assert.Equal("192.0.2.100", infraState.ClusterEndpoint)
 			assert.Equal(state.HexBytes("initSecret"), infraState.InitSecret)
 			assert.Equal("12345abc", infraState.UID)
+			assert.Equal("192.0.2.101", infraState.InClusterEndpoint)
 			if tc.provider == cloudprovider.Azure {
 				assert.Equal(tc.expectedAttestationURL, infraState.Azure.AttestationURL)
 			}

cli/internal/terraform/variables.go

@@ -66,6 +66,8 @@ type AWSClusterVariables struct {
 	NodeGroups map[string]AWSNodeGroup `hcl:"node_groups" cty:"node_groups"`
 	// CustomEndpoint is the (optional) custom dns hostname for the kubernetes api server.
 	CustomEndpoint string `hcl:"custom_endpoint" cty:"custom_endpoint"`
+	// InternalLoadBalancer is true if an internal load balancer should be created.
+	InternalLoadBalancer bool `hcl:"internal_load_balancer" cty:"internal_load_balancer"`
 }
 
 // GetCreateMAA gets the CreateMAA variable.
@@ -131,6 +133,8 @@ type GCPClusterVariables struct {
 	NodeGroups map[string]GCPNodeGroup `hcl:"node_groups" cty:"node_groups"`
 	// CustomEndpoint is the (optional) custom dns hostname for the kubernetes api server.
 	CustomEndpoint string `hcl:"custom_endpoint" cty:"custom_endpoint"`
+	// InternalLoadBalancer is true if an internal load balancer should be created.
+	InternalLoadBalancer bool `hcl:"internal_load_balancer" cty:"internal_load_balancer"`
 }
 
 // GetCreateMAA gets the CreateMAA variable.
@@ -203,6 +207,8 @@ type AzureClusterVariables struct {
 	NodeGroups map[string]AzureNodeGroup `hcl:"node_groups" cty:"node_groups"`
 	// CustomEndpoint is the (optional) custom dns hostname for the kubernetes api server.
 	CustomEndpoint string `hcl:"custom_endpoint" cty:"custom_endpoint"`
+	// InternalLoadBalancer is true if an internal load balancer should be created.
+	InternalLoadBalancer bool `hcl:"internal_load_balancer" cty:"internal_load_balancer"`
 }
 
 // GetCreateMAA gets the CreateMAA variable.
@@ -275,6 +281,8 @@ type OpenStackClusterVariables struct {
 	Debug bool `hcl:"debug" cty:"debug"`
 	// CustomEndpoint is the (optional) custom dns hostname for the kubernetes api server.
 	CustomEndpoint string `hcl:"custom_endpoint" cty:"custom_endpoint"`
+	// InternalLoadBalancer is true if an internal load balancer should be created.
+	InternalLoadBalancer bool `hcl:"internal_load_balancer" cty:"internal_load_balancer"`
 }
 
 // GetCreateMAA gets the CreateMAA variable.
@@ -346,6 +354,8 @@ type QEMUVariables struct {
 	KernelCmdline *string `hcl:"constellation_cmdline" cty:"constellation_cmdline"`
 	// CustomEndpoint is the (optional) custom dns hostname for the kubernetes api server.
 	CustomEndpoint string `hcl:"custom_endpoint" cty:"custom_endpoint"`
+	// InternalLoadBalancer is true if an internal load balancer should be created.
+	InternalLoadBalancer bool `hcl:"internal_load_balancer" cty:"internal_load_balancer"`
 }
 
 // GetCreateMAA gets the CreateMAA variable.

cli/internal/terraform/variables_test.go

@@ -74,6 +74,7 @@ node_groups = {
   }
 }
 custom_endpoint        = "example.com"
+internal_load_balancer = false
 `
 	got := vars.String()
 	assert.Equal(t, want, got)
@@ -148,6 +149,7 @@ node_groups = {
   }
 }
 custom_endpoint        = "example.com"
+internal_load_balancer = false
 `
 	got := vars.String()
 	assert.Equal(t, want, got)
@@ -213,6 +215,7 @@ node_groups = {
   }
 }
 custom_endpoint        = "example.com"
+internal_load_balancer = false
 `
 	got := vars.String()
 	assert.Equal(t, want, got)
@@ -279,6 +282,7 @@ openstack_username         = "my-username"
 openstack_password         = "my-password"
 debug                      = true
 custom_endpoint            = "example.com"
+internal_load_balancer     = false
 `
 	got := vars.String()
 	assert.Equal(t, want, got)
@@ -333,6 +337,7 @@ nvram                   = "/usr/share/OVMF/OVMF_VARS.fd"
 constellation_initrd    = "/var/lib/libvirt/images/cluster-name-initrd"
 constellation_cmdline   = "console=ttyS0,115200n8"
 custom_endpoint         = "example.com"
+internal_load_balancer  = false
 `
 	got := vars.String()
 	assert.Equal(t, want, got)

go.sum

@@ -792,6 +792,8 @@ github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw=
 github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
 github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de h1:9TO3cAIGXtEhnIaL+V+BEER86oLrvS+kWobKpbJuye0=
 github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de/go.mod h1:zAbeS9B/r2mtpb6U+EI2rYA5OAXxsYw6wTamcNW+zcE=
+github.com/lithammer/dedent v1.1.0 h1:VNzHMVCBNG1j0fh3OrsFRkVUwStdDArbgBWoPAffktY=
+github.com/lithammer/dedent v1.1.0/go.mod h1:jrXYCQtgg0nJiN+StA2KgR7w6CiQNv9Fd/Z9BP0jIOc=
 github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
 github.com/magiconair/properties v1.8.5/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60=
 github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=

hack/go.sum

@@ -736,6 +736,8 @@ github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw=
 github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
 github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de h1:9TO3cAIGXtEhnIaL+V+BEER86oLrvS+kWobKpbJuye0=
 github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de/go.mod h1:zAbeS9B/r2mtpb6U+EI2rYA5OAXxsYw6wTamcNW+zcE=
+github.com/lithammer/dedent v1.1.0 h1:VNzHMVCBNG1j0fh3OrsFRkVUwStdDArbgBWoPAffktY=
+github.com/lithammer/dedent v1.1.0/go.mod h1:jrXYCQtgg0nJiN+StA2KgR7w6CiQNv9Fd/Z9BP0jIOc=
 github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
 github.com/magiconair/properties v1.8.5/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60=
 github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=

internal/cloud/aws/aws.go

@@ -130,65 +130,14 @@ func (c *Cloud) InitSecretHash(ctx context.Context) ([]byte, error) {
 }
 
 // GetLoadBalancerEndpoint returns the endpoint of the load balancer.
-// TODO(malt3): remove old infrastructure code once we have migrated to using DNS as the load balancer endpoint.
 func (c *Cloud) GetLoadBalancerEndpoint(ctx context.Context) (host, port string, err error) {
-	// try new architecture first
-	uid, err := c.readInstanceTag(ctx, cloud.TagUID)
+	hostname, err := c.getLoadBalancerDNSName(ctx)
 	if err != nil {
-		return "", "", fmt.Errorf("retrieving uid tag: %w", err)
-	}
-	describeIPsOutput, err := c.ec2.DescribeAddresses(ctx, &ec2.DescribeAddressesInput{
-		Filters: []ec2Types.Filter{
-			{
-				Name:   aws.String(cloud.TagUID),
-				Values: []string{uid},
-			},
-			{
-				Name:   aws.String("constellation-ip-endpoint"),
-				Values: []string{"legacy-primary-zone"},
-			},
-		},
-	})
-	if err == nil && len(describeIPsOutput.Addresses) == 1 && describeIPsOutput.Addresses[0].PublicIp != nil {
-		return *describeIPsOutput.Addresses[0].PublicIp, strconv.FormatInt(constants.KubernetesPort, 10), nil
-	}
-	// fallback to old architecture
-	// this will be removed in the future
-	hostname, err := c.getLoadBalancerIPOldInfrastructure(ctx)
-	if err != nil {
-		return "", "", fmt.Errorf("retrieving load balancer ip: %w", err)
+		return "", "", fmt.Errorf("retrieving load balancer dns name: %w", err)
 	}
 	return hostname, strconv.FormatInt(constants.KubernetesPort, 10), nil
 }
 
-// getLoadBalancerIPOldInfrastructure returns the IP of the load balancer.
-// This is only used for the old infrastructure.
-// This will be removed in the future.
-func (c *Cloud) getLoadBalancerIPOldInfrastructure(ctx context.Context) (string, error) {
-	loadbalancer, err := c.getLoadBalancer(ctx)
-	if err != nil {
-		return "", fmt.Errorf("finding Constellation load balancer: %w", err)
-	}
-
-	// TODO(malt3): Add support for multiple availability zones in the lb frontend.
-	// This can only be done after we have migrated to using DNS as the load balancer endpoint.
-	// At that point, we don't need to care about the number of availability zones anymore.
-	if len(loadbalancer.AvailabilityZones) != 1 {
-		return "", fmt.Errorf("%d availability zones found; expected 1", len(loadbalancer.AvailabilityZones))
-	}
-
-	if len(loadbalancer.AvailabilityZones[0].LoadBalancerAddresses) != 1 {
-		return "", fmt.Errorf("%d load balancer addresses found; expected 1", len(loadbalancer.AvailabilityZones[0].LoadBalancerAddresses))
-	}
-	if loadbalancer.AvailabilityZones[0].LoadBalancerAddresses[0].IpAddress == nil {
-		return "", errors.New("load balancer address is nil")
-	}
-
-	return *loadbalancer.AvailabilityZones[0].LoadBalancerAddresses[0].IpAddress, nil
-}
-
-/*
-// TODO(malt3): uncomment and use as soon as we switch the primary endpoint to DNS.
 func (c *Cloud) getLoadBalancerDNSName(ctx context.Context) (string, error) {
 	loadbalancer, err := c.getLoadBalancer(ctx)
 	if err != nil {
@@ -199,7 +148,6 @@ func (c *Cloud) getLoadBalancerDNSName(ctx context.Context) (string, error) {
 	}
 	return *loadbalancer.DNSName, nil
 }
-*/
 
 func (c *Cloud) getLoadBalancer(ctx context.Context) (*elasticloadbalancingv2types.LoadBalancer, error) {
 	uid, err := c.readInstanceTag(ctx, cloud.TagUID)

internal/cloud/aws/aws_test.go

@@ -17,7 +17,7 @@ import (
 	ec2Types "github.com/aws/aws-sdk-go-v2/service/ec2/types"
 	"github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2"
 	elbTypes "github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2/types"
-	tagTypes "github.com/aws/aws-sdk-go-v2/service/resourcegroupstaggingapi/types"
+	rgtTypes "github.com/aws/aws-sdk-go-v2/service/resourcegroupstaggingapi/types"
 
 	"github.com/aws/aws-sdk-go-v2/service/resourcegroupstaggingapi"
 	"github.com/edgelesssys/constellation/v2/internal/cloud"
@@ -445,251 +445,69 @@ func TestList(t *testing.T) {
 
 func TestGetLoadBalancerEndpoint(t *testing.T) {
 	lbAddr := "192.0.2.1"
-	someErr := errors.New("some error")
+	successfulEC2 := &stubEC2{
+		selfInstance: &ec2.DescribeInstancesOutput{
+			Reservations: []ec2Types.Reservation{
+				{
+					Instances: []ec2Types.Instance{
+						{
+							InstanceId: aws.String("id-1"),
+							Tags: []ec2Types.Tag{
+								{
+									Key:   aws.String(cloud.TagRole),
+									Value: aws.String("controlplane"),
+								},
+								{
+									Key:   aws.String(cloud.TagUID),
+									Value: aws.String("uid"),
+								},
+							},
+						},
+					},
+				},
+			},
+		},
+	}
 
 	testCases := map[string]struct {
 		imds         *stubIMDS
-		ec2API       *stubEC2
 		loadbalancer *stubLoadbalancer
 		resourceapi  *stubResourceGroupTagging
 		wantHost     string
 		wantErr      bool
 	}{
-		"success retrieving loadbalancer endpoint": {
+		"success": {
 			imds: &stubIMDS{
 				instanceDocumentResp: &imds.GetInstanceIdentityDocumentOutput{
 					InstanceIdentityDocument: imds.InstanceIdentityDocument{
-						InstanceID: "test-instance-id",
+						AvailabilityZone: "test-zone",
 					},
 				},
 			},
-			ec2API: &stubEC2{
-				selfInstance: &ec2.DescribeInstancesOutput{
-					Reservations: []ec2Types.Reservation{
-						{
-							Instances: []ec2Types.Instance{
-								{
-									InstanceId: aws.String("test-instance-id"),
-									Tags: []ec2Types.Tag{
-										{
-											Key:   aws.String(cloud.TagUID),
-											Value: aws.String("uid"),
-										},
-									},
-								},
-							},
-						},
-					},
-				},
-				describeAddressesResp: &ec2.DescribeAddressesOutput{
-					Addresses: []ec2Types.Address{
-						{
-							Tags: []ec2Types.Tag{
-								{Key: aws.String(cloud.TagUID), Value: aws.String("uid")},
-								{Key: aws.String("constellation-ip-endpoint"), Value: aws.String("legacy-primary-zone")},
-							},
-							PublicIp: aws.String(lbAddr),
-						},
-					},
-				},
-			},
-			wantHost: lbAddr,
-		},
-		"success retrieving loadbalancer endpoint legacy": {
-			imds: &stubIMDS{
-				instanceDocumentResp: &imds.GetInstanceIdentityDocumentOutput{
-					InstanceIdentityDocument: imds.InstanceIdentityDocument{
-						InstanceID: "test-instance-id",
-					},
-				},
-			},
-			ec2API: &stubEC2{
-				selfInstance: &ec2.DescribeInstancesOutput{
-					Reservations: []ec2Types.Reservation{
-						{
-							Instances: []ec2Types.Instance{
-								{
-									InstanceId: aws.String("test-instance-id"),
-									Tags: []ec2Types.Tag{
-										{
-											Key:   aws.String(cloud.TagUID),
-											Value: aws.String("uid"),
-										},
-									},
-								},
-							},
-						},
-					},
-				},
-				describeAddressesErr: errors.New("using legacy infrastructure"),
-			},
 			loadbalancer: &stubLoadbalancer{
 				describeLoadBalancersOut: &elasticloadbalancingv2.DescribeLoadBalancersOutput{
 					LoadBalancers: []elbTypes.LoadBalancer{
 						{
+							LoadBalancerName: aws.String("test-lb"),
 							AvailabilityZones: []elbTypes.AvailabilityZone{
 								{
-									LoadBalancerAddresses: []elbTypes.LoadBalancerAddress{
-										{
-											IpAddress: aws.String(lbAddr),
-										},
-									},
+									ZoneName: aws.String("test-zone"),
 								},
 							},
+							DNSName: aws.String(lbAddr),
 						},
 					},
 				},
 			},
 			resourceapi: &stubResourceGroupTagging{
 				getResourcesOut1: &resourcegroupstaggingapi.GetResourcesOutput{
-					ResourceTagMappingList: []tagTypes.ResourceTagMapping{
-						{
-							ResourceARN: aws.String("arn:aws:elasticloadbalancing:us-east-1:123456789012:loadbalancer/app/test-loadbalancer/50dc6c495c0c9188"),
-						},
-					},
-				},
-			},
-			wantHost: lbAddr,
-		},
-		"too many ARNs legacy": {
-			imds: &stubIMDS{
-				instanceDocumentResp: &imds.GetInstanceIdentityDocumentOutput{
-					InstanceIdentityDocument: imds.InstanceIdentityDocument{
-						InstanceID: "test-instance-id",
-					},
-				},
-			},
-			ec2API: &stubEC2{
-				selfInstance: &ec2.DescribeInstancesOutput{
-					Reservations: []ec2Types.Reservation{
-						{
-							Instances: []ec2Types.Instance{
-								{
-									InstanceId: aws.String("test-instance-id"),
-									Tags: []ec2Types.Tag{
-										{
-											Key:   aws.String(cloud.TagUID),
-											Value: aws.String("uid"),
-										},
-									},
-								},
-							},
-						},
-					},
-				},
-				describeAddressesErr: errors.New("using legacy infrastructure"),
-			},
-			loadbalancer: &stubLoadbalancer{
-				describeLoadBalancersOut: &elasticloadbalancingv2.DescribeLoadBalancersOutput{
-					LoadBalancers: []elbTypes.LoadBalancer{
-						{
-							AvailabilityZones: []elbTypes.AvailabilityZone{
-								{
-									LoadBalancerAddresses: []elbTypes.LoadBalancerAddress{
-										{
-											IpAddress: aws.String(lbAddr),
-										},
-									},
-								},
-							},
-						},
-					},
-				},
-			},
-			resourceapi: &stubResourceGroupTagging{
-				getResourcesOut1: &resourcegroupstaggingapi.GetResourcesOutput{
-					ResourceTagMappingList: []tagTypes.ResourceTagMapping{
-						{
-							ResourceARN: aws.String("arn:aws:elasticloadbalancing:us-east-1:123456789012:loadbalancer/app/test-loadbalancer/50dc6c495c0c9188"),
-						},
-						{
-							ResourceARN: aws.String("arn:aws:elasticloadbalancing:us-east-1:123456789012:loadbalancer/app/test-loadbalancer/50dc6c495c0c9188"),
-						},
-					},
-				},
-			},
-			wantErr: true,
-		},
-		"too many ARNs (paged) legacy": {
-			imds: &stubIMDS{
-				instanceDocumentResp: &imds.GetInstanceIdentityDocumentOutput{
-					InstanceIdentityDocument: imds.InstanceIdentityDocument{
-						InstanceID: "test-instance-id",
-					},
-				},
-			},
-			ec2API: &stubEC2{
-				selfInstance: &ec2.DescribeInstancesOutput{
-					Reservations: []ec2Types.Reservation{
-						{
-							Instances: []ec2Types.Instance{
-								{
-									InstanceId: aws.String("test-instance-id"),
-									Tags: []ec2Types.Tag{
-										{
-											Key:   aws.String(cloud.TagUID),
-											Value: aws.String("uid"),
-										},
-									},
-								},
-							},
-						},
-					},
-				},
-				describeAddressesErr: errors.New("using legacy infrastructure"),
-			},
-			loadbalancer: &stubLoadbalancer{
-				describeLoadBalancersOut: &elasticloadbalancingv2.DescribeLoadBalancersOutput{
-					LoadBalancers: []elbTypes.LoadBalancer{
-						{
-							AvailabilityZones: []elbTypes.AvailabilityZone{
-								{
-									LoadBalancerAddresses: []elbTypes.LoadBalancerAddress{
-										{
-											IpAddress: aws.String(lbAddr),
-										},
-									},
-								},
-							},
-						},
-					},
-				},
-			},
-			resourceapi: &stubResourceGroupTagging{
-				getResourcesOut1: &resourcegroupstaggingapi.GetResourcesOutput{
-					ResourceTagMappingList: []tagTypes.ResourceTagMapping{
-						{
-							ResourceARN: aws.String("arn:aws:elasticloadbalancing:us-east-1:123456789012:loadbalancer/app/test-loadbalancer/50dc6c495c0c9188"),
-						},
-					},
-					PaginationToken: aws.String("token"),
+					PaginationToken: aws.String("next-token"),
 				},
 				getResourcesOut2: &resourcegroupstaggingapi.GetResourcesOutput{
-					ResourceTagMappingList: []tagTypes.ResourceTagMapping{
+					ResourceTagMappingList: []rgtTypes.ResourceTagMapping{
 						{
-							ResourceARN: aws.String("arn:aws:elasticloadbalancing:us-east-1:123456789012:loadbalancer/app/test-loadbalancer/50dc6c495c0c9188"),
-						},
-					},
-				},
-			},
-			wantErr: true,
-		},
-		"loadbalancer has no availability zones legacy": {
-			imds: &stubIMDS{
-				instanceDocumentResp: &imds.GetInstanceIdentityDocumentOutput{
-					InstanceIdentityDocument: imds.InstanceIdentityDocument{
-						InstanceID: "test-instance-id",
-					},
-				},
-			},
-			ec2API: &stubEC2{
-				selfInstance: &ec2.DescribeInstancesOutput{
-					Reservations: []ec2Types.Reservation{
-						{
-							Instances: []ec2Types.Instance{
-								{
-									InstanceId: aws.String("test-instance-id"),
-									Tags: []ec2Types.Tag{
+							ResourceARN: aws.String("arn:aws:elasticloadbalancing:us-east-1:123456789012:loadbalancer/app/test-lb/1234567890abcdef"),
+							Tags: []rgtTypes.Tag{
 								{
 									Key:   aws.String(cloud.TagUID),
 									Value: aws.String("uid"),
@@ -699,45 +517,72 @@ func TestGetLoadBalancerEndpoint(t *testing.T) {
 					},
 				},
 			},
+			wantHost: lbAddr,
+		},
+		"no load balancer found": {
+			imds: &stubIMDS{
+				instanceDocumentResp: &imds.GetInstanceIdentityDocumentOutput{
+					InstanceIdentityDocument: imds.InstanceIdentityDocument{
+						AvailabilityZone: "test-zone",
+					},
+				},
+			},
+			loadbalancer: &stubLoadbalancer{
+				describeLoadBalancersOut: &elasticloadbalancingv2.DescribeLoadBalancersOutput{
+					LoadBalancers: []elbTypes.LoadBalancer{},
+				},
+			},
+			resourceapi: &stubResourceGroupTagging{
+				getResourcesOut1: &resourcegroupstaggingapi.GetResourcesOutput{
+					PaginationToken: aws.String("next-token"),
+				},
+				getResourcesOut2: &resourcegroupstaggingapi.GetResourcesOutput{
+					ResourceTagMappingList: []rgtTypes.ResourceTagMapping{
+						{
+							ResourceARN: aws.String("arn:aws:elasticloadbalancing:us-east-1:123456789012:loadbalancer/app/test-lb/1234567890abcdef"),
+							Tags: []rgtTypes.Tag{
+								{
+									Key:   aws.String(cloud.TagUID),
+									Value: aws.String("uid"),
+								},
+							},
+						},
+					},
+				},
+			},
+			wantErr: true,
+		},
+		"no load balancer DNS name": {
+			imds: &stubIMDS{
+				instanceDocumentResp: &imds.GetInstanceIdentityDocumentOutput{
+					InstanceIdentityDocument: imds.InstanceIdentityDocument{
+						AvailabilityZone: "test-zone",
+					},
 				},
-				describeAddressesErr: errors.New("using legacy infrastructure"),
 			},
 			loadbalancer: &stubLoadbalancer{
 				describeLoadBalancersOut: &elasticloadbalancingv2.DescribeLoadBalancersOutput{
 					LoadBalancers: []elbTypes.LoadBalancer{
 						{
-							AvailabilityZones: []elbTypes.AvailabilityZone{},
+							LoadBalancerName: aws.String("test-lb"),
+							AvailabilityZones: []elbTypes.AvailabilityZone{
+								{
+									ZoneName: aws.String("test-zone"),
+								},
+							},
 						},
 					},
 				},
 			},
 			resourceapi: &stubResourceGroupTagging{
 				getResourcesOut1: &resourcegroupstaggingapi.GetResourcesOutput{
-					ResourceTagMappingList: []tagTypes.ResourceTagMapping{
+					PaginationToken: aws.String("next-token"),
+				},
+				getResourcesOut2: &resourcegroupstaggingapi.GetResourcesOutput{
+					ResourceTagMappingList: []rgtTypes.ResourceTagMapping{
 						{
-							ResourceARN: aws.String("arn:aws:elasticloadbalancing:us-east-1:123456789012:loadbalancer/app/test-loadbalancer/50dc6c495c0c9188"),
-						},
-					},
-				},
-			},
-			wantErr: true,
-		},
-		"failure to get resources by tag legacy": {
-			imds: &stubIMDS{
-				instanceDocumentResp: &imds.GetInstanceIdentityDocumentOutput{
-					InstanceIdentityDocument: imds.InstanceIdentityDocument{
-						InstanceID: "test-instance-id",
-					},
-				},
-			},
-			ec2API: &stubEC2{
-				selfInstance: &ec2.DescribeInstancesOutput{
-					Reservations: []ec2Types.Reservation{
-						{
-							Instances: []ec2Types.Instance{
-								{
-									InstanceId: aws.String("test-instance-id"),
-									Tags: []ec2Types.Tag{
+							ResourceARN: aws.String("arn:aws:elasticloadbalancing:us-east-1:123456789012:loadbalancer/app/test-lb/1234567890abcdef"),
+							Tags: []rgtTypes.Tag{
 								{
 									Key:   aws.String(cloud.TagUID),
 									Value: aws.String("uid"),
@@ -747,28 +592,93 @@ func TestGetLoadBalancerEndpoint(t *testing.T) {
 					},
 				},
 			},
+			wantErr: true,
+		},
+		"describe load balancers fails": {
+			imds: &stubIMDS{
+				instanceDocumentResp: &imds.GetInstanceIdentityDocumentOutput{
+					InstanceIdentityDocument: imds.InstanceIdentityDocument{
+						AvailabilityZone: "test-zone",
+					},
+				},
+			},
+			resourceapi: &stubResourceGroupTagging{
+				getResourcesOut1: &resourcegroupstaggingapi.GetResourcesOutput{
+					PaginationToken: aws.String("next-token"),
+				},
+				getResourcesOut2: &resourcegroupstaggingapi.GetResourcesOutput{
+					ResourceTagMappingList: []rgtTypes.ResourceTagMapping{
+						{
+							ResourceARN: aws.String("arn:aws:elasticloadbalancing:us-east-1:123456789012:loadbalancer/app/test-lb/1234567890abcdef"),
+							Tags: []rgtTypes.Tag{
+								{
+									Key:   aws.String(cloud.TagUID),
+									Value: aws.String("uid"),
+								},
+							},
+						},
+					},
+				},
+			},
+			loadbalancer: &stubLoadbalancer{
+				describeLoadBalancersErr: assert.AnError,
+			},
+			wantErr: true,
+		},
+		"get resources fails": {
+			imds: &stubIMDS{
+				instanceDocumentResp: &imds.GetInstanceIdentityDocumentOutput{
+					InstanceIdentityDocument: imds.InstanceIdentityDocument{},
 				},
-				describeAddressesErr: errors.New("using legacy infrastructure"),
 			},
 			loadbalancer: &stubLoadbalancer{
 				describeLoadBalancersOut: &elasticloadbalancingv2.DescribeLoadBalancersOutput{
 					LoadBalancers: []elbTypes.LoadBalancer{
 						{
+							LoadBalancerName: aws.String("test-lb"),
 							AvailabilityZones: []elbTypes.AvailabilityZone{
 								{
-									LoadBalancerAddresses: []elbTypes.LoadBalancerAddress{
-										{
-											IpAddress: aws.String(lbAddr),
-										},
-									},
+									ZoneName: aws.String("test-zone"),
 								},
 							},
+							DNSName: aws.String(lbAddr),
 						},
 					},
 				},
 			},
 			resourceapi: &stubResourceGroupTagging{
-				getResourcesErr: someErr,
+				getResourcesErr: assert.AnError,
+			},
+			wantErr: true,
+		},
+		"no resources found": {
+			imds: &stubIMDS{
+				instanceDocumentResp: &imds.GetInstanceIdentityDocumentOutput{
+					InstanceIdentityDocument: imds.InstanceIdentityDocument{},
+				},
+			},
+			loadbalancer: &stubLoadbalancer{
+				describeLoadBalancersOut: &elasticloadbalancingv2.DescribeLoadBalancersOutput{
+					LoadBalancers: []elbTypes.LoadBalancer{
+						{
+							LoadBalancerName: aws.String("test-lb"),
+							AvailabilityZones: []elbTypes.AvailabilityZone{
+								{
+									ZoneName: aws.String("test-zone"),
+								},
+							},
+							DNSName: aws.String(lbAddr),
+						},
+					},
+				},
+			},
+			resourceapi: &stubResourceGroupTagging{
+				getResourcesOut1: &resourcegroupstaggingapi.GetResourcesOutput{
+					PaginationToken: aws.String("next-token"),
+				},
+				getResourcesOut2: &resourcegroupstaggingapi.GetResourcesOutput{
+					ResourceTagMappingList: []rgtTypes.ResourceTagMapping{},
+				},
 			},
 			wantErr: true,
 		},
@@ -779,9 +689,9 @@ func TestGetLoadBalancerEndpoint(t *testing.T) {
 			assert := assert.New(t)
 			m := &Cloud{
 				imds:              tc.imds,
-				ec2:               tc.ec2API,
 				loadbalancer:      tc.loadbalancer,
 				resourceapiClient: tc.resourceapi,
+				ec2:               successfulEC2,
 			}
 
 			gotHost, gotPort, err := m.GetLoadBalancerEndpoint(context.Background())

internal/cloud/azure/BUILD.bazel

@@ -16,6 +16,7 @@ go_library(
         "//internal/cloud/azureshared",
         "//internal/cloud/metadata",
         "//internal/constants",
+        "//internal/logger",
         "//internal/role",
         "@com_github_azure_azure_sdk_for_go_sdk_azcore//runtime",
         "@com_github_azure_azure_sdk_for_go_sdk_azidentity//:azidentity",
@@ -23,6 +24,9 @@ go_library(
         "@com_github_azure_azure_sdk_for_go_sdk_resourcemanager_compute_armcompute_v5//:armcompute",
         "@com_github_azure_azure_sdk_for_go_sdk_resourcemanager_network_armnetwork_v4//:armnetwork",
         "@com_github_microsoft_applicationinsights_go//appinsights",
+        "@io_k8s_kubernetes//pkg/util/iptables",
+        "@io_k8s_utils//exec",
+        "@org_uber_go_zap//:zap",
     ],
 )
 

internal/cloud/azure/azure.go

@@ -29,7 +29,11 @@ import (
 	"github.com/edgelesssys/constellation/v2/internal/cloud/azureshared"
 	"github.com/edgelesssys/constellation/v2/internal/cloud/metadata"
 	"github.com/edgelesssys/constellation/v2/internal/constants"
+	"github.com/edgelesssys/constellation/v2/internal/logger"
 	"github.com/edgelesssys/constellation/v2/internal/role"
+	"go.uber.org/zap"
+	"k8s.io/kubernetes/pkg/util/iptables"
+	"k8s.io/utils/exec"
 )
 
 // Cloud provides Azure metadata and API access.
@@ -102,13 +106,25 @@ func New(ctx context.Context) (*Cloud, error) {
 //
 // The returned string is an IP address without a port, but the method name needs to satisfy the
 // metadata interface.
-func (c *Cloud) GetLoadBalancerEndpoint(ctx context.Context) (host, port string, err error) {
+func (c *Cloud) GetLoadBalancerEndpoint(ctx context.Context) (host, port string, retErr error) {
+	var multiErr error
+
+	// Try to retrieve the public IP first
 	hostname, err := c.getLoadBalancerPublicIP(ctx)
-	if err != nil {
-		return "", "", fmt.Errorf("retrieving load balancer public IP: %w", err)
-	}
+	if err == nil {
 		return hostname, strconv.FormatInt(constants.KubernetesPort, 10), nil
 	}
+	multiErr = fmt.Errorf("retrieving load balancer public IP: %w", err)
+
+	// If that fails, try to retrieve the private IP
+	hostname, err = c.getLoadBalancerPrivateIP(ctx)
+	if err == nil {
+		return hostname, strconv.FormatInt(constants.KubernetesPort, 10), nil
+	}
+	multiErr = errors.Join(multiErr, fmt.Errorf("retrieving load balancer private IP: %w", err))
+
+	return "", "", multiErr
+}
 
 // List retrieves all instances belonging to the current constellation.
 func (c *Cloud) List(ctx context.Context) ([]metadata.InstanceMetadata, error) {
@@ -308,6 +324,39 @@ func (c *Cloud) getVMInterfaces(ctx context.Context, vm armcompute.VirtualMachin
 	return networkInterfaces, nil
 }
 
+// getLoadBalancerPrivateIP retrieves the first load balancer IP from cloud provider metadata.
+func (c *Cloud) getLoadBalancerPrivateIP(ctx context.Context) (string, error) {
+	resourceGroup, err := c.imds.resourceGroup(ctx)
+	if err != nil {
+		return "", fmt.Errorf("retrieving resource group: %w", err)
+	}
+	uid, err := c.imds.uid(ctx)
+	if err != nil {
+		return "", fmt.Errorf("retrieving instance UID: %w", err)
+	}
+
+	lb, err := c.getLoadBalancer(ctx, resourceGroup, uid)
+	if err != nil {
+		return "", fmt.Errorf("retrieving load balancer: %w", err)
+	}
+	if lb == nil || lb.Properties == nil {
+		return "", errors.New("could not dereference load balancer IP configuration")
+	}
+
+	var privIP string
+	for _, fipConf := range lb.Properties.FrontendIPConfigurations {
+		if fipConf != nil && fipConf.Properties != nil && fipConf.Properties.PrivateIPAddress != nil {
+			privIP = *fipConf.Properties.PrivateIPAddress
+			break
+		}
+	}
+	if privIP == "" {
+		return "", errors.New("could not resolve private IP address for load balancer")
+	}
+
+	return privIP, nil
+}
+
 // getLoadBalancerPublicIP retrieves the first load balancer IP from cloud provider metadata.
 func (c *Cloud) getLoadBalancerPublicIP(ctx context.Context) (string, error) {
 	resourceGroup, err := c.imds.resourceGroup(ctx)
@@ -348,6 +397,9 @@ func (c *Cloud) getLoadBalancerPublicIP(ctx context.Context) (string, error) {
 
 /*
 // TODO(malt3): uncomment and use as soon as we switch the primary endpoint to DNS.
+// Addition from 3u13r: We have to think about how to handle DNS for internal load balancers
+// that only have a private IP address and therefore no DNS name by default.
+//
 // getLoadBalancerDNSName retrieves the dns name of the load balancer.
 // On Azure, the DNS name is the DNS name of the public IP address of the load balancer.
 func (c *Cloud) getLoadBalancerDNSName(ctx context.Context) (string, error) {
@@ -388,6 +440,68 @@ func (c *Cloud) getLoadBalancerDNSName(ctx context.Context) (string, error) {
 }
 */
 
+// PrepareControlPlaneNode sets up iptables for the control plane node only
+// if an internal load balancer is used.
+//
+// This is needed since during `kubeadm init` the API server must talk to the
+// kubeAPIEndpoint, which is the load balancer IP address. During that time, the
+// only healthy VM is the VM itself. Therefore, traffic is sent to the load balancer
+// and the 5-tuple is (VM IP, <some port>, LB IP, 6443, TCP).
+// Now the load balancer does not re-write the source IP address only the destination (DNAT).
+// Therefore the 5-tuple is (VM IP, <some port>, VM IP, 6443, TCP).
+// Now the VM responds to the SYN packet with a SYN-ACK packet, but the outgoing
+// connection waits on a response from the load balancer and not the VM therefore
+// dropping the packet.
+//
+// OpenShift also uses the same mechanism to redirect traffic to the API server:
+// https://github.com/openshift/machine-config-operator/blob/e453bd20bac0e48afa74e9a27665abaf454d93cd/templates/master/00-master/azure/files/opt-libexec-openshift-azure-routes-sh.yaml
+func (c *Cloud) PrepareControlPlaneNode(ctx context.Context, log *logger.Logger) error {
+	selfMetadata, err := c.Self(ctx)
+	if err != nil {
+		return fmt.Errorf("failed to get self metadata: %w", err)
+	}
+
+	// skipping iptables setup for worker nodes
+	if selfMetadata.Role != role.ControlPlane {
+		log.Infof("not a control plane node, skipping iptables setup")
+		return nil
+	}
+
+	// skipping iptables setup if no internal LB exists e.g.
+	// for public LB architectures
+	loadbalancerIP, err := c.getLoadBalancerPrivateIP(ctx)
+	if err != nil {
+		log.With(zap.Error(err)).Warnf("skipping iptables setup, failed to get load balancer private IP")
+		return nil
+	}
+
+	log.Infof("Setting up iptables for control plane node with load balancer IP %s", loadbalancerIP)
+
+	iptablesExec := iptables.New(exec.New(), iptables.ProtocolIPv4)
+	if err != nil {
+		return fmt.Errorf("failed to create iptables client: %w", err)
+	}
+
+	const chainName = "azure-lb-nat"
+	if _, err := iptablesExec.EnsureChain(iptables.TableNAT, chainName); err != nil {
+		return fmt.Errorf("failed to create iptables chain: %w", err)
+	}
+
+	if _, err := iptablesExec.EnsureRule(iptables.Append, iptables.TableNAT, "PREROUTING", "-j", chainName); err != nil {
+		return fmt.Errorf("failed to add rule to iptables chain: %w", err)
+	}
+
+	if _, err := iptablesExec.EnsureRule(iptables.Append, iptables.TableNAT, "OUTPUT", "-j", chainName); err != nil {
+		return fmt.Errorf("failed to add rule to iptables chain: %w", err)
+	}
+
+	if _, err := iptablesExec.EnsureRule(iptables.Append, iptables.TableNAT, chainName, "--dst", loadbalancerIP, "-p", "tcp", "--dport", "6443", "-j", "REDIRECT"); err != nil {
+		return fmt.Errorf("failed to add rule to iptables chain: %w", err)
+	}
+
+	return nil
+}
+
 // convertToInstanceMetadata converts a armcomputev2.VirtualMachineScaleSetVM to a metadata.InstanceMetadata.
 func convertToInstanceMetadata(vm armcompute.VirtualMachineScaleSetVM, networkInterfaces []armnetwork.Interface,
 ) (metadata.InstanceMetadata, error) {

internal/config/config.go

@@ -82,6 +82,9 @@ type Config struct {
 	//   A fallback to DNS name is always available.
 	CustomEndpoint string `yaml:"customEndpoint" validate:"omitempty,hostname_rfc1123"`
 	// description: |
+	//   Flag to enable/disable the internal load balancer. If enabled, the Constellation is only accessible from within the VPC.
+	InternalLoadBalancer bool `yaml:"internalLoadBalancer" validate:"omitempty"`
+	// description: |
 	//   Supported cloud providers and their specific configurations.
 	Provider ProviderConfig `yaml:"provider" validate:"dive"`
 	// description: |
@@ -830,6 +833,12 @@ func (c *Config) Validate(force bool) error {
 		}
 	}
 
+	if c.InternalLoadBalancer {
+		if c.GetProvider() != cloudprovider.AWS && c.GetProvider() != cloudprovider.GCP {
+			return &ValidationError{validationErrMsgs: []string{"internalLoadBalancer is only supported for AWS and GCP"}}
+		}
+	}
+
 	err := validate.Struct(c)
 	if err == nil {
 		return nil

internal/config/config_doc.go

@@ -35,7 +35,7 @@ func init() {
 	ConfigDoc.Type = "Config"
 	ConfigDoc.Comments[encoder.LineComment] = "Config defines configuration used by CLI."
 	ConfigDoc.Description = "Config defines configuration used by CLI."
-	ConfigDoc.Fields = make([]encoder.Doc, 10)
+	ConfigDoc.Fields = make([]encoder.Doc, 11)
 	ConfigDoc.Fields[0].Name = "version"
 	ConfigDoc.Fields[0].Type = "string"
 	ConfigDoc.Fields[0].Note = ""
@@ -71,21 +71,26 @@ func init() {
 	ConfigDoc.Fields[6].Note = ""
 	ConfigDoc.Fields[6].Description = "Optional custom endpoint (DNS name) for the Constellation API server.\nThis can be used to point a custom dns name at the Constellation API server\nand is added to the Subject Alternative Name (SAN) field of the TLS certificate used by the API server.\nA fallback to DNS name is always available."
 	ConfigDoc.Fields[6].Comments[encoder.LineComment] = "Optional custom endpoint (DNS name) for the Constellation API server."
-	ConfigDoc.Fields[7].Name = "provider"
-	ConfigDoc.Fields[7].Type = "ProviderConfig"
+	ConfigDoc.Fields[7].Name = "internalLoadBalancer"
+	ConfigDoc.Fields[7].Type = "bool"
 	ConfigDoc.Fields[7].Note = ""
-	ConfigDoc.Fields[7].Description = "Supported cloud providers and their specific configurations."
-	ConfigDoc.Fields[7].Comments[encoder.LineComment] = "Supported cloud providers and their specific configurations."
-	ConfigDoc.Fields[8].Name = "nodeGroups"
-	ConfigDoc.Fields[8].Type = "map[string]NodeGroup"
+	ConfigDoc.Fields[7].Description = "Flag to enable/disable the internal load balancer. If enabled, the Constellation is only accessible from within the VPC."
+	ConfigDoc.Fields[7].Comments[encoder.LineComment] = "Flag to enable/disable the internal load balancer. If enabled, the Constellation is only accessible from within the VPC."
+	ConfigDoc.Fields[8].Name = "provider"
+	ConfigDoc.Fields[8].Type = "ProviderConfig"
 	ConfigDoc.Fields[8].Note = ""
-	ConfigDoc.Fields[8].Description = "Node groups to be created in the cluster."
-	ConfigDoc.Fields[8].Comments[encoder.LineComment] = "Node groups to be created in the cluster."
-	ConfigDoc.Fields[9].Name = "attestation"
-	ConfigDoc.Fields[9].Type = "AttestationConfig"
+	ConfigDoc.Fields[8].Description = "Supported cloud providers and their specific configurations."
+	ConfigDoc.Fields[8].Comments[encoder.LineComment] = "Supported cloud providers and their specific configurations."
+	ConfigDoc.Fields[9].Name = "nodeGroups"
+	ConfigDoc.Fields[9].Type = "map[string]NodeGroup"
 	ConfigDoc.Fields[9].Note = ""
-	ConfigDoc.Fields[9].Description = "Configuration for attestation validation. This configuration provides sensible defaults for the Constellation version it was created for.\nSee the docs for an overview on attestation: https://docs.edgeless.systems/constellation/architecture/attestation"
-	ConfigDoc.Fields[9].Comments[encoder.LineComment] = "Configuration for attestation validation. This configuration provides sensible defaults for the Constellation version it was created for.\nSee the docs for an overview on attestation: https://docs.edgeless.systems/constellation/architecture/attestation"
+	ConfigDoc.Fields[9].Description = "Node groups to be created in the cluster."
+	ConfigDoc.Fields[9].Comments[encoder.LineComment] = "Node groups to be created in the cluster."
+	ConfigDoc.Fields[10].Name = "attestation"
+	ConfigDoc.Fields[10].Type = "AttestationConfig"
+	ConfigDoc.Fields[10].Note = ""
+	ConfigDoc.Fields[10].Description = "Configuration for attestation validation. This configuration provides sensible defaults for the Constellation version it was created for.\nSee the docs for an overview on attestation: https://docs.edgeless.systems/constellation/architecture/attestation"
+	ConfigDoc.Fields[10].Comments[encoder.LineComment] = "Configuration for attestation validation. This configuration provides sensible defaults for the Constellation version it was created for.\nSee the docs for an overview on attestation: https://docs.edgeless.systems/constellation/architecture/attestation"
 
 	ProviderConfigDoc.Type = "ProviderConfig"
 	ProviderConfigDoc.Comments[encoder.LineComment] = "ProviderConfig are cloud-provider specific configuration values used by the CLI."