terraform: add missing policies for AWS ALB (#3063)

* terraform: add missing policies for AWS ALB

.github/actions/e2e_lb/action.yml

@@ -5,6 +5,9 @@ inputs:
   kubeconfig:
     description: "The kubeconfig of the cluster to test."
     required: true
+  cloudProvider:
+    description: "The CSP this test runs on. Some tests exercise functionality not supported everywhere."
+    required: false
 
 runs:
   using: "composite"
@@ -20,6 +23,24 @@ runs:
         kubectl apply -f lb.yml
         bazel run //e2e/internal/lb:lb_test
 
+    - name: Test AWS Ingress
+      if: inputs.cloudProvider == 'aws'
+      shell: bash
+      env:
+        KUBECONFIG: ${{ inputs.kubeconfig }}
+      working-directory: ./.github/actions/e2e_lb
+      run: |
+        kubectl apply -f aws-ingress.yml
+        kubectl wait -n lb-test ing/whoami --for=jsonpath='{.status.loadBalancer.ingress}' --timeout=5m
+        host=$(kubectl get -n lb-test ingress whoami -o=jsonpath='{.status.loadBalancer.ingress[0].hostname}')
+        for i in $(seq 30); do
+          curl --silent --fail --connect-timeout 5 --output /dev/null http://$host && exit 0
+          sleep 10
+        done
+        echo "::error::Ingress did not become ready in the alloted time."
+        kubectl describe ing -n lb-test
+        exit 1
+
     - name: Delete deployment
       if: always()
       shell: bash
@@ -28,4 +49,5 @@ runs:
       working-directory: ./.github/actions/e2e_lb
       run: |
         kubectl delete -f lb.yml
+        kubectl delete --ignore-not-found -f aws-ingress.yml
         kubectl delete -f ns.yml --timeout=5m

.github/actions/e2e_lb/aws-ingress.yml

@@ -0,0 +1,35 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: whoami-internal
+  namespace: lb-test
+spec:
+  selector:
+    app: whoami
+  ports:
+    - port: 80
+      targetPort: 80
+  type: NodePort
+
+---
+
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  namespace: lb-test
+  name: whoami
+  annotations:
+    alb.ingress.kubernetes.io/scheme: internet-facing
+    alb.ingress.kubernetes.io/target-type: instance
+spec:
+  ingressClassName: alb
+  rules:
+    - http:
+        paths:
+        - path: /
+          pathType: Prefix
+          backend:
+            service:
+              name: whoami-internal
+              port:
+                number: 80

.github/actions/e2e_test/action.yml

@@ -364,6 +364,7 @@ runs:
       uses: ./.github/actions/e2e_lb
       with:
         kubeconfig: ${{ steps.constellation-create.outputs.kubeconfig }}
+        cloudProvider: ${{ inputs.cloudProvider }}
 
     - name: Run Performance Benchmark
       if: inputs.test == 'perf-bench'

docs/docs/workflows/lb.md

@@ -4,12 +4,25 @@ Constellation integrates the native load balancers of each CSP. Therefore, to ex
 
 ## Internet-facing LB service on AWS
 
-To expose your application service externally you might want to use a Kubernetes Service of type `LoadBalancer`. On AWS, load-balancing is achieved through the [AWS Load Balancing Controller](https://kubernetes-sigs.github.io/aws-load-balancer-controller) as in the managed EKS.
+To expose your application service externally you might want to use a Kubernetes Service of type `LoadBalancer`. On AWS, load-balancing is achieved through the [AWS Load Balancer Controller](https://kubernetes-sigs.github.io/aws-load-balancer-controller) as in the managed EKS.
 
-Since recent versions, the controller deploy an internal LB by default requiring to set an annotation `service.beta.kubernetes.io/aws-load-balancer-scheme: internet-facing` to have an internet-facing LB. For more details, see the [official docs](https://kubernetes-sigs.github.io/aws-load-balancer-controller/v2.2/guide/service/nlb/).
+Since recent versions, the controller deploy an internal LB by default requiring to set an annotation `service.beta.kubernetes.io/aws-load-balancer-scheme: internet-facing` to have an internet-facing LB. For more details, see the [official docs](https://kubernetes-sigs.github.io/aws-load-balancer-controller/v2.7/guide/service/nlb/).
 
 For general information on LB with AWS see [Network load balancing on Amazon EKS](https://docs.aws.amazon.com/eks/latest/userguide/network-load-balancing.html).
 
 :::caution
 Before terminating the cluster, all LB backed services should be deleted, so that the controller can cleanup the related resources.
 :::
+
+## Ingress on AWS
+
+The AWS Load Balancer Controller also provisions `Ingress` resources of class `alb`.
+AWS Application Load Balancers (ALBs) can be configured with a [`target-type`](https://kubernetes-sigs.github.io/aws-load-balancer-controller/v2.7/guide/ingress/annotations/#target-type).
+The target type `ip` requires using the EKS container network solution, which makes it incompatible with Constellation.
+If a service can be exposed on a `NodePort`, the target type `instance` can be used.
+
+See [Application load balancing on Amazon EKS](https://docs.aws.amazon.com/eks/latest/userguide/alb-ingress.html) for more information.
+
+:::caution
+Ingress handlers backed by AWS ALBs reside outside the Constellation cluster, so they shouldn't be handling sensitive traffic!
+:::

terraform/BUILD.bazel

@@ -77,6 +77,7 @@ go_library(
         "infrastructure/aws/modules/public_private_subnet/output.tf",
         "infrastructure/openstack/modules/stackit_loadbalancer/main.tf",
         "infrastructure/openstack/modules/stackit_loadbalancer/variables.tf",
+        "infrastructure/iam/aws/alb_policy.json",
     ],
     importpath = "github.com/edgelesssys/constellation/v2/terraform",
     visibility = ["//visibility:public"],

terraform/infrastructure/iam/aws/alb_policy.json

@@ -0,0 +1,242 @@
+{
+    "Version": "2012-10-17",
+    "Statement": [
+        {
+            "Effect": "Allow",
+            "Action": [
+                "iam:CreateServiceLinkedRole"
+            ],
+            "Resource": "*",
+            "Condition": {
+                "StringEquals": {
+                    "iam:AWSServiceName": "elasticloadbalancing.amazonaws.com"
+                }
+            }
+        },
+        {
+            "Effect": "Allow",
+            "Action": [
+                "ec2:DescribeAccountAttributes",
+                "ec2:DescribeAddresses",
+                "ec2:DescribeAvailabilityZones",
+                "ec2:DescribeInternetGateways",
+                "ec2:DescribeVpcs",
+                "ec2:DescribeVpcPeeringConnections",
+                "ec2:DescribeSubnets",
+                "ec2:DescribeSecurityGroups",
+                "ec2:DescribeInstances",
+                "ec2:DescribeNetworkInterfaces",
+                "ec2:DescribeTags",
+                "ec2:GetCoipPoolUsage",
+                "ec2:DescribeCoipPools",
+                "elasticloadbalancing:DescribeLoadBalancers",
+                "elasticloadbalancing:DescribeLoadBalancerAttributes",
+                "elasticloadbalancing:DescribeListeners",
+                "elasticloadbalancing:DescribeListenerCertificates",
+                "elasticloadbalancing:DescribeSSLPolicies",
+                "elasticloadbalancing:DescribeRules",
+                "elasticloadbalancing:DescribeTargetGroups",
+                "elasticloadbalancing:DescribeTargetGroupAttributes",
+                "elasticloadbalancing:DescribeTargetHealth",
+                "elasticloadbalancing:DescribeTags",
+                "elasticloadbalancing:DescribeTrustStores"
+            ],
+            "Resource": "*"
+        },
+        {
+            "Effect": "Allow",
+            "Action": [
+                "cognito-idp:DescribeUserPoolClient",
+                "acm:ListCertificates",
+                "acm:DescribeCertificate",
+                "iam:ListServerCertificates",
+                "iam:GetServerCertificate",
+                "waf-regional:GetWebACL",
+                "waf-regional:GetWebACLForResource",
+                "waf-regional:AssociateWebACL",
+                "waf-regional:DisassociateWebACL",
+                "wafv2:GetWebACL",
+                "wafv2:GetWebACLForResource",
+                "wafv2:AssociateWebACL",
+                "wafv2:DisassociateWebACL",
+                "shield:GetSubscriptionState",
+                "shield:DescribeProtection",
+                "shield:CreateProtection",
+                "shield:DeleteProtection"
+            ],
+            "Resource": "*"
+        },
+        {
+            "Effect": "Allow",
+            "Action": [
+                "ec2:AuthorizeSecurityGroupIngress",
+                "ec2:RevokeSecurityGroupIngress"
+            ],
+            "Resource": "*"
+        },
+        {
+            "Effect": "Allow",
+            "Action": [
+                "ec2:CreateSecurityGroup"
+            ],
+            "Resource": "*"
+        },
+        {
+            "Effect": "Allow",
+            "Action": [
+                "ec2:CreateTags"
+            ],
+            "Resource": "arn:aws:ec2:*:*:security-group/*",
+            "Condition": {
+                "StringEquals": {
+                    "ec2:CreateAction": "CreateSecurityGroup"
+                },
+                "Null": {
+                    "aws:RequestTag/elbv2.k8s.aws/cluster": "false"
+                }
+            }
+        },
+        {
+            "Effect": "Allow",
+            "Action": [
+                "ec2:CreateTags",
+                "ec2:DeleteTags"
+            ],
+            "Resource": "arn:aws:ec2:*:*:security-group/*",
+            "Condition": {
+                "Null": {
+                    "aws:RequestTag/elbv2.k8s.aws/cluster": "true",
+                    "aws:ResourceTag/elbv2.k8s.aws/cluster": "false"
+                }
+            }
+        },
+        {
+            "Effect": "Allow",
+            "Action": [
+                "ec2:AuthorizeSecurityGroupIngress",
+                "ec2:RevokeSecurityGroupIngress",
+                "ec2:DeleteSecurityGroup"
+            ],
+            "Resource": "*",
+            "Condition": {
+                "Null": {
+                    "aws:ResourceTag/elbv2.k8s.aws/cluster": "false"
+                }
+            }
+        },
+        {
+            "Effect": "Allow",
+            "Action": [
+                "elasticloadbalancing:CreateLoadBalancer",
+                "elasticloadbalancing:CreateTargetGroup"
+            ],
+            "Resource": "*",
+            "Condition": {
+                "Null": {
+                    "aws:RequestTag/elbv2.k8s.aws/cluster": "false"
+                }
+            }
+        },
+        {
+            "Effect": "Allow",
+            "Action": [
+                "elasticloadbalancing:CreateListener",
+                "elasticloadbalancing:DeleteListener",
+                "elasticloadbalancing:CreateRule",
+                "elasticloadbalancing:DeleteRule"
+            ],
+            "Resource": "*"
+        },
+        {
+            "Effect": "Allow",
+            "Action": [
+                "elasticloadbalancing:AddTags",
+                "elasticloadbalancing:RemoveTags"
+            ],
+            "Resource": [
+                "arn:aws:elasticloadbalancing:*:*:targetgroup/*/*",
+                "arn:aws:elasticloadbalancing:*:*:loadbalancer/net/*/*",
+                "arn:aws:elasticloadbalancing:*:*:loadbalancer/app/*/*"
+            ],
+            "Condition": {
+                "Null": {
+                    "aws:RequestTag/elbv2.k8s.aws/cluster": "true",
+                    "aws:ResourceTag/elbv2.k8s.aws/cluster": "false"
+                }
+            }
+        },
+        {
+            "Effect": "Allow",
+            "Action": [
+                "elasticloadbalancing:AddTags",
+                "elasticloadbalancing:RemoveTags"
+            ],
+            "Resource": [
+                "arn:aws:elasticloadbalancing:*:*:listener/net/*/*/*",
+                "arn:aws:elasticloadbalancing:*:*:listener/app/*/*/*",
+                "arn:aws:elasticloadbalancing:*:*:listener-rule/net/*/*/*",
+                "arn:aws:elasticloadbalancing:*:*:listener-rule/app/*/*/*"
+            ]
+        },
+        {
+            "Effect": "Allow",
+            "Action": [
+                "elasticloadbalancing:ModifyLoadBalancerAttributes",
+                "elasticloadbalancing:SetIpAddressType",
+                "elasticloadbalancing:SetSecurityGroups",
+                "elasticloadbalancing:SetSubnets",
+                "elasticloadbalancing:DeleteLoadBalancer",
+                "elasticloadbalancing:ModifyTargetGroup",
+                "elasticloadbalancing:ModifyTargetGroupAttributes",
+                "elasticloadbalancing:DeleteTargetGroup"
+            ],
+            "Resource": "*",
+            "Condition": {
+                "Null": {
+                    "aws:ResourceTag/elbv2.k8s.aws/cluster": "false"
+                }
+            }
+        },
+        {
+            "Effect": "Allow",
+            "Action": [
+                "elasticloadbalancing:AddTags"
+            ],
+            "Resource": [
+                "arn:aws:elasticloadbalancing:*:*:targetgroup/*/*",
+                "arn:aws:elasticloadbalancing:*:*:loadbalancer/net/*/*",
+                "arn:aws:elasticloadbalancing:*:*:loadbalancer/app/*/*"
+            ],
+            "Condition": {
+                "StringEquals": {
+                    "elasticloadbalancing:CreateAction": [
+                        "CreateTargetGroup",
+                        "CreateLoadBalancer"
+                    ]
+                },
+                "Null": {
+                    "aws:RequestTag/elbv2.k8s.aws/cluster": "false"
+                }
+            }
+        },
+        {
+            "Effect": "Allow",
+            "Action": [
+                "elasticloadbalancing:RegisterTargets",
+                "elasticloadbalancing:DeregisterTargets"
+            ],
+            "Resource": "arn:aws:elasticloadbalancing:*:*:targetgroup/*/*"
+        },
+        {
+            "Effect": "Allow",
+            "Action": [
+                "elasticloadbalancing:SetWebAcl",
+                "elasticloadbalancing:ModifyListener",
+                "elasticloadbalancing:AddListenerCertificates",
+                "elasticloadbalancing:RemoveListenerCertificates",
+                "elasticloadbalancing:ModifyRule"
+            ],
+            "Resource": "*"
+        }
+    ]
+}

terraform/infrastructure/iam/aws/main.tf

@@ -242,3 +242,20 @@ resource "aws_iam_role_policy_attachment" "csi_driver_policy_control_plane" {
   role       = aws_iam_role.control_plane_role.name
   policy_arn = "arn:aws:iam::aws:policy/service-role/AmazonEBSCSIDriverPolicy"
 }
+
+// This policy is required by the AWS load balancer controller and can be found at
+// https://github.com/kubernetes-sigs/aws-load-balancer-controller/blob/b44633a/docs/install/iam_policy.json.
+resource "aws_iam_policy" "lb_policy" {
+  name   = "${var.name_prefix}_lb_policy"
+  policy = file("${path.module}/alb_policy.json")
+}
+
+resource "aws_iam_role_policy_attachment" "attach_lb_policy_worker" {
+  role       = aws_iam_role.worker_node_role.name
+  policy_arn = aws_iam_policy.lb_policy.arn
+}
+
+resource "aws_iam_role_policy_attachment" "attach_lb_policy_control_plane" {
+  role       = aws_iam_role.control_plane_role.name
+  policy_arn = aws_iam_policy.lb_policy.arn
+}