diff --git a/.github/actions/e2e_lb/action.yml b/.github/actions/e2e_lb/action.yml index 1fd20f412..1a776e426 100644 --- a/.github/actions/e2e_lb/action.yml +++ b/.github/actions/e2e_lb/action.yml @@ -5,6 +5,9 @@ inputs: kubeconfig: description: "The kubeconfig of the cluster to test." required: true + cloudProvider: + description: "The CSP this test runs on. Some tests exercise functionality not supported everywhere." + required: false runs: using: "composite" @@ -20,6 +23,24 @@ runs: kubectl apply -f lb.yml bazel run //e2e/internal/lb:lb_test + - name: Test AWS Ingress + if: inputs.cloudProvider == 'aws' + shell: bash + env: + KUBECONFIG: ${{ inputs.kubeconfig }} + working-directory: ./.github/actions/e2e_lb + run: | + kubectl apply -f aws-ingress.yml + kubectl wait -n lb-test ing/whoami --for=jsonpath='{.status.loadBalancer.ingress}' --timeout=5m + host=$(kubectl get -n lb-test ingress whoami -o=jsonpath='{.status.loadBalancer.ingress[0].hostname}') + for i in $(seq 30); do + curl --silent --fail --connect-timeout 5 --output /dev/null http://$host && exit 0 + sleep 10 + done + echo "::error::Ingress did not become ready in the alloted time." + kubectl describe ing -n lb-test + exit 1 + - name: Delete deployment if: always() shell: bash @@ -28,4 +49,5 @@ runs: working-directory: ./.github/actions/e2e_lb run: | kubectl delete -f lb.yml + kubectl delete --ignore-not-found -f aws-ingress.yml kubectl delete -f ns.yml --timeout=5m diff --git a/.github/actions/e2e_lb/aws-ingress.yml b/.github/actions/e2e_lb/aws-ingress.yml new file mode 100644 index 000000000..641ecffc0 --- /dev/null +++ b/.github/actions/e2e_lb/aws-ingress.yml @@ -0,0 +1,35 @@ +apiVersion: v1 +kind: Service +metadata: + name: whoami-internal + namespace: lb-test +spec: + selector: + app: whoami + ports: + - port: 80 + targetPort: 80 + type: NodePort + +--- + +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + namespace: lb-test + name: whoami + annotations: + alb.ingress.kubernetes.io/scheme: internet-facing + alb.ingress.kubernetes.io/target-type: instance +spec: + ingressClassName: alb + rules: + - http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: whoami-internal + port: + number: 80 \ No newline at end of file diff --git a/.github/actions/e2e_test/action.yml b/.github/actions/e2e_test/action.yml index 36d352b07..36f6d3338 100644 --- a/.github/actions/e2e_test/action.yml +++ b/.github/actions/e2e_test/action.yml @@ -365,6 +365,7 @@ runs: uses: ./.github/actions/e2e_lb with: kubeconfig: ${{ steps.constellation-create.outputs.kubeconfig }} + cloudProvider: ${{ inputs.cloudProvider }} - name: Run Performance Benchmark if: inputs.test == 'perf-bench' diff --git a/docs/docs/workflows/lb.md b/docs/docs/workflows/lb.md index 11e403237..868e61076 100644 --- a/docs/docs/workflows/lb.md +++ b/docs/docs/workflows/lb.md @@ -4,12 +4,25 @@ Constellation integrates the native load balancers of each CSP. Therefore, to ex ## Internet-facing LB service on AWS -To expose your application service externally you might want to use a Kubernetes Service of type `LoadBalancer`. On AWS, load-balancing is achieved through the [AWS Load Balancing Controller](https://kubernetes-sigs.github.io/aws-load-balancer-controller) as in the managed EKS. +To expose your application service externally you might want to use a Kubernetes Service of type `LoadBalancer`. On AWS, load-balancing is achieved through the [AWS Load Balancer Controller](https://kubernetes-sigs.github.io/aws-load-balancer-controller) as in the managed EKS. -Since recent versions, the controller deploy an internal LB by default requiring to set an annotation `service.beta.kubernetes.io/aws-load-balancer-scheme: internet-facing` to have an internet-facing LB. For more details, see the [official docs](https://kubernetes-sigs.github.io/aws-load-balancer-controller/v2.2/guide/service/nlb/). +Since recent versions, the controller deploy an internal LB by default requiring to set an annotation `service.beta.kubernetes.io/aws-load-balancer-scheme: internet-facing` to have an internet-facing LB. For more details, see the [official docs](https://kubernetes-sigs.github.io/aws-load-balancer-controller/v2.7/guide/service/nlb/). For general information on LB with AWS see [Network load balancing on Amazon EKS](https://docs.aws.amazon.com/eks/latest/userguide/network-load-balancing.html). :::caution Before terminating the cluster, all LB backed services should be deleted, so that the controller can cleanup the related resources. ::: + +## Ingress on AWS + +The AWS Load Balancer Controller also provisions `Ingress` resources of class `alb`. +AWS Application Load Balancers (ALBs) can be configured with a [`target-type`](https://kubernetes-sigs.github.io/aws-load-balancer-controller/v2.7/guide/ingress/annotations/#target-type). +The target type `ip` requires using the EKS container network solution, which makes it incompatible with Constellation. +If a service can be exposed on a `NodePort`, the target type `instance` can be used. + +See [Application load balancing on Amazon EKS](https://docs.aws.amazon.com/eks/latest/userguide/alb-ingress.html) for more information. + +:::caution +Ingress handlers backed by AWS ALBs reside outside the Constellation cluster, so they shouldn't be handling sensitive traffic! +::: diff --git a/terraform/BUILD.bazel b/terraform/BUILD.bazel index f4f99f242..88e71216d 100644 --- a/terraform/BUILD.bazel +++ b/terraform/BUILD.bazel @@ -77,6 +77,7 @@ go_library( "infrastructure/aws/modules/public_private_subnet/output.tf", "infrastructure/openstack/modules/stackit_loadbalancer/main.tf", "infrastructure/openstack/modules/stackit_loadbalancer/variables.tf", + "infrastructure/iam/aws/alb_policy.json", ], importpath = "github.com/edgelesssys/constellation/v2/terraform", visibility = ["//visibility:public"], diff --git a/terraform/infrastructure/iam/aws/alb_policy.json b/terraform/infrastructure/iam/aws/alb_policy.json new file mode 100644 index 000000000..e8a05f8e6 --- /dev/null +++ b/terraform/infrastructure/iam/aws/alb_policy.json @@ -0,0 +1,242 @@ +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "iam:CreateServiceLinkedRole" + ], + "Resource": "*", + "Condition": { + "StringEquals": { + "iam:AWSServiceName": "elasticloadbalancing.amazonaws.com" + } + } + }, + { + "Effect": "Allow", + "Action": [ + "ec2:DescribeAccountAttributes", + "ec2:DescribeAddresses", + "ec2:DescribeAvailabilityZones", + "ec2:DescribeInternetGateways", + "ec2:DescribeVpcs", + "ec2:DescribeVpcPeeringConnections", + "ec2:DescribeSubnets", + "ec2:DescribeSecurityGroups", + "ec2:DescribeInstances", + "ec2:DescribeNetworkInterfaces", + "ec2:DescribeTags", + "ec2:GetCoipPoolUsage", + "ec2:DescribeCoipPools", + "elasticloadbalancing:DescribeLoadBalancers", + "elasticloadbalancing:DescribeLoadBalancerAttributes", + "elasticloadbalancing:DescribeListeners", + "elasticloadbalancing:DescribeListenerCertificates", + "elasticloadbalancing:DescribeSSLPolicies", + "elasticloadbalancing:DescribeRules", + "elasticloadbalancing:DescribeTargetGroups", + "elasticloadbalancing:DescribeTargetGroupAttributes", + "elasticloadbalancing:DescribeTargetHealth", + "elasticloadbalancing:DescribeTags", + "elasticloadbalancing:DescribeTrustStores" + ], + "Resource": "*" + }, + { + "Effect": "Allow", + "Action": [ + "cognito-idp:DescribeUserPoolClient", + "acm:ListCertificates", + "acm:DescribeCertificate", + "iam:ListServerCertificates", + "iam:GetServerCertificate", + "waf-regional:GetWebACL", + "waf-regional:GetWebACLForResource", + "waf-regional:AssociateWebACL", + "waf-regional:DisassociateWebACL", + "wafv2:GetWebACL", + "wafv2:GetWebACLForResource", + "wafv2:AssociateWebACL", + "wafv2:DisassociateWebACL", + "shield:GetSubscriptionState", + "shield:DescribeProtection", + "shield:CreateProtection", + "shield:DeleteProtection" + ], + "Resource": "*" + }, + { + "Effect": "Allow", + "Action": [ + "ec2:AuthorizeSecurityGroupIngress", + "ec2:RevokeSecurityGroupIngress" + ], + "Resource": "*" + }, + { + "Effect": "Allow", + "Action": [ + "ec2:CreateSecurityGroup" + ], + "Resource": "*" + }, + { + "Effect": "Allow", + "Action": [ + "ec2:CreateTags" + ], + "Resource": "arn:aws:ec2:*:*:security-group/*", + "Condition": { + "StringEquals": { + "ec2:CreateAction": "CreateSecurityGroup" + }, + "Null": { + "aws:RequestTag/elbv2.k8s.aws/cluster": "false" + } + } + }, + { + "Effect": "Allow", + "Action": [ + "ec2:CreateTags", + "ec2:DeleteTags" + ], + "Resource": "arn:aws:ec2:*:*:security-group/*", + "Condition": { + "Null": { + "aws:RequestTag/elbv2.k8s.aws/cluster": "true", + "aws:ResourceTag/elbv2.k8s.aws/cluster": "false" + } + } + }, + { + "Effect": "Allow", + "Action": [ + "ec2:AuthorizeSecurityGroupIngress", + "ec2:RevokeSecurityGroupIngress", + "ec2:DeleteSecurityGroup" + ], + "Resource": "*", + "Condition": { + "Null": { + "aws:ResourceTag/elbv2.k8s.aws/cluster": "false" + } + } + }, + { + "Effect": "Allow", + "Action": [ + "elasticloadbalancing:CreateLoadBalancer", + "elasticloadbalancing:CreateTargetGroup" + ], + "Resource": "*", + "Condition": { + "Null": { + "aws:RequestTag/elbv2.k8s.aws/cluster": "false" + } + } + }, + { + "Effect": "Allow", + "Action": [ + "elasticloadbalancing:CreateListener", + "elasticloadbalancing:DeleteListener", + "elasticloadbalancing:CreateRule", + "elasticloadbalancing:DeleteRule" + ], + "Resource": "*" + }, + { + "Effect": "Allow", + "Action": [ + "elasticloadbalancing:AddTags", + "elasticloadbalancing:RemoveTags" + ], + "Resource": [ + "arn:aws:elasticloadbalancing:*:*:targetgroup/*/*", + "arn:aws:elasticloadbalancing:*:*:loadbalancer/net/*/*", + "arn:aws:elasticloadbalancing:*:*:loadbalancer/app/*/*" + ], + "Condition": { + "Null": { + "aws:RequestTag/elbv2.k8s.aws/cluster": "true", + "aws:ResourceTag/elbv2.k8s.aws/cluster": "false" + } + } + }, + { + "Effect": "Allow", + "Action": [ + "elasticloadbalancing:AddTags", + "elasticloadbalancing:RemoveTags" + ], + "Resource": [ + "arn:aws:elasticloadbalancing:*:*:listener/net/*/*/*", + "arn:aws:elasticloadbalancing:*:*:listener/app/*/*/*", + "arn:aws:elasticloadbalancing:*:*:listener-rule/net/*/*/*", + "arn:aws:elasticloadbalancing:*:*:listener-rule/app/*/*/*" + ] + }, + { + "Effect": "Allow", + "Action": [ + "elasticloadbalancing:ModifyLoadBalancerAttributes", + "elasticloadbalancing:SetIpAddressType", + "elasticloadbalancing:SetSecurityGroups", + "elasticloadbalancing:SetSubnets", + "elasticloadbalancing:DeleteLoadBalancer", + "elasticloadbalancing:ModifyTargetGroup", + "elasticloadbalancing:ModifyTargetGroupAttributes", + "elasticloadbalancing:DeleteTargetGroup" + ], + "Resource": "*", + "Condition": { + "Null": { + "aws:ResourceTag/elbv2.k8s.aws/cluster": "false" + } + } + }, + { + "Effect": "Allow", + "Action": [ + "elasticloadbalancing:AddTags" + ], + "Resource": [ + "arn:aws:elasticloadbalancing:*:*:targetgroup/*/*", + "arn:aws:elasticloadbalancing:*:*:loadbalancer/net/*/*", + "arn:aws:elasticloadbalancing:*:*:loadbalancer/app/*/*" + ], + "Condition": { + "StringEquals": { + "elasticloadbalancing:CreateAction": [ + "CreateTargetGroup", + "CreateLoadBalancer" + ] + }, + "Null": { + "aws:RequestTag/elbv2.k8s.aws/cluster": "false" + } + } + }, + { + "Effect": "Allow", + "Action": [ + "elasticloadbalancing:RegisterTargets", + "elasticloadbalancing:DeregisterTargets" + ], + "Resource": "arn:aws:elasticloadbalancing:*:*:targetgroup/*/*" + }, + { + "Effect": "Allow", + "Action": [ + "elasticloadbalancing:SetWebAcl", + "elasticloadbalancing:ModifyListener", + "elasticloadbalancing:AddListenerCertificates", + "elasticloadbalancing:RemoveListenerCertificates", + "elasticloadbalancing:ModifyRule" + ], + "Resource": "*" + } + ] +} diff --git a/terraform/infrastructure/iam/aws/main.tf b/terraform/infrastructure/iam/aws/main.tf index 2394841b3..c2dbf7c20 100644 --- a/terraform/infrastructure/iam/aws/main.tf +++ b/terraform/infrastructure/iam/aws/main.tf @@ -242,3 +242,20 @@ resource "aws_iam_role_policy_attachment" "csi_driver_policy_control_plane" { role = aws_iam_role.control_plane_role.name policy_arn = "arn:aws:iam::aws:policy/service-role/AmazonEBSCSIDriverPolicy" } + +// This policy is required by the AWS load balancer controller and can be found at +// https://github.com/kubernetes-sigs/aws-load-balancer-controller/blob/b44633a/docs/install/iam_policy.json. +resource "aws_iam_policy" "lb_policy" { + name = "${var.name_prefix}_lb_policy" + policy = file("${path.module}/alb_policy.json") +} + +resource "aws_iam_role_policy_attachment" "attach_lb_policy_worker" { + role = aws_iam_role.worker_node_role.name + policy_arn = aws_iam_policy.lb_policy.arn +} + +resource "aws_iam_role_policy_attachment" "attach_lb_policy_control_plane" { + role = aws_iam_role.control_plane_role.name + policy_arn = aws_iam_policy.lb_policy.arn +}