move chart outside services

This commit is contained in:
Adrian Stobbe 2023-07-13 18:25:30 +02:00
parent 59d465c4be
commit a172b6f881
30 changed files with 92 additions and 361 deletions

View File

@ -66,6 +66,16 @@ func New(log *logger.Logger) (*Client, error) {
}, nil
}
// InstallAWSLoadBalancerController installs the AWS Load Balancer Controller.
func (h *Client) InstallAWSLoadBalancerController(ctx context.Context, release helm.Release) error {
h.ReleaseName = release.ReleaseName
if err := h.setWaitMode(release.WaitMode); err != nil {
return err
}
return h.install(ctx, release.Chart, release.Values)
}
// InstallConstellationServices installs the constellation-services chart. In the future this chart should bundle all microservices.
func (h *Client) InstallConstellationServices(ctx context.Context, release helm.Release, extraVals map[string]any) error {
h.ReleaseName = release.ReleaseName

View File

@ -34,4 +34,5 @@ type helmClient interface {
InstallCertManager(ctx context.Context, release helm.Release) error
InstallOperators(ctx context.Context, release helm.Release, extraVals map[string]any) error
InstallConstellationServices(ctx context.Context, release helm.Release, extraVals map[string]any) error
InstallAWSLoadBalancerController(ctx context.Context, release helm.Release) error
}

View File

@ -242,6 +242,13 @@ func (k *KubeWrapper) InitCluster(
return nil, fmt.Errorf("installing cert-manager: %w", err)
}
log.Infof("Installing AWS Load Balancer Controller")
if helmReleases.AWSLoadBalancerController.ReleaseName != "" {
if err = k.helmClient.InstallAWSLoadBalancerController(ctx, helmReleases.AWSLoadBalancerController); err != nil {
return nil, fmt.Errorf("installing AWS Load Balancer Controller: %w", err)
}
}
operatorVals, err := k.setupOperatorVals(ctx)
if err != nil {
return nil, fmt.Errorf("setting up operator vals: %w", err)

View File

@ -592,6 +592,10 @@ func (s *stubHelmClient) InstallCilium(_ context.Context, _ k8sapi.Client, _ hel
return s.ciliumError
}
func (s *stubHelmClient) InstallAWSLoadBalancerController(_ context.Context, _ helm.Release) error {
return s.ciliumError
}
func (s *stubHelmClient) InstallCertManager(_ context.Context, _ helm.Release) error {
return s.certManagerError
}

View File

@ -31,5 +31,5 @@ type File struct {
// GetClusterName returns the name of the cluster.
func GetClusterName(cfgName string, idFile File) string {
return cfgName + idFile.UID
return cfgName + "-" + idFile.UID
}

View File

@ -375,25 +375,27 @@ go_library(
"charts/edgeless/constellation-services/charts/aws-csi-driver/templates/storageclass_integrity.yaml",
"charts/edgeless/constellation-services/charts/aws-csi-driver/templates/volumesnapshotclass.yaml",
"charts/edgeless/constellation-services/charts/aws-csi-driver/values.yaml",
"charts/edgeless/constellation-services/charts/aws-load-balancer-controller/Chart.yaml",
"charts/edgeless/constellation-services/charts/aws-load-balancer-controller/README.md",
"charts/edgeless/constellation-services/charts/aws-load-balancer-controller/ci/extra_args",
"charts/edgeless/constellation-services/charts/aws-load-balancer-controller/ci/values.yaml",
"charts/edgeless/constellation-services/charts/aws-load-balancer-controller/crds/crds.yaml",
"charts/edgeless/constellation-services/charts/aws-load-balancer-controller/crds/kustomization.yaml",
"charts/edgeless/constellation-services/charts/aws-load-balancer-controller/templates/NOTES.txt",
"charts/edgeless/constellation-services/charts/aws-load-balancer-controller/templates/_helpers.tpl",
"charts/edgeless/constellation-services/charts/aws-load-balancer-controller/templates/deployment.yaml",
"charts/edgeless/constellation-services/charts/aws-load-balancer-controller/templates/ingressclass.yaml",
"charts/edgeless/constellation-services/charts/aws-load-balancer-controller/templates/pdb.yaml",
"charts/edgeless/constellation-services/charts/aws-load-balancer-controller/templates/rbac.yaml",
"charts/edgeless/constellation-services/charts/aws-load-balancer-controller/templates/service.yaml",
"charts/edgeless/constellation-services/charts/aws-load-balancer-controller/templates/serviceaccount.yaml",
"charts/edgeless/constellation-services/charts/aws-load-balancer-controller/templates/servicemonitor.yaml",
"charts/edgeless/constellation-services/charts/aws-load-balancer-controller/templates/webhook.yaml",
"charts/edgeless/constellation-services/charts/aws-load-balancer-controller/test.yaml",
"charts/edgeless/constellation-services/charts/aws-load-balancer-controller/values.yaml",
"charts/edgeless/constellation-services/charts/aws-load-balancer-controller/.helmignore",
"charts/.DS_Store",
"charts/edgeless/.DS_Store",
"charts/edgeless/constellation-services/.DS_Store",
"charts/edgeless/constellation-services/charts/.DS_Store",
"charts/aws-load-balancer-controller/.helmignore",
"charts/aws-load-balancer-controller/Chart.yaml",
"charts/aws-load-balancer-controller/README.md",
"charts/aws-load-balancer-controller/ci/extra_args",
"charts/aws-load-balancer-controller/ci/values.yaml",
"charts/aws-load-balancer-controller/crds/crds.yaml",
"charts/aws-load-balancer-controller/templates/NOTES.txt",
"charts/aws-load-balancer-controller/templates/_helpers.tpl",
"charts/aws-load-balancer-controller/templates/deployment.yaml",
"charts/aws-load-balancer-controller/templates/ingressclass.yaml",
"charts/aws-load-balancer-controller/templates/pdb.yaml",
"charts/aws-load-balancer-controller/templates/rbac.yaml",
"charts/aws-load-balancer-controller/templates/service.yaml",
"charts/aws-load-balancer-controller/templates/serviceaccount.yaml",
"charts/aws-load-balancer-controller/templates/servicemonitor.yaml",
"charts/aws-load-balancer-controller/templates/webhook.yaml",
"charts/aws-load-balancer-controller/values.yaml",
],
importpath = "github.com/edgelesssys/constellation/v2/cli/internal/helm",
visibility = ["//cli:__subpackages__"],

View File

@ -2,11 +2,11 @@
# This is a YAML-formatted file.
# Declare variables to be passed into your templates.
replicaCount: 2
replicaCount: 1 # TODO from 2
image:
repository: public.ecr.aws/eks/aws-load-balancer-controller
tag: v2.5.3
tag: v2.5.4
pullPolicy: IfNotPresent
imagePullSecrets: []
@ -61,7 +61,9 @@ resources: {}
# ref: https://kubernetes.io/docs/concepts/configuration/pod-priority-preemption/#priorityclass
priorityClassName: system-cluster-critical
nodeSelector: {}
nodeSelector: {
node-role.kubernetes.io/control-plane: ""
}
tolerations: []
@ -215,7 +217,7 @@ targetgroupbindingMaxConcurrentReconciles:
# Maximum duration of exponential backoff for targetGroupBinding reconcile failures
targetgroupbindingMaxExponentialBackoffDelay:
# Period at which the controller forces the repopulation of its local object stores. (default 1h0m0s)
# Period at which the controller forces the repopulation of its local object stores. (default 10h0m0s)
syncPeriod:
# Namespace the controller watches for updates to Kubernetes objects, If empty, all namespaces are watched.

View File

@ -21,3 +21,6 @@
.idea/
*.tmproj
.vscode/
#charts/aws-load-balancer-controller/crds/kustomization.yaml
#charts/aws-load-balancer-controller/test.yaml

View File

@ -1,4 +0,0 @@
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
resources:
- crds.yaml

View File

@ -1,326 +0,0 @@
# Default values for aws-load-balancer-controller.
# This is a YAML-formatted file.
# Declare variables to be passed into your templates.
replicaCount: 2
image:
repository: public.ecr.aws/eks/aws-load-balancer-controller
tag: v2.5.3
pullPolicy: IfNotPresent
imagePullSecrets: []
nameOverride: ""
fullnameOverride: ""
serviceAccount:
# Specifies whether a service account should be created
create: true
# Annotations to add to the service account
annotations: {}
# The name of the service account to use.
# If not set and create is true, a name is generated using the fullname template
name:
# Automount API credentials for a Service Account.
automountServiceAccountToken: true
# List of image pull secrets to add to the Service Account.
imagePullSecrets:
# - name: docker
rbac:
# Specifies whether rbac resources should be created
create: true
podSecurityContext:
fsGroup: 65534
securityContext:
# capabilities:
# drop:
# - ALL
readOnlyRootFilesystem: true
runAsNonRoot: true
allowPrivilegeEscalation: false
# Time period for the controller pod to do a graceful shutdown
terminationGracePeriodSeconds: 10
resources:
limits:
cpu: 100m
memory: 128Mi
requests:
cpu: 100m
memory: 128Mi
# priorityClassName specifies the PriorityClass to indicate the importance of controller pods
# ref: https://kubernetes.io/docs/concepts/configuration/pod-priority-preemption/#priorityclass
priorityClassName: system-cluster-critical
nodeSelector: {}
tolerations: []
# affinity specifies a custom affinity for the controller pods
affinity: {}
# configureDefaultAffinity specifies whether to configure a default affinity for the controller pods to prevent
# co-location on the same node. This will get ignored if you specify a custom affinity configuration.
configureDefaultAffinity: true
# topologySpreadConstraints is a stable feature of k8s v1.19 which provides the ability to
# control how Pods are spread across your cluster among failure-domains such as regions, zones,
# nodes, and other user-defined topology domains.
#
# more details here: https://kubernetes.io/docs/concepts/workloads/pods/pod-topology-spread-constraints/
topologySpreadConstraints: {}
updateStrategy:
type: RollingUpdate
rollingUpdate:
maxSurge: 1
maxUnavailable: 1
# serviceAnnotations contains annotations to be added to the provisioned webhook service resource
serviceAnnotations: {}
# deploymentAnnotations contains annotations for the controller deployment
deploymentAnnotations: {}
podAnnotations: {}
podLabels: {}
# additionalLabels -- Labels to add to each object of the chart.
additionalLabels: {}
# Enable cert-manager
enableCertManager: false
# The name of the Kubernetes cluster. A non-empty value is required
clusterName: test-cluster
# cluster contains configurations specific to the kubernetes cluster
cluster:
# Cluster DNS domain (required for requesting TLS certificates)
dnsDomain: cluster.local
# The ingress class this controller will satisfy. If not specified, controller will match all
# ingresses without ingress class annotation and ingresses of type alb
ingressClass: alb
# ingressClassParams specify the IngressCLassParams that enforce settings for a set of Ingresses when using with ingress Controller.
ingressClassParams:
create: true
# The name of ingressClassParams resource will be referred in ingressClass
name:
spec: {}
# You always can set specifications in `helm install` command through `--set` or `--set-string`
# If you do want to specify specifications in values.yaml, uncomment the following
# lines, adjust them as necessary, and remove the curly braces after 'spec:'.
# namespaceSelector:
# matchLabels:
# group:
# scheme:
# ipAddressType:
# tags:
# To use IngressClass resource instead of annotation, before you need to install the IngressClass resource pointing to controller.
# If specified as true, the IngressClass resource will be created.
createIngressClassResource: true
# The AWS region for the kubernetes cluster. Set to use KIAM or kube2iam for example.
region:
# The VPC ID for the Kubernetes cluster. Set this manually when your pods are unable to use the metadata service to determine this automatically
vpcId:
# Custom AWS API Endpoints (serviceID1=URL1,serviceID2=URL2)
awsApiEndpoints:
# awsApiThrottle specifies custom AWS API throttle settings (serviceID1:operationRegex1=rate:burst,serviceID2:operationRegex2=rate:burst)
# example: --set awsApiThrottle="{Elastic Load Balancing v2:RegisterTargets|DeregisterTargets=4:20,Elastic Load Balancing v2:.*=10:40}"
awsApiThrottle:
# Maximum retries for AWS APIs (default 10)
awsMaxRetries:
# If enabled, targetHealth readiness gate will get injected to the pod spec for the matching endpoint pods (default true)
enablePodReadinessGateInject:
# Enable Shield addon for ALB (default true)
enableShield:
# Enable WAF addon for ALB (default true)
enableWaf:
# Enable WAF V2 addon for ALB (default true)
enableWafv2:
# Maximum number of concurrently running reconcile loops for ingress (default 3)
ingressMaxConcurrentReconciles:
# Set the controller log level - info(default), debug (default "info")
logLevel:
# The address the metric endpoint binds to. (default ":8080")
metricsBindAddr: ""
# The TCP port the Webhook server binds to. (default 9443)
webhookBindPort:
# webhookTLS specifies TLS cert/key for the webhook
webhookTLS:
caCert:
cert:
key:
# array of namespace selectors for the webhook
webhookNamespaceSelectors:
- key: elbv2.k8s.aws/pod-readiness-gate-inject
operator: In
values:
- enabled
# keepTLSSecret specifies whether to reuse existing TLS secret for chart upgrade
keepTLSSecret: true
# Maximum number of concurrently running reconcile loops for service (default 3)
serviceMaxConcurrentReconciles:
# Maximum number of concurrently running reconcile loops for targetGroupBinding
targetgroupbindingMaxConcurrentReconciles:
# Maximum duration of exponential backoff for targetGroupBinding reconcile failures
targetgroupbindingMaxExponentialBackoffDelay:
# Period at which the controller forces the repopulation of its local object stores. (default 1h0m0s)
syncPeriod:
# Namespace the controller watches for updates to Kubernetes objects, If empty, all namespaces are watched.
watchNamespace:
# disableIngressClassAnnotation disables the usage of kubernetes.io/ingress.class annotation, false by default
disableIngressClassAnnotation:
# disableIngressGroupNameAnnotation disables the usage of alb.ingress.kubernetes.io/group.name annotation, false by default
disableIngressGroupNameAnnotation:
# defaultSSLPolicy specifies the default SSL policy to use for TLS/HTTPS listeners
defaultSSLPolicy:
# Liveness probe configuration for the controller
livenessProbe:
failureThreshold: 2
httpGet:
path: /healthz
port: 61779
scheme: HTTP
initialDelaySeconds: 30
timeoutSeconds: 10
# Environment variables to set for aws-load-balancer-controller pod.
# We strongly discourage programming access credentials in the controller environment. You should setup IRSA or
# comparable solutions like kube2iam, kiam etc instead.
env:
# ENV_1: ""
# ENV_2: ""
# Specifies if aws-load-balancer-controller should be started in hostNetwork mode.
#
# This is required if using a custom CNI where the managed control plane nodes are unable to initiate
# network connections to the pods, for example using Calico CNI plugin on EKS. This is not required or
# recommended if using the Amazon VPC CNI plugin.
hostNetwork: false
# Specifies the dnsPolicy that should be used for pods in the deployment
#
# This may need to be used to be changed given certain conditions. For instance, if one uses the cilium CNI
# with certain settings, one may need to set `hostNetwork: true` and webhooks won't work unless `dnsPolicy`
# is set to `ClusterFirstWithHostNet`. See https://kubernetes.io/docs/concepts/services-networking/dns-pod-service/#pod-s-dns-policy
dnsPolicy:
# extraVolumeMounts are the additional volume mounts. This enables setting up IRSA on non-EKS Kubernetes cluster
extraVolumeMounts:
- name: aws-iam-token
mountPath: /var/run/secrets/eks.amazonaws.com/serviceaccount
readOnly: true
# extraVolumes for the extraVolumeMounts. Useful to mount a projected service account token for example.
extraVolumes:
- name: aws-iam-token
projected:
defaultMode: 420
sources:
- serviceAccountToken:
audience: sts.amazonaws.com
expirationSeconds: 86400
path: token
# defaultTags are the tags to apply to all AWS resources managed by this controller
defaultTags:
default_tag1: value1
default_tag2: value2
# podDisruptionBudget specifies the disruption budget for the controller pods.
# Disruption budget will be configured only when the replicaCount is greater than 1
podDisruptionBudget:
maxUnavailable: 1
# externalManagedTags is the list of tag keys on AWS resources that will be managed externally
externalManagedTags: []
# enableEndpointSlices enables k8s EndpointSlices for IP targets instead of Endpoints (default false)
enableEndpointSlices:
# enableBackendSecurityGroup enables shared security group for backend traffic (default true)
enableBackendSecurityGroup:
# backendSecurityGroup specifies backend security group id (default controller auto create backend security group)
backendSecurityGroup:
# disableRestrictedSecurityGroupRules specifies whether to disable creating port-range restricted security group rules for traffic
disableRestrictedSecurityGroupRules:
# controllerConfig specifies controller configuration
controllerConfig:
# featureGates set of key: value pairs that describe AWS load balance controller features
featureGates: {}
# ServiceTypeLoadBalancerOnly: true
# EndpointsFailOpen: true
# objectSelector for webhook
objectSelector:
matchExpressions:
# - key: <key>
# operator: <operator>
# values:
# - <value>
matchLabels:
# key: value
serviceMonitor:
# Specifies whether a service monitor should be created
enabled: false
# Labels to add to the service account
additionalLabels: {}
# Prometheus scrape interval
interval: 1m
# Namespace to create the service monitor in
namespace:
# clusterSecretsPermissions lets you configure RBAC permissions for secret resources
# Access to secrets resource is required only if you use the OIDC feature, and instead of
# enabling access to all secrets, we recommend configuring namespaced role/rolebinding.
# This option is for backwards compatibility only, and will potentially be deprecated in future.
clusterSecretsPermissions:
# allowAllSecrets allows the controller to access all secrets in the cluster.
# This is to get backwards compatible behavior, but *NOT* recommended for security reasons
allowAllSecrets: false
# ingressClassConfig contains configurations specific to the ingress class
ingressClassConfig:
default: false

View File

@ -1,3 +1,6 @@
#aws-load-balancer-controller:
#fullnameOverride: aws-load-balancer-controller
global:
# Port on which the KeyService will listen. Global since join-service also uses the value.
keyServicePort: 9000

View File

@ -51,6 +51,8 @@ var (
certManagerInfo = chartInfo{releaseName: "cert-manager", chartName: "cert-manager", path: "charts/cert-manager"}
constellationOperatorsInfo = chartInfo{releaseName: "constellation-operators", chartName: "constellation-operators", path: "charts/edgeless/operators"}
constellationServicesInfo = chartInfo{releaseName: "constellation-services", chartName: "constellation-services", path: "charts/edgeless/constellation-services"}
awsInfo = chartInfo{releaseName: "aws-load-balancer-controller", chartName: "aws-load-balancer-controller", path: "charts/aws-load-balancer-controller"}
)
// ChartLoader loads embedded helm charts.
@ -129,6 +131,13 @@ func (i *ChartLoader) Load(config *config.Config, conformanceMode bool, helmWait
}
releases := helm.Releases{Cilium: ciliumRelease, CertManager: certManagerRelease, Operators: operatorRelease, ConstellationServices: conServicesRelease}
if config.HasProvider(cloudprovider.AWS) {
awsRelease, err := i.loadRelease(awsInfo, helmWaitMode)
if err != nil {
return nil, fmt.Errorf("loading aws-services: %w", err)
}
releases.AWSLoadBalancerController = awsRelease
}
rel, err := json.Marshal(releases)
if err != nil {
@ -159,9 +168,11 @@ func (i *ChartLoader) loadRelease(info chartInfo, helmWaitMode helm.WaitMode) (h
updateVersions(chart, compatibility.EnsurePrefixV(constants.VersionInfo()))
values, err = i.loadConstellationServicesValues()
case awsInfo.releaseName:
values, err = i.loadAWSLoadBalancerControllerValues()
}
if err != nil {
if err != nil || values == nil {
return helm.Release{}, fmt.Errorf("loading %s values: %w", info.releaseName, err)
}
@ -173,6 +184,20 @@ func (i *ChartLoader) loadRelease(info chartInfo, helmWaitMode helm.WaitMode) (h
return helm.Release{Chart: chartRaw, Values: values, ReleaseName: info.releaseName, WaitMode: helmWaitMode}, nil
}
func (i *ChartLoader) loadAWSLoadBalancerControllerValues() (map[string]any, error) {
valuesFile, err := helmFS.ReadFile(awsInfo.path + "/values.yaml")
if err != nil {
return nil, err
}
values, err := chartutil.ReadValues(valuesFile)
if err != nil {
return nil, err
}
values["clusterName"] = i.clusterName
// TODO add custom settings like nodeSelector here or keep in values.yaml?
return values, nil
}
// loadCiliumValues is used to separate the marshalling step from the loading step.
// This reduces the time unit tests take to execute.
func (i *ChartLoader) loadCiliumValues() (map[string]any, error) {

View File

@ -4,7 +4,7 @@
# script is mostly copied from cli/internal/helm/update-csi-charts.sh
set -euo pipefail
set -o errtrace
set -o errtrac
shopt -s inherit_errexit
echo "Updating AWS Load Balancer Controller Helm chart..."
@ -18,7 +18,7 @@ fi
callDir=$(pwd)
repo_tmp_dir=$(mktemp -d)
chart_base_path="charts/edgeless/constellation-services/charts"
chart_base_path="charts"
chart_name="aws-load-balancer-controller"
chart_url="https://github.com/aws/eks-charts"
@ -35,6 +35,9 @@ git clone \
git sparse-checkout add "${chart_dir}"
git checkout
cd "${callDir}"
rm "${repo_tmp_dir}/${chart_dir}/crds/kustomization.yaml"
rm "${repo_tmp_dir}/${chart_dir}/test.yaml"
rm "${repo_tmp_dir}/${chart_dir}/values.yaml"
# remove old chart
rm -r "${chart_base_path:?}/${chart_name}"

View File

@ -17,10 +17,11 @@ type Release struct {
// Releases bundles all helm releases to be deployed to Constellation.
type Releases struct {
Cilium Release
CertManager Release
Operators Release
ConstellationServices Release
AWSLoadBalancerController Release
Cilium Release
CertManager Release
Operators Release
ConstellationServices Release
}
// MergeMaps returns a new map that is the merger of it's inputs.