diff --git a/infrastructure/ingress-nginx/kustomization.yaml b/infrastructure/ingress-nginx/kustomization.yaml new file mode 100644 index 0000000000000000000000000000000000000000..77c66d9c9fd9b5b4c4ea64ab34ce5cbe25e8df79 --- /dev/null +++ b/infrastructure/ingress-nginx/kustomization.yaml @@ -0,0 +1,7 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: cert-manager +resources: + - namespace.yaml + - repository.yaml + - release.yaml diff --git a/infrastructure/ingress-nginx/namespace.yaml b/infrastructure/ingress-nginx/namespace.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f2e980136694979c0ae1564ef49d750a545de9c7 --- /dev/null +++ b/infrastructure/ingress-nginx/namespace.yaml @@ -0,0 +1,6 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: nginx-system + labels: + name: nginx-system diff --git a/infrastructure/ingress-nginx/release.yaml b/infrastructure/ingress-nginx/release.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e2888ca734fc9cf78a83d155b8e4f4feefb1b94f --- /dev/null +++ b/infrastructure/ingress-nginx/release.yaml @@ -0,0 +1,530 @@ +apiVersion: helm.toolkit.fluxcd.io/v2beta1 +kind: HelmRelease +metadata: + name: nginx-ingress + namespace: nginx-system +spec: + releaseName: nginx-ingress + chart: + spec: + chart: nginx-ingress + sourceRef: + kind: HelmRepository + name: nginx-ingress + version: 4.0.5 + interval: 5m + values: + ## nginx configuration + ## Ref: https://github.com/kubernetes/ingress-nginx/blob/main/docs/user-guide/nginx-configuration/index.md + ## + + ## Overrides for generated resource names + # See templates/_helpers.tpl + # nameOverride: + # fullnameOverride: + + controller: + # Process Ingress objects without ingressClass annotation/ingressClassName field + # Overrides value for --watch-ingress-without-class flag of the controller binary + # Defaults to false + watchIngressWithoutClass: false + + # Required for use with CNI based kubernetes installations (such as ones set up by kubeadm), + # since CNI and hostport don't mix yet. Can be deprecated once https://github.com/kubernetes/kubernetes/issues/23920 + # is merged + hostNetwork: false + + ## Use host ports 80 and 443 + ## Disabled by default + ## + hostPort: + enabled: true + ports: + http: 80 + https: 443 + + # This section refers to the creation of the IngressClass resource + # IngressClass resources are supported since k8s >= 1.18 and required since k8s >= 1.19 + ingressClassResource: + name: nginx + enabled: true + default: true + controllerValue: "k8s.io/ingress-nginx" + + # Parameters is a link to a custom resource containing additional + # configuration for the controller. This is optional if the controller + # does not require extra parameters. + parameters: {} + + ## Allows customization of the source of the IP address or FQDN to report + ## in the ingress status field. By default, it reads the information provided + ## by the service. If disable, the status field reports the IP address of the + ## node or nodes where an ingress controller pod is running. + publishService: + enabled: true + ## Allows overriding of the publish service to bind to + ## Must be <namespace>/<service_name> + ## + pathOverride: "" + + ## Limit the scope of the controller + ## + scope: + enabled: false + namespace: "" # defaults to $(POD_NAMESPACE) + + ## Allows customization of the configmap / nginx-configmap namespace + ## + configMapNamespace: "" # defaults to $(POD_NAMESPACE) + + ## Allows customization of the tcp-services-configmap + ## + tcp: + configMapNamespace: "" # defaults to $(POD_NAMESPACE) + ## Annotations to be added to the tcp config configmap + annotations: {} + + ## Allows customization of the udp-services-configmap + ## + udp: + configMapNamespace: "" # defaults to $(POD_NAMESPACE) + ## Annotations to be added to the udp config configmap + annotations: {} + + livenessProbe: + httpGet: + # should match container.healthCheckPath + path: "/healthz" + port: 10254 + scheme: HTTP + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 1 + successThreshold: 1 + failureThreshold: 5 + readinessProbe: + httpGet: + # should match container.healthCheckPath + path: "/healthz" + port: 10254 + scheme: HTTP + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 1 + successThreshold: 1 + failureThreshold: 3 + + replicaCount: 2 + + minAvailable: 1 + + # Define requests resources to avoid probe issues due to CPU utilization in busy nodes + # ref: https://github.com/kubernetes/ingress-nginx/issues/4735#issuecomment-551204903 + # Ideally, there should be no limits. + # https://engineering.indeedblog.com/blog/2019/12/cpu-throttling-regression-fix/ + resources: + # limits: + # cpu: 100m + # memory: 90Mi + requests: + cpu: 100m + memory: 90Mi + + ## Enable mimalloc as a drop-in replacement for malloc. + ## ref: https://github.com/microsoft/mimalloc + ## + enableMimalloc: true + + ## Override NGINX template + customTemplate: + configMapName: "" + configMapKey: "" + + service: + enabled: true + + annotations: {} + labels: {} + # clusterIP: "" + + ## List of IP addresses at which the controller services are available + ## Ref: https://kubernetes.io/docs/user-guide/services/#external-ips + ## + externalIPs: [] + + # loadBalancerIP: "" + loadBalancerSourceRanges: [] + + enableHttp: true + enableHttps: true + + ## Set external traffic policy to: "Local" to preserve source IP on + ## providers supporting it + ## Ref: https://kubernetes.io/docs/tutorials/services/source-ip/#source-ip-for-services-with-typeloadbalancer + # externalTrafficPolicy: "" + + # Must be either "None" or "ClientIP" if set. Kubernetes will default to "None". + # Ref: https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies + # sessionAffinity: "" + + # specifies the health check node port (numeric port number) for the service. If healthCheckNodePort isn’t specified, + # the service controller allocates a port from your cluster’s NodePort range. + # Ref: https://kubernetes.io/docs/tasks/access-application-cluster/create-external-load-balancer/#preserving-the-client-source-ip + # healthCheckNodePort: 0 + + ports: + http: 80 + https: 443 + + targetPorts: + http: http + https: https + + type: LoadBalancer + + # type: NodePort + # nodePorts: + # http: 32080 + # https: 32443 + # tcp: + # 8080: 32808 + nodePorts: + http: "" + https: "" + tcp: {} + udp: {} + + ## Enables an additional internal load balancer (besides the external one). + ## Annotations are mandatory for the load balancer to come up. Varies with the cloud service. + internal: + enabled: false + annotations: {} + + # loadBalancerIP: "" + + ## Restrict access For LoadBalancer service. Defaults to 0.0.0.0/0. + loadBalancerSourceRanges: [] + + ## Set external traffic policy to: "Local" to preserve source IP on + ## providers supporting it + ## Ref: https://kubernetes.io/docs/tutorials/services/source-ip/#source-ip-for-services-with-typeloadbalancer + # externalTrafficPolicy: "" + + admissionWebhooks: + annotations: {} + enabled: true + failurePolicy: Fail + # timeoutSeconds: 10 + port: 8443 + certificate: "/usr/local/certificates/cert" + key: "/usr/local/certificates/key" + namespaceSelector: {} + objectSelector: {} + + # Use an existing PSP instead of creating one + existingPsp: "" + + service: + annotations: {} + # clusterIP: "" + externalIPs: [] + # loadBalancerIP: "" + loadBalancerSourceRanges: [] + servicePort: 443 + type: ClusterIP + + createSecretJob: + resources: {} + # limits: + # cpu: 10m + # memory: 20Mi + # requests: + # cpu: 10m + # memory: 20Mi + + patchWebhookJob: + resources: {} + + patch: + enabled: false + image: + registry: k8s.gcr.io + image: ingress-nginx/kube-webhook-certgen + # for backwards compatibility consider setting the full image url via the repository value below + # use *either* current default registry/image or repository format or installing chart by providing the values.yaml will fail + # repository: + tag: v1.0 + digest: sha256:f3b6b39a6062328c095337b4cadcefd1612348fdd5190b1dcbcb9b9e90bd8068 + pullPolicy: IfNotPresent + ## Provide a priority class name to the webhook patching job + ## + priorityClassName: "" + podAnnotations: {} + nodeSelector: + kubernetes.io/os: linux + tolerations: [] + runAsUser: 2000 + + metrics: + port: 10254 + # if this port is changed, change healthz-port: in extraArgs: accordingly + enabled: true + + service: + annotations: {} + # prometheus.io/scrape: "true" + # prometheus.io/port: "10254" + + # clusterIP: "" + + ## List of IP addresses at which the stats-exporter service is available + ## Ref: https://kubernetes.io/docs/user-guide/services/#external-ips + ## + externalIPs: [] + + # loadBalancerIP: "" + loadBalancerSourceRanges: [] + servicePort: 10254 + type: ClusterIP + # externalTrafficPolicy: "" + # nodePort: "" + + serviceMonitor: + enabled: true + additionalLabels: {} + # The label to use to retrieve the job name from. + # jobLabel: "app.kubernetes.io/name" + namespace: "" + namespaceSelector: {} + # Default: scrape .Release.Namespace only + # To scrape all, use the following: + # namespaceSelector: + # any: true + scrapeInterval: 30s + # honorLabels: true + targetLabels: [] + metricRelabelings: [] + + prometheusRule: + enabled: true + additionalLabels: {} + # namespace: "" + rules: [] + # # These are just examples rules, please adapt them to your needs + - alert: NGINXConfigFailed + expr: count(nginx_ingress_controller_config_last_reload_successful == 0) > 0 + for: 30m + labels: + severity: critical + annotations: + description: bad ingress config - nginx config test failed + summary: uninstall the latest ingress changes to allow config reloads to resume + - alert: NGINXCertificateExpiry + expr: (avg(nginx_ingress_controller_ssl_expire_time_seconds) by (host) - time()) < 604800 + for: 30m + labels: + severity: critical + annotations: + description: ssl certificate(s) will expire in less then a week + summary: renew expiring certificates to avoid downtime + - alert: NGINXTooMany500s + expr: 100 * ( sum( nginx_ingress_controller_requests{status=~"5.+"} ) / sum(nginx_ingress_controller_requests) ) > 5 + for: 30m + labels: + severity: warning + annotations: + description: Too many 5XXs + summary: More than 5% of all requests returned 5XX, this requires your attention + - alert: NGINXTooMany400s + expr: 100 * ( sum( nginx_ingress_controller_requests{status=~"4.+"} ) / sum(nginx_ingress_controller_requests) ) > 5 + for: 30m + labels: + severity: warning + annotations: + description: Too many 4XXs + summary: More than 5% of all requests returned 4XX, this requires your attention + + ## Improve connection draining when ingress controller pod is deleted using a lifecycle hook: + ## With this new hook, we increased the default terminationGracePeriodSeconds from 30 seconds + ## to 300, allowing the draining of connections up to five minutes. + ## If the active connections end before that, the pod will terminate gracefully at that time. + ## To effectively take advantage of this feature, the Configmap feature + ## worker-shutdown-timeout new value is 240s instead of 10s. + ## + lifecycle: + preStop: + exec: + command: + - /wait-shutdown + + priorityClassName: "" + + ## Rollback limit + ## + revisionHistoryLimit: 10 + + ## Default 404 backend + ## + defaultBackend: + ## + enabled: true + + name: defaultbackend + image: + registry: k8s.gcr.io + image: defaultbackend-amd64 + # for backwards compatibility consider setting the full image url via the repository value below + # use *either* current default registry/image or repository format or installing chart by providing the values.yaml will fail + # repository: + tag: "1.5" + pullPolicy: IfNotPresent + # nobody user -> uid 65534 + runAsUser: 65534 + runAsNonRoot: true + readOnlyRootFilesystem: true + allowPrivilegeEscalation: false + + # Use an existing PSP instead of creating one + existingPsp: "" + + extraArgs: {} + + serviceAccount: + create: true + name: "" + automountServiceAccountToken: true + ## Additional environment variables to set for defaultBackend pods + extraEnvs: [] + + port: 8080 + + ## Readiness and liveness probes for default backend + ## Ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-probes/ + ## + livenessProbe: + failureThreshold: 3 + initialDelaySeconds: 30 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 5 + readinessProbe: + failureThreshold: 6 + initialDelaySeconds: 0 + periodSeconds: 5 + successThreshold: 1 + timeoutSeconds: 5 + + ## Node tolerations for server scheduling to nodes with taints + ## Ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/ + ## + tolerations: [] + # - key: "key" + # operator: "Equal|Exists" + # value: "value" + # effect: "NoSchedule|PreferNoSchedule|NoExecute(1.6 only)" + + affinity: {} + + ## Security Context policies for controller pods + ## See https://kubernetes.io/docs/tasks/administer-cluster/sysctl-cluster/ for + ## notes on enabling and using sysctls + ## + podSecurityContext: {} + + # labels to add to the pod container metadata + podLabels: {} + # key: value + + ## Node labels for default backend pod assignment + ## Ref: https://kubernetes.io/docs/user-guide/node-selection/ + ## + nodeSelector: + kubernetes.io/os: linux + + ## Annotations to be added to default backend pods + ## + podAnnotations: {} + + replicaCount: 1 + + minAvailable: 1 + + resources: {} + # limits: + # cpu: 10m + # memory: 20Mi + # requests: + # cpu: 10m + # memory: 20Mi + + extraVolumeMounts: [] + ## Additional volumeMounts to the default backend container. + # - name: copy-portal-skins + # mountPath: /var/lib/lemonldap-ng/portal/skins + + extraVolumes: [] + ## Additional volumes to the default backend pod. + # - name: copy-portal-skins + # emptyDir: {} + + autoscaling: + annotations: {} + enabled: false + minReplicas: 1 + maxReplicas: 2 + targetCPUUtilizationPercentage: 50 + targetMemoryUtilizationPercentage: 50 + + service: + annotations: {} + + # clusterIP: "" + + ## List of IP addresses at which the default backend service is available + ## Ref: https://kubernetes.io/docs/user-guide/services/#external-ips + ## + externalIPs: [] + + # loadBalancerIP: "" + loadBalancerSourceRanges: [] + servicePort: 80 + type: ClusterIP + + priorityClassName: "" + + ## Enable RBAC as per https://github.com/kubernetes/ingress-nginx/blob/main/docs/deploy/rbac.md and https://github.com/kubernetes/ingress-nginx/issues/266 + rbac: + create: true + scope: false + + # If true, create & use Pod Security Policy resources + # https://kubernetes.io/docs/concepts/policy/pod-security-policy/ + podSecurityPolicy: + enabled: true + + serviceAccount: + create: true + name: "" + automountServiceAccountToken: true + + ## Optional array of imagePullSecrets containing private registry credentials + ## Ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/ + imagePullSecrets: [] + # - name: secretName + + # TCP service key:value pairs + # Ref: https://github.com/kubernetes/ingress-nginx/blob/main/docs/user-guide/exposing-tcp-udp-services.md + ## + tcp: {} + # 8080: "default/example-tcp-svc:9000" + + # UDP service key:value pairs + # Ref: https://github.com/kubernetes/ingress-nginx/blob/main/docs/user-guide/exposing-tcp-udp-services.md + ## + udp: {} + # 53: "kube-system/kube-dns:53" + + # A base64ed Diffie-Hellman parameter + # This can be generated with: openssl dhparam 4096 2> /dev/null | base64 + # Ref: https://github.com/kubernetes/ingress-nginx/tree/main/docs/examples/customization/ssl-dh-param + dhParam: diff --git a/infrastructure/ingress-nginx/repository.yaml b/infrastructure/ingress-nginx/repository.yaml new file mode 100644 index 0000000000000000000000000000000000000000..55a23ab31c4aaaab11d1587ba2b63f7db831f479 --- /dev/null +++ b/infrastructure/ingress-nginx/repository.yaml @@ -0,0 +1,7 @@ +apiVersion: source.toolkit.fluxcd.io/v1beta1 +kind: HelmRepository +metadata: + name: nginx-ingress +spec: + interval: 30m + url: https://kubernetes.github.io/ingress-nginx diff --git a/infrastructure/kustomization.yaml b/infrastructure/kustomization.yaml index e6c03593cc63ca80123e32443824f5d8bcef62b1..b263c688d0054eb72348caaec87b1e62bcd5d59d 100644 --- a/infrastructure/kustomization.yaml +++ b/infrastructure/kustomization.yaml @@ -6,3 +6,4 @@ resources: - rook - cert-manager - prometheus + - ingress-nginx