feat(nginx-system): Add affinity configuration

This patch forces a spreading across multiple Nodes for ingress
controller Pods. This should ensure that Pods a single node failing,
doesn't take down all ingress Pods.

Further this patch introduces a node affinity, that prefers nodes with
better CPU feature support. In order to utilise hardware acceleration as
much as possible.

References:
https://www.intel.com/content/dam/develop/external/us/en/documents/open-ssl-performance-paper-345527.pdf

infrastructure/nginx-system/release.yaml

@@ -31,6 +31,47 @@ metadata:
 data:
   values.yaml: |
     controller:
+      affinity:
+        # Force spread across nodes
+        podAntiAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+          - labelSelector:
+              matchExpressions:
+              - key: app.kubernetes.io/name
+                operator: In
+                values:
+                - ingress-nginx
+              - key: app.kubernetes.io/instance
+                operator: In
+                values:
+                - nginx-ingress
+              - key: app.kubernetes.io/component
+                operator: In
+                values:
+                - controller
+            topologyKey: "kubernetes.io/hostname"
+        nodeAffinity:
+          preferredDuringSchedulingIgnoredDuringExecution:
+          - weight: 5
+            preference:
+              matchExpressions:
+              - key: feature.node.kubernetes.io/cpu-cpuid.AESNI
+                operator: Exists
+          - weight: 1
+            preference:
+              matchExpressions:
+              - key: feature.node.kubernetes.io/cpu-cpuid.SHA
+                operator: Exists
+          - weight: 1
+            preference:
+              matchExpressions:
+              - key: feature.node.kubernetes.io/cpu-cpuid.AVX
+                operator: Exists
+          - weight: 1
+            preference:
+              matchExpressions:
+              - key: feature.node.kubernetes.io/cpu-cpuid.AVX2
+                operator: Exists
       ingressClassResource:
         name: nginx
         enabled: true