feat(flux-system): Add alert for failing reconsiliations

This patch adds an alert for failing flux reconsiliations that should
prevent failing flux deployments from staying unnoticed.

infrastructure/flux-system/alerts.yaml

+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+  name: flux-system-alerts
+spec:
+  groups:
+    - name: GitOpsToolkit
+      rules:
+        - alert: ReconciliationFailure
+          expr: max(gotk_reconcile_condition{status="False",type="Ready"}) by (exported_namespace, name, kind) + on(exported_namespace, name, kind) (max(gotk_reconcile_condition{status="Deleted"}) by (exported_namespace, name, kind)) * 2 == 1
+          for: 10m
+          labels:
+            severity: critical
+          annotations:
+            summary: '{{ $labels.kind }} {{ $labels.exported_namespace }}/{{ $labels.name }} reconciliation has been failing for more than ten minutes.'

infrastructure/flux-system/kustomization.yaml

@@ -3,6 +3,7 @@ kind: Kustomization
 namespace: flux-system
 resources:
   - monitoring.yaml
+  - alerts.yaml
   - ../../shared/networkpolicies/allow-from-monitoring.yaml
 patchesStrategicMerge:
   - networkpolicy.yaml