Merge pull request #609 from lilic/add-prom-operator-alerts

prometheus-operator.libsonnet: Add PrometheusOperatorWatchErrors alert

jsonnet/kube-prometheus/alerts/prometheus-operator.libsonnet

@@ -4,6 +4,19 @@
       {
         name: 'prometheus-operator',
         rules: [
+          {
+            alert: 'PrometheusOperatorWatchErrors',
+            expr: |||
+              (sum by (controller,namespace) (rate(prometheus_operator_watch_operations_failed_total{%(prometheusOperatorSelector)s}[1h])) / sum by (controller,namespace) (rate(prometheus_operator_watch_operations_total{%(prometheusOperatorSelector)s}[1h]))) > 0.1
+            ||| % $._config,
+            labels: {
+              severity: 'warning',
+            },
+            annotations: {
+              message: 'Errors while performing watch operations in controller {{$labels.controller}} in {{$labels.namespace}} namespace.',
+            },
+            'for': '15m',
+          },
           {
             alert: 'PrometheusOperatorReconcileErrors',
             expr: |||

manifests/prometheus-rules.yaml

@@ -1793,6 +1793,15 @@ spec:
         severity: warning
   - name: prometheus-operator
     rules:
+    - alert: PrometheusOperatorWatchErrors
+      annotations:
+        message: Errors while performing watch operations in controller {{$labels.controller}}
+          in {{$labels.namespace}} namespace.
+      expr: |
+        (sum by (controller,namespace) (rate(prometheus_operator_watch_operations_failed_total{job="prometheus-operator",namespace="monitoring"}[1h])) / sum by (controller,namespace) (rate(prometheus_operator_watch_operations_total{job="prometheus-operator",namespace="monitoring"}[1h]))) > 0.1
+      for: 15m
+      labels:
+        severity: warning
     - alert: PrometheusOperatorReconcileErrors
       annotations:
         message: Errors while reconciling {{ $labels.controller }} in {{ $labels.namespace