Merge pull request #610 from lilic/add-more-alerts

Add PrometheusOperatorListErrors and fix PrometheusOperatorWatchErrors threshold 

jsonnet/kube-prometheus/alerts/prometheus-operator.libsonnet

@@ -4,16 +4,29 @@
       {
         name: 'prometheus-operator',
         rules: [
+          {
+            alert: 'PrometheusOperatorListErrors',
+            expr: |||
+              (sum by (controller,namespace) (rate(prometheus_operator_list_operations_failed_total{%(prometheusOperatorSelector)s}[1h])) / sum by (controller,namespace) (rate(prometheus_operator_list_operations_total{%(prometheusOperatorSelector)s}[1h]))) > 0.4
+            ||| % $._config,
+            labels: {
+              severity: 'warning',
+            },
+            annotations: {
+              message: 'Errors while performing List operations in controller {{$labels.controller}} in {{$labels.namespace}} namespace.',
+            },
+            'for': '15m',
+          },
           {
             alert: 'PrometheusOperatorWatchErrors',
             expr: |||
-              (sum by (controller,namespace) (rate(prometheus_operator_watch_operations_failed_total{%(prometheusOperatorSelector)s}[1h])) / sum by (controller,namespace) (rate(prometheus_operator_watch_operations_total{%(prometheusOperatorSelector)s}[1h]))) > 0.1
+              (sum by (controller,namespace) (rate(prometheus_operator_watch_operations_failed_total{%(prometheusOperatorSelector)s}[1h])) / sum by (controller,namespace) (rate(prometheus_operator_watch_operations_total{%(prometheusOperatorSelector)s}[1h]))) > 0.4
             ||| % $._config,
             labels: {
               severity: 'warning',
             },
             annotations: {
-              message: 'Errors while performing watch operations in controller {{$labels.controller}} in {{$labels.namespace}} namespace.',
+              message: 'Errors while performing Watch operations in controller {{$labels.controller}} in {{$labels.namespace}} namespace.',
             },
             'for': '15m',
           },

manifests/prometheus-rules.yaml

@@ -1793,12 +1793,21 @@ spec:
         severity: warning
   - name: prometheus-operator
     rules:
+    - alert: PrometheusOperatorListErrors
+      annotations:
+        message: Errors while performing List operations in controller {{$labels.controller}}
+          in {{$labels.namespace}} namespace.
+      expr: |
+        (sum by (controller,namespace) (rate(prometheus_operator_list_operations_failed_total{job="prometheus-operator",namespace="monitoring"}[1h])) / sum by (controller,namespace) (rate(prometheus_operator_list_operations_total{job="prometheus-operator",namespace="monitoring"}[1h]))) > 0.4
+      for: 15m
+      labels:
+        severity: warning
     - alert: PrometheusOperatorWatchErrors
       annotations:
-        message: Errors while performing watch operations in controller {{$labels.controller}}
+        message: Errors while performing Watch operations in controller {{$labels.controller}}
           in {{$labels.namespace}} namespace.
       expr: |
-        (sum by (controller,namespace) (rate(prometheus_operator_watch_operations_failed_total{job="prometheus-operator",namespace="monitoring"}[1h])) / sum by (controller,namespace) (rate(prometheus_operator_watch_operations_total{job="prometheus-operator",namespace="monitoring"}[1h]))) > 0.1
+        (sum by (controller,namespace) (rate(prometheus_operator_watch_operations_failed_total{job="prometheus-operator",namespace="monitoring"}[1h])) / sum by (controller,namespace) (rate(prometheus_operator_watch_operations_total{job="prometheus-operator",namespace="monitoring"}[1h]))) > 0.4
       for: 15m
       labels:
         severity: warning