diff --git a/jsonnet/kube-prometheus/alerts/prometheus-operator.libsonnet b/jsonnet/kube-prometheus/alerts/prometheus-operator.libsonnet index 580aa866d246234fe515d9bf67eac70f502cdac3..a8ddd13affe7800b470e03e5412ce2b709d3e1d3 100644 --- a/jsonnet/kube-prometheus/alerts/prometheus-operator.libsonnet +++ b/jsonnet/kube-prometheus/alerts/prometheus-operator.libsonnet @@ -4,16 +4,29 @@ { name: 'prometheus-operator', rules: [ + { + alert: 'PrometheusOperatorListErrors', + expr: ||| + (sum by (controller,namespace) (rate(prometheus_operator_list_operations_failed_total{%(prometheusOperatorSelector)s}[1h])) / sum by (controller,namespace) (rate(prometheus_operator_list_operations_total{%(prometheusOperatorSelector)s}[1h]))) > 0.4 + ||| % $._config, + labels: { + severity: 'warning', + }, + annotations: { + message: 'Errors while performing List operations in controller {{$labels.controller}} in {{$labels.namespace}} namespace.', + }, + 'for': '15m', + }, { alert: 'PrometheusOperatorWatchErrors', expr: ||| - (sum by (controller,namespace) (rate(prometheus_operator_watch_operations_failed_total{%(prometheusOperatorSelector)s}[1h])) / sum by (controller,namespace) (rate(prometheus_operator_watch_operations_total{%(prometheusOperatorSelector)s}[1h]))) > 0.1 + (sum by (controller,namespace) (rate(prometheus_operator_watch_operations_failed_total{%(prometheusOperatorSelector)s}[1h])) / sum by (controller,namespace) (rate(prometheus_operator_watch_operations_total{%(prometheusOperatorSelector)s}[1h]))) > 0.4 ||| % $._config, labels: { severity: 'warning', }, annotations: { - message: 'Errors while performing watch operations in controller {{$labels.controller}} in {{$labels.namespace}} namespace.', + message: 'Errors while performing Watch operations in controller {{$labels.controller}} in {{$labels.namespace}} namespace.', }, 'for': '15m', }, diff --git a/manifests/prometheus-rules.yaml b/manifests/prometheus-rules.yaml index 94c3fe4ee7b6faab4b0f1ea7aada8649efe5548b..3bd6d63ab2e45fccc81af56202c21c9105bfb350 100644 --- a/manifests/prometheus-rules.yaml +++ b/manifests/prometheus-rules.yaml @@ -1793,12 +1793,21 @@ spec: severity: warning - name: prometheus-operator rules: + - alert: PrometheusOperatorListErrors + annotations: + message: Errors while performing List operations in controller {{$labels.controller}} + in {{$labels.namespace}} namespace. + expr: | + (sum by (controller,namespace) (rate(prometheus_operator_list_operations_failed_total{job="prometheus-operator",namespace="monitoring"}[1h])) / sum by (controller,namespace) (rate(prometheus_operator_list_operations_total{job="prometheus-operator",namespace="monitoring"}[1h]))) > 0.4 + for: 15m + labels: + severity: warning - alert: PrometheusOperatorWatchErrors annotations: - message: Errors while performing watch operations in controller {{$labels.controller}} + message: Errors while performing Watch operations in controller {{$labels.controller}} in {{$labels.namespace}} namespace. expr: | - (sum by (controller,namespace) (rate(prometheus_operator_watch_operations_failed_total{job="prometheus-operator",namespace="monitoring"}[1h])) / sum by (controller,namespace) (rate(prometheus_operator_watch_operations_total{job="prometheus-operator",namespace="monitoring"}[1h]))) > 0.1 + (sum by (controller,namespace) (rate(prometheus_operator_watch_operations_failed_total{job="prometheus-operator",namespace="monitoring"}[1h])) / sum by (controller,namespace) (rate(prometheus_operator_watch_operations_total{job="prometheus-operator",namespace="monitoring"}[1h]))) > 0.4 for: 15m labels: severity: warning