diff --git a/jsonnet/kube-prometheus/alerts/prometheus-operator.libsonnet b/jsonnet/kube-prometheus/alerts/prometheus-operator.libsonnet index a430c5050a26abbbc3e2a3e8b72197aeaafd8061..580aa866d246234fe515d9bf67eac70f502cdac3 100644 --- a/jsonnet/kube-prometheus/alerts/prometheus-operator.libsonnet +++ b/jsonnet/kube-prometheus/alerts/prometheus-operator.libsonnet @@ -4,6 +4,19 @@ { name: 'prometheus-operator', rules: [ + { + alert: 'PrometheusOperatorWatchErrors', + expr: ||| + (sum by (controller,namespace) (rate(prometheus_operator_watch_operations_failed_total{%(prometheusOperatorSelector)s}[1h])) / sum by (controller,namespace) (rate(prometheus_operator_watch_operations_total{%(prometheusOperatorSelector)s}[1h]))) > 0.1 + ||| % $._config, + labels: { + severity: 'warning', + }, + annotations: { + message: 'Errors while performing watch operations in controller {{$labels.controller}} in {{$labels.namespace}} namespace.', + }, + 'for': '15m', + }, { alert: 'PrometheusOperatorReconcileErrors', expr: ||| diff --git a/manifests/prometheus-rules.yaml b/manifests/prometheus-rules.yaml index aefacdf1e2497f77d9a5f458d626d7f9dbe01f94..94c3fe4ee7b6faab4b0f1ea7aada8649efe5548b 100644 --- a/manifests/prometheus-rules.yaml +++ b/manifests/prometheus-rules.yaml @@ -1793,6 +1793,15 @@ spec: severity: warning - name: prometheus-operator rules: + - alert: PrometheusOperatorWatchErrors + annotations: + message: Errors while performing watch operations in controller {{$labels.controller}} + in {{$labels.namespace}} namespace. + expr: | + (sum by (controller,namespace) (rate(prometheus_operator_watch_operations_failed_total{job="prometheus-operator",namespace="monitoring"}[1h])) / sum by (controller,namespace) (rate(prometheus_operator_watch_operations_total{job="prometheus-operator",namespace="monitoring"}[1h]))) > 0.1 + for: 15m + labels: + severity: warning - alert: PrometheusOperatorReconcileErrors annotations: message: Errors while reconciling {{ $labels.controller }} in {{ $labels.namespace