diff --git a/jsonnet/kube-prometheus/components/alertmanager.libsonnet b/jsonnet/kube-prometheus/components/alertmanager.libsonnet index cd9ab9dbeb57af038cdb464c0c2bf0cce21cee87..4d17c8ab2ac3e86e210b8f09d13b52bc3c4077c2 100644 --- a/jsonnet/kube-prometheus/components/alertmanager.libsonnet +++ b/jsonnet/kube-prometheus/components/alertmanager.libsonnet @@ -35,6 +35,10 @@ local defaults = { source_matchers: ['severity = warning'], target_matchers: ['severity = info'], equal: ['namespace', 'alertname'], + }, { + source_matchers: ['alertname = InfoInhibitor'], + target_matchers: ['severity = info'], + equal: ['namespace'], }], route: { group_by: ['namespace'], @@ -44,6 +48,7 @@ local defaults = { receiver: 'Default', routes: [ { receiver: 'Watchdog', matchers: ['alertname = Watchdog'] }, + { receiver: 'null', matchers: ['alertname = InfoInhibitor'] }, { receiver: 'Critical', matchers: ['severity = critical'] }, ], }, @@ -51,6 +56,7 @@ local defaults = { { name: 'Default' }, { name: 'Watchdog' }, { name: 'Critical' }, + { name: 'null' }, ], }, replicas: 3, diff --git a/jsonnet/kube-prometheus/components/mixin/alerts/general.libsonnet b/jsonnet/kube-prometheus/components/mixin/alerts/general.libsonnet index cd5c71657d4ec09914cb360488e2facf239580b1..8e69d98f6d8933d956f5a5c5acb926c7d9b16284 100644 --- a/jsonnet/kube-prometheus/components/mixin/alerts/general.libsonnet +++ b/jsonnet/kube-prometheus/components/mixin/alerts/general.libsonnet @@ -33,6 +33,24 @@ severity: 'none', }, }, + { + alert: 'InfoInhibitor', + annotations: { + summary: 'Info-level alert inhibition.', + description: ||| + This is an alert that is used to inhibit info alerts. + By themselves, the info-level alerts are sometimes very noisy, but they are relevant when combined with + other alerts. + This alert fires whenever there's a severity="info" alert, and stops firing when another alert with a + severity of 'warning' or 'critical' starts firing on the same namespace. + This alert should be routed to a null receiver and configured to inhibit alerts with severity="info". + |||, + }, + expr: 'ALERTS{severity = "info"} == 1 unless on(namespace) ALERTS{alertname != "InfoInhibitor", severity =~ "warning|critical", alertstate="firing"} == 1', + labels: { + severity: 'none', + }, + }, ], }, ], diff --git a/manifests/alertmanager-secret.yaml b/manifests/alertmanager-secret.yaml index 1b24f70b8b49b9fab9b0e3f8c2b392021bc72392..e27e46df2473cd08072ffea18902c6592a6c6452 100644 --- a/manifests/alertmanager-secret.yaml +++ b/manifests/alertmanager-secret.yaml @@ -28,10 +28,17 @@ stringData: - "severity = warning" "target_matchers": - "severity = info" + - "equal": + - "namespace" + "source_matchers": + - "alertname = InfoInhibitor" + "target_matchers": + - "severity = info" "receivers": - "name": "Default" - "name": "Watchdog" - "name": "Critical" + - "name": "null" "route": "group_by": - "namespace" @@ -43,6 +50,9 @@ stringData: - "matchers": - "alertname = Watchdog" "receiver": "Watchdog" + - "matchers": + - "alertname = InfoInhibitor" + "receiver": "null" - "matchers": - "severity = critical" "receiver": "Critical" diff --git a/manifests/kubePrometheus-prometheusRule.yaml b/manifests/kubePrometheus-prometheusRule.yaml index e563e794ecc253e42573ba817d8d3548d119e8c5..297954d3deef33b67d70f6da2ee27296d75cdb67 100644 --- a/manifests/kubePrometheus-prometheusRule.yaml +++ b/manifests/kubePrometheus-prometheusRule.yaml @@ -38,6 +38,21 @@ spec: expr: vector(1) labels: severity: none + - alert: InfoInhibitor + annotations: + description: | + This is an alert that is used to inhibit info alerts. + By themselves, the info-level alerts are sometimes very noisy, but they are relevant when combined with + other alerts. + This alert fires whenever there's a severity="info" alert, and stops firing when another alert with a + severity of 'warning' or 'critical' starts firing on the same namespace. + This alert should be routed to a null receiver and configured to inhibit alerts with severity="info". + runbook_url: https://runbooks.prometheus-operator.dev/runbooks/general/infoinhibitor + summary: Info-level alert inhibition. + expr: ALERTS{severity = "info"} == 1 unless on(namespace) ALERTS{alertname != + "InfoInhibitor", severity =~ "warning|critical", alertstate="firing"} == 1 + labels: + severity: none - name: node-network rules: - alert: NodeNetworkInterfaceFlapping