diff --git a/jsonnet/kube-prometheus/alerts/alertmanager.libsonnet b/jsonnet/kube-prometheus/alerts/alertmanager.libsonnet index 42ae3f98353e23d5d4dbc527ff3d17b7a4018220..bf58862dd26a16b718a2b3d931faa0ef2819d744 100644 --- a/jsonnet/kube-prometheus/alerts/alertmanager.libsonnet +++ b/jsonnet/kube-prometheus/alerts/alertmanager.libsonnet @@ -37,7 +37,7 @@ }, expr: ||| alertmanager_cluster_members{%(alertmanagerSelector)s} - != on (service) + != on (service) GROUP_LEFT() count by (service) (alertmanager_cluster_members{%(alertmanagerSelector)s}) ||| % $._config, 'for': '5m', diff --git a/jsonnet/kube-prometheus/alerts/tests.yaml b/jsonnet/kube-prometheus/alerts/tests.yaml index 8cfc3aa7c648c6a273803a7f7ef52f4e1079d8a2..532bb895561757e0024a66ef045dbce11f90afc8 100644 --- a/jsonnet/kube-prometheus/alerts/tests.yaml +++ b/jsonnet/kube-prometheus/alerts/tests.yaml @@ -24,6 +24,10 @@ tests: - exp_labels: service: 'alertmanager-main' severity: critical + job: 'alertmanager-main' + instance: 10.10.10.0 + namespace: monitoring + pod: alertmanager-main-0 exp_annotations: message: 'Alertmanager has not found all other members of the cluster.' - eval_time: 17m @@ -32,6 +36,10 @@ tests: - exp_labels: service: 'alertmanager-main' severity: critical + job: 'alertmanager-main' + instance: 10.10.10.0 + namespace: monitoring + pod: alertmanager-main-0 exp_annotations: message: 'Alertmanager has not found all other members of the cluster.' - eval_time: 23m @@ -40,5 +48,110 @@ tests: - exp_labels: service: 'alertmanager-main' severity: critical + job: 'alertmanager-main' + instance: 10.10.10.0 + namespace: monitoring + pod: alertmanager-main-0 + exp_annotations: + message: 'Alertmanager has not found all other members of the cluster.' + - interval: 1m + input_series: + - series: 'alertmanager_cluster_members{job="alertmanager-main",instance="10.10.10.0",namespace="monitoring",pod="alertmanager-main-0",service="alertmanager-main"}' + values: '3 3 3 3 3 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1' + - series: 'alertmanager_cluster_members{job="alertmanager-main",instance="10.10.10.1",namespace="monitoring",pod="alertmanager-main-1",service="alertmanager-main"}' + values: '3 3 3 3 3 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2' + - series: 'alertmanager_cluster_members{job="alertmanager-main",instance="10.10.10.2",namespace="monitoring",pod="alertmanager-main-2",service="alertmanager-main"}' + values: '3 3 3 3 3 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2' + alert_rule_test: + - eval_time: 5m + alertname: AlertmanagerMembersInconsistent + - eval_time: 11m + alertname: AlertmanagerMembersInconsistent + exp_alerts: + - exp_labels: + service: 'alertmanager-main' + severity: critical + job: 'alertmanager-main' + instance: 10.10.10.0 + namespace: monitoring + pod: alertmanager-main-0 + exp_annotations: + message: 'Alertmanager has not found all other members of the cluster.' + - exp_labels: + service: 'alertmanager-main' + severity: critical + job: 'alertmanager-main' + instance: 10.10.10.1 + namespace: monitoring + pod: alertmanager-main-1 + exp_annotations: + message: 'Alertmanager has not found all other members of the cluster.' + - exp_labels: + service: 'alertmanager-main' + severity: critical + job: 'alertmanager-main' + instance: 10.10.10.2 + namespace: monitoring + pod: alertmanager-main-2 + exp_annotations: + message: 'Alertmanager has not found all other members of the cluster.' + - eval_time: 17m + alertname: AlertmanagerMembersInconsistent + exp_alerts: + - exp_labels: + service: 'alertmanager-main' + severity: critical + job: 'alertmanager-main' + instance: 10.10.10.0 + namespace: monitoring + pod: alertmanager-main-0 + exp_annotations: + message: 'Alertmanager has not found all other members of the cluster.' + - exp_labels: + service: 'alertmanager-main' + severity: critical + job: 'alertmanager-main' + instance: 10.10.10.1 + namespace: monitoring + pod: alertmanager-main-1 + exp_annotations: + message: 'Alertmanager has not found all other members of the cluster.' + - exp_labels: + service: 'alertmanager-main' + severity: critical + job: 'alertmanager-main' + instance: 10.10.10.2 + namespace: monitoring + pod: alertmanager-main-2 + exp_annotations: + message: 'Alertmanager has not found all other members of the cluster.' + - eval_time: 23m + alertname: AlertmanagerMembersInconsistent + exp_alerts: + - exp_labels: + service: 'alertmanager-main' + severity: critical + job: 'alertmanager-main' + instance: 10.10.10.0 + namespace: monitoring + pod: alertmanager-main-0 + exp_annotations: + message: 'Alertmanager has not found all other members of the cluster.' + - exp_labels: + service: 'alertmanager-main' + severity: critical + job: 'alertmanager-main' + instance: 10.10.10.1 + namespace: monitoring + pod: alertmanager-main-1 + exp_annotations: + message: 'Alertmanager has not found all other members of the cluster.' + - exp_labels: + service: 'alertmanager-main' + severity: critical + job: 'alertmanager-main' + instance: 10.10.10.2 + namespace: monitoring + pod: alertmanager-main-2 exp_annotations: message: 'Alertmanager has not found all other members of the cluster.' diff --git a/jsonnetfile.lock.json b/jsonnetfile.lock.json index 3b9d80cd6aac08ebbdb928e549260864cb1b09a0..3665f14f323d6070ed73e18a3ede39c959be4f3a 100644 --- a/jsonnetfile.lock.json +++ b/jsonnetfile.lock.json @@ -8,7 +8,7 @@ "subdir": "contrib/kube-prometheus/jsonnet/kube-prometheus" } }, - "version": "dff8f44fbce268596c86b8d586c64c17953feab3" + "version": "9cc151ced4308573a91f4cc3fcdbc951213b03e0" }, { "name": "ksonnet", diff --git a/manifests/prometheus-rules.yaml b/manifests/prometheus-rules.yaml index a7987abc6c1b5ebc5ce9e0c96766931a86be774f..872cf310546dc184770c9203b0c90585816dd065 100644 --- a/manifests/prometheus-rules.yaml +++ b/manifests/prometheus-rules.yaml @@ -790,7 +790,7 @@ spec: message: Alertmanager has not found all other members of the cluster. expr: | alertmanager_cluster_members{job="alertmanager-main"} - != on (service) + != on (service) GROUP_LEFT() count by (service) (alertmanager_cluster_members{job="alertmanager-main"}) for: 5m labels: