diff --git a/jsonnet/kube-prometheus/alerts/general.libsonnet b/jsonnet/kube-prometheus/alerts/general.libsonnet index 6ac2570340a0e35ccc28450f66a68e2377ea20e6..8802097e071f81fdb6cb83f842fc292937cbbdb7 100644 --- a/jsonnet/kube-prometheus/alerts/general.libsonnet +++ b/jsonnet/kube-prometheus/alerts/general.libsonnet @@ -16,9 +16,15 @@ }, }, { - alert: 'DeadMansSwitch', + alert: 'Watchdog', annotations: { - message: 'This is a DeadMansSwitch meant to ensure that the entire alerting pipeline is functional.', + message: ||| + This is an alert meant to ensure that the entire alerting pipeline is functional. + This alert is always firing, therefore it should always be firing in Alertmanager + and always fire against a receiver. There are integrations with various notification + mechanisms that send a notification when this alert is not firing. For example the + "DeadMansSnitch" integration in PagerDuty. + |||, }, expr: 'vector(1)', labels: { diff --git a/jsonnetfile.lock.json b/jsonnetfile.lock.json index 0712855847b3db96af2fb41b9b305b77676e4eba..f194ca86ebc9026aeefde41d0018da4746e166ae 100644 --- a/jsonnetfile.lock.json +++ b/jsonnetfile.lock.json @@ -8,7 +8,7 @@ "subdir": "contrib/kube-prometheus/jsonnet/kube-prometheus" } }, - "version": "3f9400b1788a78c17abb0159cd84295cfc93bc15" + "version": "f12ea289a1a99cf9fe433ec3a086d436baef6466" }, { "name": "ksonnet", @@ -48,7 +48,7 @@ "subdir": "grafana-builder" } }, - "version": "ec3d4f943df01f517a083305666cd1c87bcc7e94" + "version": "24590e90dcfc9d581c4208b4bb3d15df50f1328e" }, { "name": "grafana", diff --git a/manifests/grafana-dashboardDefinitions.yaml b/manifests/grafana-dashboardDefinitions.yaml index f3374edce5df1b05b856c0332377e76078f3790d..92fdc805e0c2a2afc2330bf9baa4e5ef9d069208 100644 --- a/manifests/grafana-dashboardDefinitions.yaml +++ b/manifests/grafana-dashboardDefinitions.yaml @@ -31,7 +31,7 @@ items: "dashes": false, "datasource": "$datasource", "fill": 10, - "id": 0, + "id": 1, "legend": { "avg": false, "current": false, @@ -117,7 +117,7 @@ items: "dashes": false, "datasource": "$datasource", "fill": 10, - "id": 1, + "id": 2, "legend": { "avg": false, "current": false, @@ -215,7 +215,7 @@ items: "dashes": false, "datasource": "$datasource", "fill": 10, - "id": 2, + "id": 3, "legend": { "avg": false, "current": false, @@ -301,7 +301,7 @@ items: "dashes": false, "datasource": "$datasource", "fill": 10, - "id": 3, + "id": 4, "legend": { "avg": false, "current": false, @@ -399,7 +399,7 @@ items: "dashes": false, "datasource": "$datasource", "fill": 10, - "id": 4, + "id": 5, "legend": { "avg": false, "current": false, @@ -485,7 +485,7 @@ items: "dashes": false, "datasource": "$datasource", "fill": 10, - "id": 5, + "id": 6, "legend": { "avg": false, "current": false, @@ -583,7 +583,7 @@ items: "dashes": false, "datasource": "$datasource", "fill": 10, - "id": 6, + "id": 7, "legend": { "avg": false, "current": false, @@ -669,7 +669,7 @@ items: "dashes": false, "datasource": "$datasource", "fill": 10, - "id": 7, + "id": 8, "legend": { "avg": false, "current": false, @@ -767,7 +767,7 @@ items: "dashes": false, "datasource": "$datasource", "fill": 10, - "id": 8, + "id": 9, "legend": { "avg": false, "current": false, @@ -947,7 +947,7 @@ items: "dashes": false, "datasource": "$datasource", "fill": 1, - "id": 0, + "id": 1, "legend": { "avg": false, "current": false, @@ -1033,7 +1033,7 @@ items: "dashes": false, "datasource": "$datasource", "fill": 1, - "id": 1, + "id": 2, "legend": { "avg": false, "current": false, @@ -1131,7 +1131,7 @@ items: "dashes": false, "datasource": "$datasource", "fill": 1, - "id": 2, + "id": 3, "legend": { "avg": false, "current": false, @@ -1217,7 +1217,7 @@ items: "dashes": false, "datasource": "$datasource", "fill": 1, - "id": 3, + "id": 4, "legend": { "avg": false, "current": false, @@ -1315,7 +1315,7 @@ items: "dashes": false, "datasource": "$datasource", "fill": 1, - "id": 4, + "id": 5, "legend": { "avg": false, "current": false, @@ -1401,7 +1401,7 @@ items: "dashes": false, "datasource": "$datasource", "fill": 1, - "id": 5, + "id": 6, "legend": { "avg": false, "current": false, @@ -1499,7 +1499,7 @@ items: "dashes": false, "datasource": "$datasource", "fill": 1, - "id": 6, + "id": 7, "legend": { "avg": false, "current": false, @@ -1585,7 +1585,7 @@ items: "dashes": false, "datasource": "$datasource", "fill": 1, - "id": 7, + "id": 8, "legend": { "avg": false, "current": false, @@ -1683,7 +1683,7 @@ items: "dashes": false, "datasource": "$datasource", "fill": 1, - "id": 8, + "id": 9, "legend": { "avg": false, "current": false, @@ -1891,7 +1891,7 @@ items: "datasource": "$datasource", "fill": 1, "format": "percentunit", - "id": 0, + "id": 1, "legend": { "avg": false, "current": false, @@ -1975,7 +1975,7 @@ items: "datasource": "$datasource", "fill": 1, "format": "percentunit", - "id": 1, + "id": 2, "legend": { "avg": false, "current": false, @@ -2059,7 +2059,7 @@ items: "datasource": "$datasource", "fill": 1, "format": "percentunit", - "id": 2, + "id": 3, "legend": { "avg": false, "current": false, @@ -2143,7 +2143,7 @@ items: "datasource": "$datasource", "fill": 1, "format": "percentunit", - "id": 3, + "id": 4, "legend": { "avg": false, "current": false, @@ -2227,7 +2227,7 @@ items: "datasource": "$datasource", "fill": 1, "format": "percentunit", - "id": 4, + "id": 5, "legend": { "avg": false, "current": false, @@ -2311,7 +2311,7 @@ items: "datasource": "$datasource", "fill": 1, "format": "percentunit", - "id": 5, + "id": 6, "legend": { "avg": false, "current": false, @@ -2406,7 +2406,7 @@ items: "dashes": false, "datasource": "$datasource", "fill": 10, - "id": 6, + "id": 7, "legend": { "avg": false, "current": false, @@ -2504,7 +2504,7 @@ items: "dashes": false, "datasource": "$datasource", "fill": 1, - "id": 7, + "id": 8, "legend": { "avg": false, "current": false, @@ -2771,7 +2771,7 @@ items: "dashes": false, "datasource": "$datasource", "fill": 10, - "id": 8, + "id": 9, "legend": { "avg": false, "current": false, @@ -2869,7 +2869,7 @@ items: "dashes": false, "datasource": "$datasource", "fill": 1, - "id": 9, + "id": 10, "legend": { "avg": false, "current": false, @@ -3218,7 +3218,7 @@ items: "dashes": false, "datasource": "$datasource", "fill": 10, - "id": 0, + "id": 1, "legend": { "avg": false, "current": false, @@ -3316,7 +3316,7 @@ items: "dashes": false, "datasource": "$datasource", "fill": 1, - "id": 1, + "id": 2, "legend": { "avg": false, "current": false, @@ -3583,7 +3583,7 @@ items: "dashes": false, "datasource": "$datasource", "fill": 10, - "id": 2, + "id": 3, "legend": { "avg": false, "current": false, @@ -3681,7 +3681,7 @@ items: "dashes": false, "datasource": "$datasource", "fill": 1, - "id": 3, + "id": 4, "legend": { "avg": false, "current": false, @@ -4057,7 +4057,7 @@ items: "dashes": false, "datasource": "$datasource", "fill": 10, - "id": 0, + "id": 1, "legend": { "avg": false, "current": false, @@ -4155,7 +4155,7 @@ items: "dashes": false, "datasource": "$datasource", "fill": 1, - "id": 1, + "id": 2, "legend": { "avg": false, "current": false, @@ -4422,7 +4422,7 @@ items: "dashes": false, "datasource": "$datasource", "fill": 10, - "id": 2, + "id": 3, "legend": { "avg": false, "current": false, @@ -4520,7 +4520,7 @@ items: "dashes": false, "datasource": "$datasource", "fill": 1, - "id": 3, + "id": 4, "legend": { "avg": false, "current": false, diff --git a/manifests/prometheus-rules.yaml b/manifests/prometheus-rules.yaml index 443943c01cada85d07b068c502bc9d34fc981fab..aaf16ff56750fa86a030725ecbf4bf9fbcccb285 100644 --- a/manifests/prometheus-rules.yaml +++ b/manifests/prometheus-rules.yaml @@ -831,10 +831,14 @@ spec: for: 10m labels: severity: warning - - alert: DeadMansSwitch - annotations: - message: This is a DeadMansSwitch meant to ensure that the entire alerting - pipeline is functional. + - alert: Watchdog + annotations: + message: | + This is an alert meant to ensure that the entire alerting pipeline is functional. + This alert is always firing, therefore it should always be firing in Alertmanager + and always fire against a receiver. There are integrations with various notification + mechanisms that send a notification when this alert is not firing. For example the + "DeadMansSnitch" integration in PagerDuty. expr: vector(1) labels: severity: none