From 804f6c187bcf99434490e13b930fee7fa5d61a1c Mon Sep 17 00:00:00 2001
From: Frederic Branczyk <fbranczyk@gmail.com>
Date: Tue, 30 May 2017 17:07:13 -0700
Subject: [PATCH] kube-prometheus: add dead man's switch
---
assets/alertmanager/alertmanager.yaml | 10 ++++++----
assets/prometheus/rules/general.rules | 12 ++++++++++++
manifests/alertmanager/alertmanager-config.yaml | 2 +-
manifests/prometheus/prometheus-k8s-rules.yaml | 12 ++++++++++++
4 files changed, 31 insertions(+), 5 deletions(-)
diff --git a/assets/alertmanager/alertmanager.yaml b/assets/alertmanager/alertmanager.yaml
index f08a2106..6b5789b5 100644
--- a/assets/alertmanager/alertmanager.yaml
+++ b/assets/alertmanager/alertmanager.yaml
@@ -5,8 +5,10 @@ route:
group_wait: 30s
group_interval: 5m
repeat_interval: 12h
- receiver: 'webhook'
+ receiver: 'null'
+ routes:
+ - match:
+ alertname: DeadMansSwitch
+ receiver: 'null'
receivers:
-- name: 'webhook'
- webhook_configs:
- - url: 'http://alertmanagerwh:30500/'
+- name: 'null'
diff --git a/assets/prometheus/rules/general.rules b/assets/prometheus/rules/general.rules
index 9a8f931f..ac782297 100644
--- a/assets/prometheus/rules/general.rules
+++ b/assets/prometheus/rules/general.rules
@@ -11,6 +11,18 @@ Alert TargetDown
description = "A target of type {{ $labels.job }} is down."
}
+### Dead man's switch ###
+
+ALERT DeadMansSwitch
+ IF vector(1)
+ LABELS {
+ severity = "none",
+ }
+ ANNOTATIONS {
+ summary = "Alerting DeadMansSwitch",
+ description = "This is a DeadMansSwitch meant to ensure that the entire Alerting pipeline is functional.",
+ }
+
### File descriptor alerts ###
ALERT TooManyOpenFiles
diff --git a/manifests/alertmanager/alertmanager-config.yaml b/manifests/alertmanager/alertmanager-config.yaml
index eee36b33..62d39016 100644
--- a/manifests/alertmanager/alertmanager-config.yaml
+++ b/manifests/alertmanager/alertmanager-config.yaml
@@ -3,4 +3,4 @@ kind: Secret
metadata:
name: alertmanager-main
data:
- alertmanager.yaml: Z2xvYmFsOgogIHJlc29sdmVfdGltZW91dDogNW0Kcm91dGU6CiAgZ3JvdXBfYnk6IFsnam9iJ10KICBncm91cF93YWl0OiAzMHMKICBncm91cF9pbnRlcnZhbDogNW0KICByZXBlYXRfaW50ZXJ2YWw6IDEyaAogIHJlY2VpdmVyOiAnd2ViaG9vaycKcmVjZWl2ZXJzOgotIG5hbWU6ICd3ZWJob29rJwogIHdlYmhvb2tfY29uZmlnczoKICAtIHVybDogJ2h0dHA6Ly9hbGVydG1hbmFnZXJ3aDozMDUwMC8nCg==
+ alertmanager.yaml: Z2xvYmFsOgogIHJlc29sdmVfdGltZW91dDogNW0Kcm91dGU6CiAgZ3JvdXBfYnk6IFsnam9iJ10KICBncm91cF93YWl0OiAzMHMKICBncm91cF9pbnRlcnZhbDogNW0KICByZXBlYXRfaW50ZXJ2YWw6IDEyaAogIHJlY2VpdmVyOiAnbnVsbCcKICByb3V0ZXM6CiAgLSBtYXRjaDoKICAgICAgYWxlcnRuYW1lOiBEZWFkTWFuc1N3aXRjaAogICAgcmVjZWl2ZXI6ICdudWxsJwpyZWNlaXZlcnM6Ci0gbmFtZTogJ251bGwnCg==
diff --git a/manifests/prometheus/prometheus-k8s-rules.yaml b/manifests/prometheus/prometheus-k8s-rules.yaml
index f57b6785..c157909f 100644
--- a/manifests/prometheus/prometheus-k8s-rules.yaml
+++ b/manifests/prometheus/prometheus-k8s-rules.yaml
@@ -235,6 +235,18 @@ data:
description = "A target of type {{ $labels.job }} is down."
}
+ ### Dead man's switch ###
+
+ ALERT DeadMansSwitch
+ IF vector(1)
+ LABELS {
+ severity = "none",
+ }
+ ANNOTATIONS {
+ summary = "Alerting DeadMansSwitch",
+ description = "This is a DeadMansSwitch meant to ensure that the entire Alerting pipeline is functional.",
+ }
+
### File descriptor alerts ###
ALERT TooManyOpenFiles
--
GitLab