From 7010e32130932372c87562881bb0d4908cc12ad0 Mon Sep 17 00:00:00 2001
From: Zachary Yonash <yonash402@gmail.com>
Date: Thu, 27 Jul 2017 03:49:25 -0400
Subject: [PATCH] Added a few extra node rules (#478)
---
assets/prometheus/rules/node.rules | 32 +++++++++++++++++++
.../prometheus/prometheus-k8s-rules.yaml | 32 +++++++++++++++++++
2 files changed, 64 insertions(+)
diff --git a/assets/prometheus/rules/node.rules b/assets/prometheus/rules/node.rules
index 36ea482c..54085392 100644
--- a/assets/prometheus/rules/node.rules
+++ b/assets/prometheus/rules/node.rules
@@ -8,3 +8,35 @@ ALERT NodeExporterDown
summary = "node-exporter cannot be scraped",
description = "Prometheus could not scrape a node-exporter for more than 10m, or node-exporters have disappeared from discovery.",
}
+ALERT K8SNodeOutOfDisk
+ IF kube_node_status_out_of_disk{condition="true"} == 1
+ LABELS {
+ service = "k8s",
+ severity = "critical"
+ }
+ ANNOTATIONS {
+ summary = "Node ran out of disk space.",
+ description = "{{ $labels.node }} has run out of disk space.",
+ }
+
+ALERT K8SNodeMemoryPressure
+ IF kube_node_status_memory_pressure{condition="true"} == 1
+ LABELS {
+ service = "k8s",
+ severity = "warning"
+ }
+ ANNOTATIONS {
+ summary = "Node is under memory pressure.",
+ description = "{{ $labels.node }} is under memory pressure.",
+ }
+
+ALERT K8SNodeDiskPressure
+ IF kube_node_status_disk_pressure{condition="true"} == 1
+ LABELS {
+ service = "k8s",
+ severity = "warning"
+ }
+ ANNOTATIONS {
+ summary = "Node is under disk pressure.",
+ description = "{{ $labels.node }} is under disk pressure.",
+ }
diff --git a/manifests/prometheus/prometheus-k8s-rules.yaml b/manifests/prometheus/prometheus-k8s-rules.yaml
index 181a70c7..e1740562 100644
--- a/manifests/prometheus/prometheus-k8s-rules.yaml
+++ b/manifests/prometheus/prometheus-k8s-rules.yaml
@@ -582,6 +582,38 @@ data:
summary = "node-exporter cannot be scraped",
description = "Prometheus could not scrape a node-exporter for more than 10m, or node-exporters have disappeared from discovery.",
}
+ ALERT K8SNodeOutOfDisk
+ IF kube_node_status_out_of_disk{condition="true"} == 1
+ LABELS {
+ service = "k8s",
+ severity = "critical"
+ }
+ ANNOTATIONS {
+ summary = "Node ran out of disk space.",
+ description = "{{ $labels.node }} has run out of disk space.",
+ }
+
+ ALERT K8SNodeMemoryPressure
+ IF kube_node_status_memory_pressure{condition="true"} == 1
+ LABELS {
+ service = "k8s",
+ severity = "warning"
+ }
+ ANNOTATIONS {
+ summary = "Node is under memory pressure.",
+ description = "{{ $labels.node }} is under memory pressure.",
+ }
+
+ ALERT K8SNodeDiskPressure
+ IF kube_node_status_disk_pressure{condition="true"} == 1
+ LABELS {
+ service = "k8s",
+ severity = "warning"
+ }
+ ANNOTATIONS {
+ summary = "Node is under disk pressure.",
+ description = "{{ $labels.node }} is under disk pressure.",
+ }
prometheus.rules: |+
ALERT FailedReload
IF prometheus_config_last_reload_successful == 0
--
GitLab