From d728ab5511ea839448499998358c4e5c53c86d24 Mon Sep 17 00:00:00 2001
From: Ali Rizwan <aleerizw@gmail.com>
Date: Mon, 23 Jul 2018 12:20:54 +0200
Subject: [PATCH] Only alert for nodes that currently exist (#1661)
---
jsonnet/kube-prometheus/alerts/node.libsonnet | 4 ++--
manifests/prometheus-rules.yaml | 4 ++--
2 files changed, 4 insertions(+), 4 deletions(-)
diff --git a/jsonnet/kube-prometheus/alerts/node.libsonnet b/jsonnet/kube-prometheus/alerts/node.libsonnet
index 46a5e36d..5c24f09f 100644
--- a/jsonnet/kube-prometheus/alerts/node.libsonnet
+++ b/jsonnet/kube-prometheus/alerts/node.libsonnet
@@ -11,7 +11,7 @@
summary: 'Node disk is running full within 24 hours',
},
expr: |||
- predict_linear(node_filesystem_free{%(nodeExporterSelector)s,mountpoint!~"^/etc/(?:resolv.conf|hosts|hostname)$"}[6h], 3600 * 24) < 0
+ predict_linear(node_filesystem_free{%(nodeExporterSelector)s,mountpoint!~"^/etc/(?:resolv.conf|hosts|hostname)$"}[6h], 3600 * 24) < 0 and on(instance) up{%(nodeExporterSelector)s}
||| % $._config,
'for': '30m',
labels: {
@@ -25,7 +25,7 @@
summary: 'Node disk is running full within 2 hours',
},
expr: |||
- predict_linear(node_filesystem_free{%(nodeExporterSelector)s,mountpoint!~"^/etc/(?:resolv.conf|hosts|hostname)$"}[30m], 3600 * 2) < 0
+ predict_linear(node_filesystem_free{%(nodeExporterSelector)s,mountpoint!~"^/etc/(?:resolv.conf|hosts|hostname)$"}[30m], 3600 * 2) < 0 and on(instance) up{%(nodeExporterSelector)s}
||| % $._config,
'for': '10m',
labels: {
diff --git a/manifests/prometheus-rules.yaml b/manifests/prometheus-rules.yaml
index 49c4a995..5af7d2fa 100644
--- a/manifests/prometheus-rules.yaml
+++ b/manifests/prometheus-rules.yaml
@@ -744,7 +744,7 @@ spec:
full within the next 24 hours (mounted at {{$labels.mountpoint}})
summary: Node disk is running full within 24 hours
expr: |
- predict_linear(node_filesystem_free{job="node-exporter",mountpoint!~"^/etc/(?:resolv.conf|hosts|hostname)$"}[6h], 3600 * 24) < 0
+ predict_linear(node_filesystem_free{job="node-exporter",mountpoint!~"^/etc/(?:resolv.conf|hosts|hostname)$"}[6h], 3600 * 24) < 0 and on(instance) up{job="node-exporter"}
for: 30m
labels:
severity: warning
@@ -754,7 +754,7 @@ spec:
full within the next 2 hours (mounted at {{$labels.mountpoint}})
summary: Node disk is running full within 2 hours
expr: |
- predict_linear(node_filesystem_free{job="node-exporter",mountpoint!~"^/etc/(?:resolv.conf|hosts|hostname)$"}[30m], 3600 * 2) < 0
+ predict_linear(node_filesystem_free{job="node-exporter",mountpoint!~"^/etc/(?:resolv.conf|hosts|hostname)$"}[30m], 3600 * 2) < 0 and on(instance) up{job="node-exporter"}
for: 10m
labels:
severity: critical
--
GitLab