Only alert for nodes that currently exist (#1661)

jsonnet/kube-prometheus/alerts/node.libsonnet

@@ -11,7 +11,7 @@
               summary: 'Node disk is running full within 24 hours',
             },
             expr: |||
-              predict_linear(node_filesystem_free{%(nodeExporterSelector)s,mountpoint!~"^/etc/(?:resolv.conf|hosts|hostname)$"}[6h], 3600 * 24) < 0
+              predict_linear(node_filesystem_free{%(nodeExporterSelector)s,mountpoint!~"^/etc/(?:resolv.conf|hosts|hostname)$"}[6h], 3600 * 24) < 0 and on(instance) up{%(nodeExporterSelector)s}
             ||| % $._config,
             'for': '30m',
             labels: {
@@ -25,7 +25,7 @@
               summary: 'Node disk is running full within 2 hours',
             },
             expr: |||
-              predict_linear(node_filesystem_free{%(nodeExporterSelector)s,mountpoint!~"^/etc/(?:resolv.conf|hosts|hostname)$"}[30m], 3600 * 2) < 0
+              predict_linear(node_filesystem_free{%(nodeExporterSelector)s,mountpoint!~"^/etc/(?:resolv.conf|hosts|hostname)$"}[30m], 3600 * 2) < 0 and on(instance) up{%(nodeExporterSelector)s}
             ||| % $._config,
             'for': '10m',
             labels: {

manifests/prometheus-rules.yaml

@@ -744,7 +744,7 @@ spec:
           full within the next 24 hours (mounted at {{$labels.mountpoint}})
         summary: Node disk is running full within 24 hours
       expr: |
-        predict_linear(node_filesystem_free{job="node-exporter",mountpoint!~"^/etc/(?:resolv.conf|hosts|hostname)$"}[6h], 3600 * 24) < 0
+        predict_linear(node_filesystem_free{job="node-exporter",mountpoint!~"^/etc/(?:resolv.conf|hosts|hostname)$"}[6h], 3600 * 24) < 0 and on(instance) up{job="node-exporter"}
       for: 30m
       labels:
         severity: warning
@@ -754,7 +754,7 @@ spec:
           full within the next 2 hours (mounted at {{$labels.mountpoint}})
         summary: Node disk is running full within 2 hours
       expr: |
-        predict_linear(node_filesystem_free{job="node-exporter",mountpoint!~"^/etc/(?:resolv.conf|hosts|hostname)$"}[30m], 3600 * 2) < 0
+        predict_linear(node_filesystem_free{job="node-exporter",mountpoint!~"^/etc/(?:resolv.conf|hosts|hostname)$"}[30m], 3600 * 2) < 0 and on(instance) up{job="node-exporter"}
       for: 10m
       labels:
         severity: critical