Merge pull request #1729 from arajkumar/fix-NodeFilesystemSpaceFillingUp-thresholds

Adjust NodeFilesystemSpaceFillingUp thresholds according default kubelet GC behavior

jsonnet/kube-prometheus/components/node-exporter.libsonnet

@@ -35,9 +35,12 @@ local defaults = {
       // GC values,
       // imageGCLowThresholdPercent: 80
       // imageGCHighThresholdPercent: 85
+      // GC kicks in when imageGCHighThresholdPercent is hit and attempts to free upto imageGCLowThresholdPercent.
       // See https://kubernetes.io/docs/reference/config-api/kubelet-config.v1beta1/ for more details.
-      fsSpaceFillingUpWarningThreshold: 20,
-      fsSpaceFillingUpCriticalThreshold: 15,
+      // Warn only after imageGCHighThresholdPercent is hit, but filesystem is not freed up for a prolonged duration.
+      fsSpaceFillingUpWarningThreshold: 15,
+      // Send critical alert only after (imageGCHighThresholdPercent + 5) is hit, but filesystem is not freed up for a prolonged duration.
+      fsSpaceFillingUpCriticalThreshold: 10,
       diskDeviceSelector: 'device=~"mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|dasd.+"',
       runbookURLPattern: 'https://runbooks.prometheus-operator.dev/runbooks/node/%s',
     },

manifests/nodeExporter-prometheusRule.yaml

@@ -23,7 +23,7 @@ spec:
         summary: Filesystem is predicted to run out of space within the next 24 hours.
       expr: |
         (
-          node_filesystem_avail_bytes{job="node-exporter",fstype!=""} / node_filesystem_size_bytes{job="node-exporter",fstype!=""} * 100 < 20
+          node_filesystem_avail_bytes{job="node-exporter",fstype!=""} / node_filesystem_size_bytes{job="node-exporter",fstype!=""} * 100 < 15
         and
           predict_linear(node_filesystem_avail_bytes{job="node-exporter",fstype!=""}[6h], 24*60*60) < 0
         and
@@ -41,7 +41,7 @@ spec:
         summary: Filesystem is predicted to run out of space within the next 4 hours.
       expr: |
         (
-          node_filesystem_avail_bytes{job="node-exporter",fstype!=""} / node_filesystem_size_bytes{job="node-exporter",fstype!=""} * 100 < 15
+          node_filesystem_avail_bytes{job="node-exporter",fstype!=""} / node_filesystem_size_bytes{job="node-exporter",fstype!=""} * 100 < 10
         and
           predict_linear(node_filesystem_avail_bytes{job="node-exporter",fstype!=""}[6h], 4*60*60) < 0
         and