diff --git a/jsonnet/kube-prometheus/components/node-exporter.libsonnet b/jsonnet/kube-prometheus/components/node-exporter.libsonnet index 8de79f1825e1f3aebe7a9b0dca4be3d798b13704..5de8ebcb7308ea58ab44a19e4b43a116b4c43e55 100644 --- a/jsonnet/kube-prometheus/components/node-exporter.libsonnet +++ b/jsonnet/kube-prometheus/components/node-exporter.libsonnet @@ -35,9 +35,12 @@ local defaults = { // GC values, // imageGCLowThresholdPercent: 80 // imageGCHighThresholdPercent: 85 + // GC kicks in when imageGCHighThresholdPercent is hit and attempts to free upto imageGCLowThresholdPercent. // See https://kubernetes.io/docs/reference/config-api/kubelet-config.v1beta1/ for more details. - fsSpaceFillingUpWarningThreshold: 20, - fsSpaceFillingUpCriticalThreshold: 15, + // Warn only after imageGCHighThresholdPercent is hit, but filesystem is not freed up for a prolonged duration. + fsSpaceFillingUpWarningThreshold: 15, + // Send critical alert only after (imageGCHighThresholdPercent + 5) is hit, but filesystem is not freed up for a prolonged duration. + fsSpaceFillingUpCriticalThreshold: 10, diskDeviceSelector: 'device=~"mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|dasd.+"', runbookURLPattern: 'https://runbooks.prometheus-operator.dev/runbooks/node/%s', }, diff --git a/manifests/nodeExporter-prometheusRule.yaml b/manifests/nodeExporter-prometheusRule.yaml index f9b90b8eaab2efdba0ff443b2ccb88de85628716..e4e25bbab02c4ac81759f75b924b2f3a067f96f4 100644 --- a/manifests/nodeExporter-prometheusRule.yaml +++ b/manifests/nodeExporter-prometheusRule.yaml @@ -23,7 +23,7 @@ spec: summary: Filesystem is predicted to run out of space within the next 24 hours. expr: | ( - node_filesystem_avail_bytes{job="node-exporter",fstype!=""} / node_filesystem_size_bytes{job="node-exporter",fstype!=""} * 100 < 20 + node_filesystem_avail_bytes{job="node-exporter",fstype!=""} / node_filesystem_size_bytes{job="node-exporter",fstype!=""} * 100 < 15 and predict_linear(node_filesystem_avail_bytes{job="node-exporter",fstype!=""}[6h], 24*60*60) < 0 and @@ -41,7 +41,7 @@ spec: summary: Filesystem is predicted to run out of space within the next 4 hours. expr: | ( - node_filesystem_avail_bytes{job="node-exporter",fstype!=""} / node_filesystem_size_bytes{job="node-exporter",fstype!=""} * 100 < 15 + node_filesystem_avail_bytes{job="node-exporter",fstype!=""} / node_filesystem_size_bytes{job="node-exporter",fstype!=""} * 100 < 10 and predict_linear(node_filesystem_avail_bytes{job="node-exporter",fstype!=""}[6h], 4*60*60) < 0 and