diff --git a/assets/prometheus/rules/node.rules b/assets/prometheus/rules/node.rules index 36ea482cf5dd63ef1847656498213c92c07c4ef0..540853927f26f99d005bfdb93687bbb8039ce6d0 100644 --- a/assets/prometheus/rules/node.rules +++ b/assets/prometheus/rules/node.rules @@ -8,3 +8,35 @@ ALERT NodeExporterDown summary = "node-exporter cannot be scraped", description = "Prometheus could not scrape a node-exporter for more than 10m, or node-exporters have disappeared from discovery.", } +ALERT K8SNodeOutOfDisk + IF kube_node_status_out_of_disk{condition="true"} == 1 + LABELS { + service = "k8s", + severity = "critical" + } + ANNOTATIONS { + summary = "Node ran out of disk space.", + description = "{{ $labels.node }} has run out of disk space.", + } + +ALERT K8SNodeMemoryPressure + IF kube_node_status_memory_pressure{condition="true"} == 1 + LABELS { + service = "k8s", + severity = "warning" + } + ANNOTATIONS { + summary = "Node is under memory pressure.", + description = "{{ $labels.node }} is under memory pressure.", + } + +ALERT K8SNodeDiskPressure + IF kube_node_status_disk_pressure{condition="true"} == 1 + LABELS { + service = "k8s", + severity = "warning" + } + ANNOTATIONS { + summary = "Node is under disk pressure.", + description = "{{ $labels.node }} is under disk pressure.", + } diff --git a/manifests/prometheus/prometheus-k8s-rules.yaml b/manifests/prometheus/prometheus-k8s-rules.yaml index 181a70c788610186aff459ad42c98559f6b7719c..e17405620975326b8847603a923916eb05eafe6b 100644 --- a/manifests/prometheus/prometheus-k8s-rules.yaml +++ b/manifests/prometheus/prometheus-k8s-rules.yaml @@ -582,6 +582,38 @@ data: summary = "node-exporter cannot be scraped", description = "Prometheus could not scrape a node-exporter for more than 10m, or node-exporters have disappeared from discovery.", } + ALERT K8SNodeOutOfDisk + IF kube_node_status_out_of_disk{condition="true"} == 1 + LABELS { + service = "k8s", + severity = "critical" + } + ANNOTATIONS { + summary = "Node ran out of disk space.", + description = "{{ $labels.node }} has run out of disk space.", + } + + ALERT K8SNodeMemoryPressure + IF kube_node_status_memory_pressure{condition="true"} == 1 + LABELS { + service = "k8s", + severity = "warning" + } + ANNOTATIONS { + summary = "Node is under memory pressure.", + description = "{{ $labels.node }} is under memory pressure.", + } + + ALERT K8SNodeDiskPressure + IF kube_node_status_disk_pressure{condition="true"} == 1 + LABELS { + service = "k8s", + severity = "warning" + } + ANNOTATIONS { + summary = "Node is under disk pressure.", + description = "{{ $labels.node }} is under disk pressure.", + } prometheus.rules: |+ ALERT FailedReload IF prometheus_config_last_reload_successful == 0