diff --git a/jsonnet/kube-prometheus/alerts/node.libsonnet b/jsonnet/kube-prometheus/alerts/node.libsonnet
index 27de83b8c062382f16a67a0b1f43bdd59c64dd13..68a4242514d76c7743e4c48520081cc819fd9b5f 100644
--- a/jsonnet/kube-prometheus/alerts/node.libsonnet
+++ b/jsonnet/kube-prometheus/alerts/node.libsonnet
@@ -41,7 +41,7 @@
message: 'Clock skew detected on node-exporter {{ $labels.namespace }}/{{ $labels.pod }}. Ensure NTP is configured correctly on this host.',
},
expr: |||
- node_ntp_offset_seconds{%(nodeExporterSelector)s} < -0.03 or node_ntp_offset_seconds{%(nodeExporterSelector)s} > 0.03
+ abs(node_timex_offset_seconds{%(nodeExporterSelector)s}) > 0.03
||| % $._config,
'for': '2m',
labels: {
diff --git a/jsonnet/kube-prometheus/node-exporter/node-exporter.libsonnet b/jsonnet/kube-prometheus/node-exporter/node-exporter.libsonnet
index 8aae5b86dda47add15145add3eae37e027fb0c91..6b7f7f8a35a48559e137a8e39a6aca70e985bd8e 100644
--- a/jsonnet/kube-prometheus/node-exporter/node-exporter.libsonnet
+++ b/jsonnet/kube-prometheus/node-exporter/node-exporter.libsonnet
@@ -101,7 +101,6 @@ local k = import 'ksonnet/ksonnet.beta.3/k.libsonnet';
// Once node exporter is being released with those settings, this can be removed.
'--collector.filesystem.ignored-mount-points=^/(dev|proc|sys|var/lib/docker/.+)($|/)',
'--collector.filesystem.ignored-fs-types=^(autofs|binfmt_misc|cgroup|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|mqueue|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|sysfs|tracefs)$',
- '--collector.ntp',
]) +
container.withVolumeMounts([procVolumeMount, sysVolumeMount, rootVolumeMount]) +
container.mixin.resources.withRequests({ cpu: '102m', memory: '180Mi' }) +
diff --git a/jsonnetfile.lock.json b/jsonnetfile.lock.json
index 2798bacd25e8aa2e6b6139ffa97f0e03a9476c04..1a5963867719147615d9decbad69647439a82e6c 100644
--- a/jsonnetfile.lock.json
+++ b/jsonnetfile.lock.json
@@ -8,7 +8,7 @@
"subdir": "jsonnet/kube-prometheus"
}
},
- "version": "1fbf75767a59d7fff5a7d716657349fcd884e4d3"
+ "version": "abd16f1b991a1f5dfcaef7bb0089eb245c817f2e"
},
{
"name": "ksonnet",
@@ -28,7 +28,7 @@
"subdir": ""
}
},
- "version": "6187d7dbbf4bf0b7d34d08e6773dc0e101676c58"
+ "version": "7360753d27aa428758c918434503c1c35afcd3bb"
},
{
"name": "grafonnet",
@@ -38,7 +38,7 @@
"subdir": "grafonnet"
}
},
- "version": "d270f529db9eb750425a173188c534ab92532f47"
+ "version": "a6896d19aedc46ecf80dd64967191b9fd6f75f45"
},
{
"name": "grafana-builder",
@@ -48,7 +48,7 @@
"subdir": "grafana-builder"
}
},
- "version": "233757a0a9047eb74993c977cd5b1be0fb54e8d6"
+ "version": "05392db2e3d0f9285d3e3f39f4a0408366b0a873"
},
{
"name": "grafana",
@@ -78,7 +78,7 @@
"subdir": "Documentation/etcd-mixin"
}
},
- "version": "cd7ffbe2270d5290e708f55edfbfae840ec879bb"
+ "version": "8146e1ebdf1f54791edf33e85e0c816619c7d9cd"
}
]
}
diff --git a/manifests/grafana-dashboardDefinitions.yaml b/manifests/grafana-dashboardDefinitions.yaml
index 5efcb65ff66baa63e5fc694c04d02206c1c96682..abd59cf62de8500f5d6a33bc8c1382dc7a535220 100644
--- a/manifests/grafana-dashboardDefinitions.yaml
+++ b/manifests/grafana-dashboardDefinitions.yaml
@@ -8610,7 +8610,7 @@ items:
},
"id": 2,
"legend": {
- "alignAsTable": false,
+ "alignAsTable": true,
"avg": true,
"current": true,
"max": true,
@@ -8635,16 +8635,23 @@ items:
],
"spaceLength": 10,
- "span": 12,
- "stack": false,
+ "span": 9,
+ "stack": true,
"steppedLine": false,
"targets": [
{
- "expr": "(kubelet_volume_stats_capacity_bytes{cluster=\"$cluster\", job=\"kubelet\", persistentvolumeclaim=\"$volume\"} - kubelet_volume_stats_available_bytes{cluster=\"$cluster\", job=\"kubelet\", persistentvolumeclaim=\"$volume\"}) / kubelet_volume_stats_capacity_bytes{cluster=\"$cluster\", job=\"kubelet\", persistentvolumeclaim=\"$volume\"} * 100\n",
+ "expr": "(\n sum without(instance, node) (kubelet_volume_stats_capacity_bytes{cluster=\"$cluster\", job=\"kubelet\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})\n -\n sum without(instance, node) (kubelet_volume_stats_available_bytes{cluster=\"$cluster\", job=\"kubelet\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})\n)\n",
"format": "time_series",
"intervalFactor": 1,
- "legendFormat": "{{ Usage }}",
+ "legendFormat": "Used Space",
"refId": "A"
+ },
+ {
+ "expr": "sum without(instance, node) (kubelet_volume_stats_available_bytes{cluster=\"$cluster\", job=\"kubelet\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})\n",
+ "format": "time_series",
+ "intervalFactor": 1,
+ "legendFormat": "Free Space",
+ "refId": "B"
}
],
"thresholds": [
@@ -8670,22 +8677,106 @@ items:
},
"yaxes": [
{
- "format": "percent",
+ "format": "bytes",
"label": null,
"logBase": 1,
- "max": 100,
+ "max": null,
"min": 0,
"show": true
},
{
- "format": "percent",
+ "format": "bytes",
"label": null,
"logBase": 1,
- "max": 100,
+ "max": null,
"min": 0,
"show": true
}
]
+ },
+ {
+ "cacheTimeout": null,
+ "colorBackground": false,
+ "colorValue": false,
+ "colors": [
+ "rgba(50, 172, 45, 0.97)",
+ "rgba(237, 129, 40, 0.89)",
+ "rgba(245, 54, 54, 0.9)"
+ ],
+ "datasource": "$datasource",
+ "format": "percent",
+ "gauge": {
+ "maxValue": 100,
+ "minValue": 0,
+ "show": true,
+ "thresholdLabels": false,
+ "thresholdMarkers": true
+ },
+ "gridPos": {
+
+ },
+ "id": 3,
+ "interval": null,
+ "links": [
+
+ ],
+ "mappingType": 1,
+ "mappingTypes": [
+ {
+ "name": "value to text",
+ "value": 1
+ },
+ {
+ "name": "range to text",
+ "value": 2
+ }
+ ],
+ "maxDataPoints": 100,
+ "nullPointMode": "connected",
+ "nullText": null,
+ "postfix": "",
+ "postfixFontSize": "50%",
+ "prefix": "",
+ "prefixFontSize": "50%",
+ "rangeMaps": [
+ {
+ "from": "null",
+ "text": "N/A",
+ "to": "null"
+ }
+ ],
+ "span": 3,
+ "sparkline": {
+ "fillColor": "rgba(31, 118, 189, 0.18)",
+ "full": false,
+ "lineColor": "rgb(31, 120, 193)",
+ "show": false
+ },
+ "tableColumn": "",
+ "targets": [
+ {
+ "expr": "(\n kubelet_volume_stats_capacity_bytes{cluster=\"$cluster\", job=\"kubelet\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"}\n -\n kubelet_volume_stats_available_bytes{cluster=\"$cluster\", job=\"kubelet\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"}\n)\n/\nkubelet_volume_stats_capacity_bytes{cluster=\"$cluster\", job=\"kubelet\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"}\n* 100\n",
+ "format": "time_series",
+ "intervalFactor": 2,
+ "legendFormat": "",
+ "refId": "A"
+ }
+ ],
+ "thresholds": "80, 90",
+ "title": "Volume Space Usage",
+ "tooltip": {
+ "shared": false
+ },
+ "type": "singlestat",
+ "valueFontSize": "80%",
+ "valueMaps": [
+ {
+ "op": "=",
+ "text": "N/A",
+ "value": "null"
+ }
+ ],
+ "valueName": "current"
}
],
"repeat": null,
@@ -8712,9 +8803,9 @@ items:
"gridPos": {
},
- "id": 3,
+ "id": 4,
"legend": {
- "alignAsTable": false,
+ "alignAsTable": true,
"avg": true,
"current": true,
"max": true,
@@ -8739,16 +8830,23 @@ items:
],
"spaceLength": 10,
- "span": 12,
- "stack": false,
+ "span": 9,
+ "stack": true,
"steppedLine": false,
"targets": [
{
- "expr": "kubelet_volume_stats_inodes_used{cluster=\"$cluster\", job=\"kubelet\", persistentvolumeclaim=\"$volume\"} / kubelet_volume_stats_inodes{cluster=\"$cluster\", job=\"kubelet\", persistentvolumeclaim=\"$volume\"} * 100\n",
+ "expr": "sum without(instance, node) (kubelet_volume_stats_inodes_used{cluster=\"$cluster\", job=\"kubelet\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})\n",
"format": "time_series",
"intervalFactor": 1,
- "legendFormat": "{{ Usage }}",
+ "legendFormat": "Used inodes",
"refId": "A"
+ },
+ {
+ "expr": "(\n sum without(instance, node) (kubelet_volume_stats_inodes{cluster=\"$cluster\", job=\"kubelet\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})\n -\n sum without(instance, node) (kubelet_volume_stats_inodes_used{cluster=\"$cluster\", job=\"kubelet\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})\n)\n",
+ "format": "time_series",
+ "intervalFactor": 1,
+ "legendFormat": " Free inodes",
+ "refId": "B"
}
],
"thresholds": [
@@ -8774,22 +8872,106 @@ items:
},
"yaxes": [
{
- "format": "percent",
+ "format": "none",
"label": null,
"logBase": 1,
- "max": 100,
+ "max": null,
"min": 0,
"show": true
},
{
- "format": "percent",
+ "format": "none",
"label": null,
"logBase": 1,
- "max": 100,
+ "max": null,
"min": 0,
"show": true
}
]
+ },
+ {
+ "cacheTimeout": null,
+ "colorBackground": false,
+ "colorValue": false,
+ "colors": [
+ "rgba(50, 172, 45, 0.97)",
+ "rgba(237, 129, 40, 0.89)",
+ "rgba(245, 54, 54, 0.9)"
+ ],
+ "datasource": "$datasource",
+ "format": "percent",
+ "gauge": {
+ "maxValue": 100,
+ "minValue": 0,
+ "show": true,
+ "thresholdLabels": false,
+ "thresholdMarkers": true
+ },
+ "gridPos": {
+
+ },
+ "id": 5,
+ "interval": null,
+ "links": [
+
+ ],
+ "mappingType": 1,
+ "mappingTypes": [
+ {
+ "name": "value to text",
+ "value": 1
+ },
+ {
+ "name": "range to text",
+ "value": 2
+ }
+ ],
+ "maxDataPoints": 100,
+ "nullPointMode": "connected",
+ "nullText": null,
+ "postfix": "",
+ "postfixFontSize": "50%",
+ "prefix": "",
+ "prefixFontSize": "50%",
+ "rangeMaps": [
+ {
+ "from": "null",
+ "text": "N/A",
+ "to": "null"
+ }
+ ],
+ "span": 3,
+ "sparkline": {
+ "fillColor": "rgba(31, 118, 189, 0.18)",
+ "full": false,
+ "lineColor": "rgb(31, 120, 193)",
+ "show": false
+ },
+ "tableColumn": "",
+ "targets": [
+ {
+ "expr": "kubelet_volume_stats_inodes_used{cluster=\"$cluster\", job=\"kubelet\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"}\n/\nkubelet_volume_stats_inodes{cluster=\"$cluster\", job=\"kubelet\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"}\n* 100\n",
+ "format": "time_series",
+ "intervalFactor": 2,
+ "legendFormat": "",
+ "refId": "A"
+ }
+ ],
+ "thresholds": "80, 90",
+ "title": "Volume inodes Usage",
+ "tooltip": {
+ "shared": false
+ },
+ "type": "singlestat",
+ "valueFontSize": "80%",
+ "valueMaps": [
+ {
+ "op": "=",
+ "text": "N/A",
+ "value": "null"
+ }
+ ],
+ "valueName": "current"
}
],
"repeat": null,
diff --git a/manifests/node-exporter-daemonset.yaml b/manifests/node-exporter-daemonset.yaml
index 56e4b90b879b78416cfd8d5fc9475435c663ac18..f0729b5d7bd0b7ef28f24d30ae1e73ffdf8a45a0 100644
--- a/manifests/node-exporter-daemonset.yaml
+++ b/manifests/node-exporter-daemonset.yaml
@@ -22,7 +22,6 @@ spec:
- --path.rootfs=/host/root
- --collector.filesystem.ignored-mount-points=^/(dev|proc|sys|var/lib/docker/.+)($|/)
- --collector.filesystem.ignored-fs-types=^(autofs|binfmt_misc|cgroup|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|mqueue|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|sysfs|tracefs)$
- - --collector.ntp
image: quay.io/prometheus/node-exporter:v0.17.0
name: node-exporter
resources:
diff --git a/manifests/prometheus-rules.yaml b/manifests/prometheus-rules.yaml
index 3db51cc27926a84b17f407de4fe6b428d4d40ddc..5bdfefe4659af031759b484663823483270894d5 100644
--- a/manifests/prometheus-rules.yaml
+++ b/manifests/prometheus-rules.yaml
@@ -938,7 +938,7 @@ spec:
message: Clock skew detected on node-exporter {{ $labels.namespace }}/{{ $labels.pod
}}. Ensure NTP is configured correctly on this host.
expr: |
- node_ntp_offset_seconds{job="node-exporter"} < -0.03 or node_ntp_offset_seconds{job="node-exporter"} > 0.03
+ abs(node_timex_offset_seconds{job="node-exporter"}) > 0.03
for: 2m
labels:
severity: warning