From 69d9636b64192418d64912c032f5437361e88ea5 Mon Sep 17 00:00:00 2001
From: Ben Kochie <superq@gmail.com>
Date: Thu, 24 Oct 2024 15:59:04 +0200
Subject: [PATCH] Reduce apiserver metric cardinality (#2531)
* Reduce apiserver metric cardinality
Current releases of Kubernetes added new metrics that also have the same
excessive bucket cardinality. Remove the extra buckets from these
metrics.
* `apiserver_request_sli_duration_seconds_bucket`
* `etcd_request_duration_seconds_bucket`
Also reduce the cardinality of apiserver byte histogram metrics.
* `apiserver_request_bdy_size_bytes_bucket`
This reduces API server metrics in our configuration by about 25%.
Signed-off-by: SuperQ <superq@gmail.com>
* Apply suggestions from code review
---------
Signed-off-by: SuperQ <superq@gmail.com>
Co-authored-by: Simon Pasquier <spasquie@redhat.com>
---
.../kube-prometheus/components/k8s-control-plane.libsonnet | 7 ++++++-
.../kubernetesControlPlane-serviceMonitorApiserver.yaml | 7 ++++++-
2 files changed, 12 insertions(+), 2 deletions(-)
diff --git a/jsonnet/kube-prometheus/components/k8s-control-plane.libsonnet b/jsonnet/kube-prometheus/components/k8s-control-plane.libsonnet
index e17c6b2e..448ea4cf 100644
--- a/jsonnet/kube-prometheus/components/k8s-control-plane.libsonnet
+++ b/jsonnet/kube-prometheus/components/k8s-control-plane.libsonnet
@@ -323,7 +323,12 @@ function(params) {
},
{
sourceLabels: ['__name__', 'le'],
- regex: 'apiserver_request_duration_seconds_bucket;(0.15|0.25|0.3|0.35|0.4|0.45|0.6|0.7|0.8|0.9|1.25|1.5|1.75|2.5|3|3.5|4.5|6|7|8|9|15|25|30|50)',
+ regex: '(apiserver_request|apiserver_request_sli|etcd_request)_duration_seconds_bucket;(0.15|0.25|0.3|0.35|0.4|0.45|0.6|0.7|0.8|0.9|1.25|1.5|1.75|2.5|3|3.5|4.5|6|7|8|9|15|25|30|50)',
+ action: 'drop',
+ },
+ {
+ sourceLabels: ['__name__', 'le'],
+ regex: 'apiserver_request_body_size_bytes_bucket;(150000|350000|550000|650000|850000|950000|(1\\.15|1\\.35|1\\.55|1\\.65|1\\.85|1\\.95|2\\.15|2\\.35|2\\.55|2\\.65|2\\.85|2\\.95)e\\+06)',
action: 'drop',
},
],
diff --git a/manifests/kubernetesControlPlane-serviceMonitorApiserver.yaml b/manifests/kubernetesControlPlane-serviceMonitorApiserver.yaml
index 509467af..5bf68e3d 100644
--- a/manifests/kubernetesControlPlane-serviceMonitorApiserver.yaml
+++ b/manifests/kubernetesControlPlane-serviceMonitorApiserver.yaml
@@ -56,7 +56,12 @@ spec:
sourceLabels:
- __name__
- action: drop
- regex: apiserver_request_duration_seconds_bucket;(0.15|0.25|0.3|0.35|0.4|0.45|0.6|0.7|0.8|0.9|1.25|1.5|1.75|2.5|3|3.5|4.5|6|7|8|9|15|25|30|50)
+ regex: (apiserver_request|apiserver_request_sli|etcd_request)_duration_seconds_bucket;(0.15|0.25|0.3|0.35|0.4|0.45|0.6|0.7|0.8|0.9|1.25|1.5|1.75|2.5|3|3.5|4.5|6|7|8|9|15|25|30|50)
+ sourceLabels:
+ - __name__
+ - le
+ - action: drop
+ regex: apiserver_request_body_size_bytes_bucket;(150000|350000|550000|650000|850000|950000|(1\.15|1\.35|1\.55|1\.65|1\.85|1\.95|2\.15|2\.35|2\.55|2\.65|2\.85|2\.95)e\+06)
sourceLabels:
- __name__
- le
--
GitLab