From 145ee24e09cdfb8760775a60c65d28f55c08d17b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Johannes=20W=C3=BCrbach?= <johannes.wuerbach@googlemail.com>
Date: Tue, 7 Apr 2020 22:00:29 +0200
Subject: [PATCH] Convert custom-metrics into an addon
---
README.md | 1 +
...prometheus-rules-and-grafana-dashboards.md | 1 +
example.jsonnet | 1 +
experimental/custom-metrics-api/.gitignore | 7 -
experimental/custom-metrics-api/README.md | 21 --
...-resource-reader-cluster-role-binding.yaml | 12 --
.../custom-metrics-apiservice.yaml | 13 --
.../custom-metrics-cluster-role.yaml | 9 -
.../custom-metrics-configmap.yaml | 98 ----------
experimental/custom-metrics-api/deploy.sh | 7 -
...a-custom-metrics-cluster-role-binding.yaml | 12 --
.../custom-metrics-api/sample-app.yaml | 67 -------
experimental/custom-metrics-api/teardown.sh | 7 -
.../kube-prometheus-custom-metrics.libsonnet | 179 ++++++++++++++++++
14 files changed, 182 insertions(+), 253 deletions(-)
delete mode 100644 experimental/custom-metrics-api/.gitignore
delete mode 100644 experimental/custom-metrics-api/README.md
delete mode 100644 experimental/custom-metrics-api/custom-metrics-apiserver-resource-reader-cluster-role-binding.yaml
delete mode 100644 experimental/custom-metrics-api/custom-metrics-apiservice.yaml
delete mode 100644 experimental/custom-metrics-api/custom-metrics-cluster-role.yaml
delete mode 100644 experimental/custom-metrics-api/custom-metrics-configmap.yaml
delete mode 100644 experimental/custom-metrics-api/deploy.sh
delete mode 100644 experimental/custom-metrics-api/hpa-custom-metrics-cluster-role-binding.yaml
delete mode 100644 experimental/custom-metrics-api/sample-app.yaml
delete mode 100644 experimental/custom-metrics-api/teardown.sh
create mode 100644 jsonnet/kube-prometheus/kube-prometheus-custom-metrics.libsonnet
diff --git a/README.md b/README.md
index 4172e516..c8a99484 100644
--- a/README.md
+++ b/README.md
@@ -198,6 +198,7 @@ local kp =
// (import 'kube-prometheus/kube-prometheus-node-ports.libsonnet') +
// (import 'kube-prometheus/kube-prometheus-static-etcd.libsonnet') +
// (import 'kube-prometheus/kube-prometheus-thanos-sidecar.libsonnet') +
+ // (import 'kube-prometheus/kube-prometheus-custom-metrics.libsonnet') +
{
_config+:: {
namespace: 'monitoring',
diff --git a/docs/developing-prometheus-rules-and-grafana-dashboards.md b/docs/developing-prometheus-rules-and-grafana-dashboards.md
index 974c3cc4..91f430f8 100644
--- a/docs/developing-prometheus-rules-and-grafana-dashboards.md
+++ b/docs/developing-prometheus-rules-and-grafana-dashboards.md
@@ -18,6 +18,7 @@ local kp =
// (import 'kube-prometheus/kube-prometheus-node-ports.libsonnet') +
// (import 'kube-prometheus/kube-prometheus-static-etcd.libsonnet') +
// (import 'kube-prometheus/kube-prometheus-thanos-sidecar.libsonnet') +
+ // (import 'kube-prometheus/kube-prometheus-custom-metrics.libsonnet') +
{
_config+:: {
namespace: 'monitoring',
diff --git a/example.jsonnet b/example.jsonnet
index 77864e09..54de1e35 100644
--- a/example.jsonnet
+++ b/example.jsonnet
@@ -6,6 +6,7 @@ local kp =
// (import 'kube-prometheus/kube-prometheus-node-ports.libsonnet') +
// (import 'kube-prometheus/kube-prometheus-static-etcd.libsonnet') +
// (import 'kube-prometheus/kube-prometheus-thanos-sidecar.libsonnet') +
+ // (import 'kube-prometheus/kube-prometheus-custom-metrics.libsonnet') +
{
_config+:: {
namespace: 'monitoring',
diff --git a/experimental/custom-metrics-api/.gitignore b/experimental/custom-metrics-api/.gitignore
deleted file mode 100644
index 794c008c..00000000
--- a/experimental/custom-metrics-api/.gitignore
+++ /dev/null
@@ -1,7 +0,0 @@
-apiserver-key.pem
-apiserver.csr
-apiserver.pem
-metrics-ca-config.json
-metrics-ca.crt
-metrics-ca.key
-cm-adapter-serving-certs.yaml
diff --git a/experimental/custom-metrics-api/README.md b/experimental/custom-metrics-api/README.md
deleted file mode 100644
index e93d809f..00000000
--- a/experimental/custom-metrics-api/README.md
+++ /dev/null
@@ -1,21 +0,0 @@
-# Custom Metrics API
-
-The custom metrics API allows the HPA v2 to scale based on arbirary metrics.
-
-This directory contains an example deployment which extends the Prometheus Adapter, deployed with kube-prometheus, serve the [Custom Metrics API](https://github.com/kubernetes/community/blob/master/contributors/design-proposals/instrumentation/custom-metrics-api.md) by talking to Prometheus running inside the cluster.
-
-Make sure you have the Prometheus Adapter up and running in the `monitoring` namespace.
-
-You can deploy everything in the `monitoring` namespace using `./deploy.sh`.
-
-When you're done, you can teardown using the `./teardown.sh` script.
-
-### Sample App
-
-Additionally, this directory contains a sample app that uses the [Horizontal Pod Autoscaler](https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/) to scale the Deployment's replicas of Pods up and down as needed.
-Deploy this app by running `kubectl apply -f sample-app.yaml`.
-Make the app accessible on your system, for example by using `kubectl port-forward svc/sample-app 8080`. Next you need to put some load on its http endpoints.
-
-A tool like [hey](https://github.com/rakyll/hey) is helpful for doing so: `hey -c 20 -n 100000000 http://localhost:8080/metrics`
-
-There is an even more detailed information on this sample app at [luxas/kubeadm-workshop](https://github.com/luxas/kubeadm-workshop#deploying-the-prometheus-operator-for-monitoring-services-in-the-cluster).
diff --git a/experimental/custom-metrics-api/custom-metrics-apiserver-resource-reader-cluster-role-binding.yaml b/experimental/custom-metrics-api/custom-metrics-apiserver-resource-reader-cluster-role-binding.yaml
deleted file mode 100644
index e2b1ca43..00000000
--- a/experimental/custom-metrics-api/custom-metrics-apiserver-resource-reader-cluster-role-binding.yaml
+++ /dev/null
@@ -1,12 +0,0 @@
-apiVersion: rbac.authorization.k8s.io/v1beta1
-kind: ClusterRoleBinding
-metadata:
- name: custom-metrics-server-resources
-roleRef:
- apiGroup: rbac.authorization.k8s.io
- kind: ClusterRole
- name: custom-metrics-server-resources
-subjects:
-- kind: ServiceAccount
- name: prometheus-adapter
- namespace: monitoring
diff --git a/experimental/custom-metrics-api/custom-metrics-apiservice.yaml b/experimental/custom-metrics-api/custom-metrics-apiservice.yaml
deleted file mode 100644
index 98f87495..00000000
--- a/experimental/custom-metrics-api/custom-metrics-apiservice.yaml
+++ /dev/null
@@ -1,13 +0,0 @@
-apiVersion: apiregistration.k8s.io/v1beta1
-kind: APIService
-metadata:
- name: v1beta1.custom.metrics.k8s.io
-spec:
- service:
- name: prometheus-adapter
- namespace: monitoring
- group: custom.metrics.k8s.io
- version: v1beta1
- insecureSkipTLSVerify: true
- groupPriorityMinimum: 100
- versionPriority: 100
diff --git a/experimental/custom-metrics-api/custom-metrics-cluster-role.yaml b/experimental/custom-metrics-api/custom-metrics-cluster-role.yaml
deleted file mode 100644
index 003f0bf1..00000000
--- a/experimental/custom-metrics-api/custom-metrics-cluster-role.yaml
+++ /dev/null
@@ -1,9 +0,0 @@
-apiVersion: rbac.authorization.k8s.io/v1beta1
-kind: ClusterRole
-metadata:
- name: custom-metrics-server-resources
-rules:
-- apiGroups:
- - custom.metrics.k8s.io
- resources: ["*"]
- verbs: ["*"]
diff --git a/experimental/custom-metrics-api/custom-metrics-configmap.yaml b/experimental/custom-metrics-api/custom-metrics-configmap.yaml
deleted file mode 100644
index 36908acd..00000000
--- a/experimental/custom-metrics-api/custom-metrics-configmap.yaml
+++ /dev/null
@@ -1,98 +0,0 @@
-apiVersion: v1
-kind: ConfigMap
-metadata:
- name: adapter-config
- namespace: monitoring
-data:
- config.yaml: |
- rules:
- - seriesQuery: '{__name__=~"^container_.*",container!="POD",namespace!="",pod!=""}'
- seriesFilters: []
- resources:
- overrides:
- namespace:
- resource: namespace
- pod:
- resource: pod
- name:
- matches: ^container_(.*)_seconds_total$
- as: ""
- metricsQuery: sum(rate(<<.Series>>{<<.LabelMatchers>>,container!="POD"}[1m])) by (<<.GroupBy>>)
- - seriesQuery: '{__name__=~"^container_.*",container!="POD",namespace!="",pod!=""}'
- seriesFilters:
- - isNot: ^container_.*_seconds_total$
- resources:
- overrides:
- namespace:
- resource: namespace
- pod:
- resource: pod
- name:
- matches: ^container_(.*)_total$
- as: ""
- metricsQuery: sum(rate(<<.Series>>{<<.LabelMatchers>>,container!="POD"}[1m])) by (<<.GroupBy>>)
- - seriesQuery: '{__name__=~"^container_.*",container!="POD",namespace!="",pod!=""}'
- seriesFilters:
- - isNot: ^container_.*_total$
- resources:
- overrides:
- namespace:
- resource: namespace
- pod:
- resource: pod
- name:
- matches: ^container_(.*)$
- as: ""
- metricsQuery: sum(<<.Series>>{<<.LabelMatchers>>,container!="POD"}) by (<<.GroupBy>>)
- - seriesQuery: '{namespace!="",__name__!~"^container_.*"}'
- seriesFilters:
- - isNot: .*_total$
- resources:
- template: <<.Resource>>
- name:
- matches: ""
- as: ""
- metricsQuery: sum(<<.Series>>{<<.LabelMatchers>>}) by (<<.GroupBy>>)
- - seriesQuery: '{namespace!="",__name__!~"^container_.*"}'
- seriesFilters:
- - isNot: .*_seconds_total
- resources:
- template: <<.Resource>>
- name:
- matches: ^(.*)_total$
- as: ""
- metricsQuery: sum(rate(<<.Series>>{<<.LabelMatchers>>}[1m])) by (<<.GroupBy>>)
- - seriesQuery: '{namespace!="",__name__!~"^container_.*"}'
- seriesFilters: []
- resources:
- template: <<.Resource>>
- name:
- matches: ^(.*)_seconds_total$
- as: ""
- metricsQuery: sum(rate(<<.Series>>{<<.LabelMatchers>>}[1m])) by (<<.GroupBy>>)
- resourceRules:
- cpu:
- containerQuery: sum(rate(container_cpu_usage_seconds_total{<<.LabelMatchers>>}[1m])) by (<<.GroupBy>>)
- nodeQuery: sum(rate(container_cpu_usage_seconds_total{<<.LabelMatchers>>, id='/'}[1m])) by (<<.GroupBy>>)
- resources:
- overrides:
- node:
- resource: node
- namespace:
- resource: namespace
- pod:
- resource: pod
- containerLabel: container
- memory:
- containerQuery: sum(container_memory_working_set_bytes{<<.LabelMatchers>>}) by (<<.GroupBy>>)
- nodeQuery: sum(container_memory_working_set_bytes{<<.LabelMatchers>>,id='/'}) by (<<.GroupBy>>)
- resources:
- overrides:
- node:
- resource: node
- namespace:
- resource: namespace
- pod:
- resource: pod
- containerLabel: container
- window: 1m
diff --git a/experimental/custom-metrics-api/deploy.sh b/experimental/custom-metrics-api/deploy.sh
deleted file mode 100644
index d276afc0..00000000
--- a/experimental/custom-metrics-api/deploy.sh
+++ /dev/null
@@ -1,7 +0,0 @@
-#!/usr/bin/env bash
-
-kubectl apply -n monitoring -f custom-metrics-apiserver-resource-reader-cluster-role-binding.yaml
-kubectl apply -n monitoring -f custom-metrics-apiservice.yaml
-kubectl apply -n monitoring -f custom-metrics-cluster-role.yaml
-kubectl apply -n monitoring -f custom-metrics-configmap.yaml
-kubectl apply -n monitoring -f hpa-custom-metrics-cluster-role-binding.yaml
diff --git a/experimental/custom-metrics-api/hpa-custom-metrics-cluster-role-binding.yaml b/experimental/custom-metrics-api/hpa-custom-metrics-cluster-role-binding.yaml
deleted file mode 100644
index 530ebea5..00000000
--- a/experimental/custom-metrics-api/hpa-custom-metrics-cluster-role-binding.yaml
+++ /dev/null
@@ -1,12 +0,0 @@
-apiVersion: rbac.authorization.k8s.io/v1beta1
-kind: ClusterRoleBinding
-metadata:
- name: hpa-controller-custom-metrics
-roleRef:
- apiGroup: rbac.authorization.k8s.io
- kind: ClusterRole
- name: custom-metrics-server-resources
-subjects:
-- kind: ServiceAccount
- name: horizontal-pod-autoscaler
- namespace: kube-system
diff --git a/experimental/custom-metrics-api/sample-app.yaml b/experimental/custom-metrics-api/sample-app.yaml
deleted file mode 100644
index 470887c6..00000000
--- a/experimental/custom-metrics-api/sample-app.yaml
+++ /dev/null
@@ -1,67 +0,0 @@
-kind: ServiceMonitor
-apiVersion: monitoring.coreos.com/v1
-metadata:
- name: sample-app
- labels:
- app: sample-app
-spec:
- selector:
- matchLabels:
- app: sample-app
- endpoints:
- - port: http
- interval: 5s
----
-apiVersion: v1
-kind: Service
-metadata:
- name: sample-app
- labels:
- app: sample-app
-spec:
- ports:
- - name: http
- port: 8080
- targetPort: 8080
- selector:
- app: sample-app
----
-apiVersion: apps/v1
-kind: Deployment
-metadata:
- name: sample-app
- labels:
- app: sample-app
-spec:
- replicas: 1
- selector:
- matchLabels:
- app: sample-app
- template:
- metadata:
- labels:
- app: sample-app
- spec:
- containers:
- - image: luxas/autoscale-demo:v0.1.2
- name: metrics-provider
- ports:
- - name: http
- containerPort: 8080
----
-kind: HorizontalPodAutoscaler
-apiVersion: autoscaling/v2beta1
-metadata:
- name: sample-app
-spec:
- scaleTargetRef:
- apiVersion: apps/v1
- kind: Deployment
- name: sample-app
- minReplicas: 1
- maxReplicas: 10
- metrics:
- - type: Pods
- pods:
- metricName: http_requests
- targetAverageValue: 500m
diff --git a/experimental/custom-metrics-api/teardown.sh b/experimental/custom-metrics-api/teardown.sh
deleted file mode 100644
index b3a455f5..00000000
--- a/experimental/custom-metrics-api/teardown.sh
+++ /dev/null
@@ -1,7 +0,0 @@
-#!/usr/bin/env bash
-
-kubectl delete -n monitoring -f custom-metrics-apiserver-resource-reader-cluster-role-binding.yaml
-kubectl delete -n monitoring -f custom-metrics-apiservice.yaml
-kubectl delete -n monitoring -f custom-metrics-cluster-role.yaml
-kubectl delete -n monitoring -f custom-metrics-configmap.yaml
-kubectl delete -n monitoring -f hpa-custom-metrics-cluster-role-binding.yaml
diff --git a/jsonnet/kube-prometheus/kube-prometheus-custom-metrics.libsonnet b/jsonnet/kube-prometheus/kube-prometheus-custom-metrics.libsonnet
new file mode 100644
index 00000000..479a7ee5
--- /dev/null
+++ b/jsonnet/kube-prometheus/kube-prometheus-custom-metrics.libsonnet
@@ -0,0 +1,179 @@
+local k = import 'ksonnet/ksonnet.beta.4/k.libsonnet';
+
+// Custom metrics API allows the HPA v2 to scale based on arbirary metrics.
+// For more details on usage visit https://github.com/DirectXMan12/k8s-prometheus-adapter#quick-links
+
+{
+ _config+:: {
+ prometheusAdapter+:: {
+ // Rules for custom-metrics
+ config+:: {
+ rules+: [
+ {
+ seriesQuery: '{__name__=~"^container_.*",container!="POD",namespace!="",pod!=""}',
+ seriesFilters: [],
+ resources: {
+ overrides: {
+ namespace: {
+ resource: 'namespace'
+ },
+ pod: {
+ resource: 'pod'
+ }
+ },
+ },
+ name: {
+ matches: '^container_(.*)_seconds_total$',
+ as: ""
+ },
+ metricsQuery: 'sum(rate(<<.Series>>{<<.LabelMatchers>>,container!="POD"}[1m])) by (<<.GroupBy>>)'
+ },
+ {
+ seriesQuery: '{__name__=~"^container_.*",container!="POD",namespace!="",pod!=""}',
+ seriesFilters: [
+ { isNot: '^container_.*_seconds_total$' },
+ ],
+ resources: {
+ overrides: {
+ namespace: {
+ resource: 'namespace'
+ },
+ pod: {
+ resource: 'pod'
+ }
+ },
+ },
+ name: {
+ matches: '^container_(.*)_total$',
+ as: ''
+ },
+ metricsQuery: 'sum(rate(<<.Series>>{<<.LabelMatchers>>,container!="POD"}[1m])) by (<<.GroupBy>>)'
+ },
+ {
+ seriesQuery: '{__name__=~"^container_.*",container!="POD",namespace!="",pod!=""}',
+ seriesFilters: [
+ { isNot: '^container_.*_total$' },
+ ],
+ resources: {
+ overrides: {
+ namespace: {
+ resource: 'namespace'
+ },
+ pod: {
+ resource: 'pod'
+ }
+ },
+ },
+ name: {
+ matches: '^container_(.*)$',
+ as: ''
+ },
+ metricsQuery: 'sum(<<.Series>>{<<.LabelMatchers>>,container!="POD"}) by (<<.GroupBy>>)'
+ },
+ {
+ seriesQuery: '{namespace!="",__name__!~"^container_.*"}',
+ seriesFilters: [
+ { isNot: '.*_total$' },
+ ],
+ resources: {
+ template: '<<.Resource>>'
+ },
+ name: {
+ matches: '',
+ as: ''
+ },
+ metricsQuery: 'sum(<<.Series>>{<<.LabelMatchers>>}) by (<<.GroupBy>>)'
+ },
+ {
+ seriesQuery: '{namespace!="",__name__!~"^container_.*"}',
+ seriesFilters: [
+ { isNot: '.*_seconds_total' },
+ ],
+ resources: {
+ template: '<<.Resource>>'
+ },
+ name: {
+ matches: '^(.*)_total$',
+ as: ''
+ },
+ metricsQuery: 'sum(rate(<<.Series>>{<<.LabelMatchers>>}[1m])) by (<<.GroupBy>>)'
+ },
+ {
+ seriesQuery: '{namespace!="",__name__!~"^container_.*"}',
+ seriesFilters: [],
+ resources: {
+ template: '<<.Resource>>'
+ },
+ name: {
+ matches: '^(.*)_seconds_total$',
+ as: ''
+ },
+ metricsQuery: 'sum(rate(<<.Series>>{<<.LabelMatchers>>}[1m])) by (<<.GroupBy>>)'
+ }
+ ],
+ },
+ },
+ },
+
+ prometheusAdapter+:: {
+ customMetricsApiService: {
+ apiVersion: 'apiregistration.k8s.io/v1',
+ kind: 'APIService',
+ metadata: {
+ name: 'v1beta1.custom.metrics.k8s.io',
+ },
+ spec: {
+ service: {
+ name: $.prometheusAdapter.service.metadata.name,
+ namespace: $._config.namespace,
+ },
+ group: 'custom.metrics.k8s.io',
+ version: 'v1beta1',
+ insecureSkipTLSVerify: true,
+ groupPriorityMinimum: 100,
+ versionPriority: 100,
+ },
+ },
+ customMetricsClusterRoleServerResources:
+ local clusterRole = k.rbac.v1.clusterRole;
+ local policyRule = clusterRole.rulesType;
+
+ local rules =
+ policyRule.new() +
+ policyRule.withApiGroups(['custom.metrics.k8s.io']) +
+ policyRule.withResources(['*']) +
+ policyRule.withVerbs(['*']);
+
+ clusterRole.new() +
+ clusterRole.mixin.metadata.withName('custom-metrics-server-resources') +
+ clusterRole.withRules(rules),
+
+ customMetricsClusterRoleBindingServerResources:
+ local clusterRoleBinding = k.rbac.v1.clusterRoleBinding;
+
+ clusterRoleBinding.new() +
+ clusterRoleBinding.mixin.metadata.withName('custom-metrics-server-resources') +
+ clusterRoleBinding.mixin.roleRef.withApiGroup('rbac.authorization.k8s.io') +
+ clusterRoleBinding.mixin.roleRef.withName('custom-metrics-server-resources') +
+ clusterRoleBinding.mixin.roleRef.mixinInstance({ kind: 'ClusterRole' }) +
+ clusterRoleBinding.withSubjects([{
+ kind: 'ServiceAccount',
+ name: $.prometheusAdapter.serviceAccount.metadata.name,
+ namespace: $._config.namespace,
+ }]),
+
+ customMetricsClusterRoleBindingHPA:
+ local clusterRoleBinding = k.rbac.v1.clusterRoleBinding;
+
+ clusterRoleBinding.new() +
+ clusterRoleBinding.mixin.metadata.withName('hpa-controller-custom-metrics') +
+ clusterRoleBinding.mixin.roleRef.withApiGroup('rbac.authorization.k8s.io') +
+ clusterRoleBinding.mixin.roleRef.withName('custom-metrics-server-resources') +
+ clusterRoleBinding.mixin.roleRef.mixinInstance({ kind: 'ClusterRole' }) +
+ clusterRoleBinding.withSubjects([{
+ kind: 'ServiceAccount',
+ name: 'horizontal-pod-autoscaler',
+ namespace: 'kube-system',
+ }]),
+ }
+}
--
GitLab