From 601fea2e9a9d132ae25a5e6df5bce752b00624de Mon Sep 17 00:00:00 2001
From: Matthias Loibl <mail@matthiasloibl.com>
Date: Thu, 1 Nov 2018 18:59:22 +0100
Subject: [PATCH] contrib/kube-prometheus: Create prometheus-adapter component
contrib/kube-prometheus: Add section on Prometheus Adapter requirements to
README.md
---
README.md | 6 +-
...prometheus-rules-and-grafana-dashboards.md | 1 +
example.jsonnet | 1 +
.../kube-prometheus/kube-prometheus.libsonnet | 1 +
.../prometheus-adapter.libsonnet | 198 ++++++++++++++++++
jsonnetfile.lock.json | 2 +-
manifests/prometheus-adapter-apiService.yaml | 13 ++
manifests/prometheus-adapter-clusterRole.yaml | 16 ++
...prometheus-adapter-clusterRoleBinding.yaml | 13 ++
...s-adapter-clusterRoleBindingDelegator.yaml | 12 ++
...us-adapter-clusterRoleServerResources.yaml | 11 +
manifests/prometheus-adapter-configMap.yaml | 33 +++
manifests/prometheus-adapter-deployment.yaml | 41 ++++
...metheus-adapter-roleBindingAuthReader.yaml | 13 ++
manifests/prometheus-adapter-service.yaml | 14 ++
.../prometheus-adapter-serviceAccount.yaml | 5 +
16 files changed, 378 insertions(+), 2 deletions(-)
create mode 100644 jsonnet/kube-prometheus/prometheus-adapter/prometheus-adapter.libsonnet
create mode 100644 manifests/prometheus-adapter-apiService.yaml
create mode 100644 manifests/prometheus-adapter-clusterRole.yaml
create mode 100644 manifests/prometheus-adapter-clusterRoleBinding.yaml
create mode 100644 manifests/prometheus-adapter-clusterRoleBindingDelegator.yaml
create mode 100644 manifests/prometheus-adapter-clusterRoleServerResources.yaml
create mode 100644 manifests/prometheus-adapter-configMap.yaml
create mode 100644 manifests/prometheus-adapter-deployment.yaml
create mode 100644 manifests/prometheus-adapter-roleBindingAuthReader.yaml
create mode 100644 manifests/prometheus-adapter-service.yaml
create mode 100644 manifests/prometheus-adapter-serviceAccount.yaml
diff --git a/README.md b/README.md
index 69d402f3..b035aceb 100644
--- a/README.md
+++ b/README.md
@@ -45,13 +45,16 @@ This stack is meant for cluster monitoring, so it is pre-configured to collect m
## Prerequisites
-You will need a Kubernetes cluster, that's it! By default it is assumed, that the kubelet uses token authN and authZ, as otherwise Prometheus needs a client certificate, which gives it full access to the kubelet, rather than just the metrics. Token authN and authZ allows more fine grained and easier access control.
+You will need a Kubernetes cluster, that's it! By default it is assumed, that the kubelet uses token authentication and authorization, as otherwise Prometheus needs a client certificate, which gives it full access to the kubelet, rather than just the metrics. Token authentication and authorization allows more fine grained and easier access control.
This means the kubelet configuration must contain these flags:
* `--authentication-token-webhook=true` This flag enables, that a `ServiceAccount` token can be used to authenticate against the kubelet(s).
* `--authorization-mode=Webhook` This flag enables, that the kubelet will perform an RBAC request with the API to determine, whether the requesting entity (Prometheus in this case) is allow to access a resource, in specific for this project the `/metrics` endpoint.
+This stack provides [resource metrics](https://github.com/kubernetes/metrics#resource-metrics-api) by deploying the [Prometheus Adapter](https://github.com/DirectXMan12/k8s-prometheus-adapter/).
+This adapter is an Extension API Server and Kubernetes needs to be have this feature enabled, otherwise the adapter has no effect, but is still deployed.
+
### minikube
In order to just try out this stack, start minikube with the following command:
@@ -155,6 +158,7 @@ local kp = (import 'kube-prometheus/kube-prometheus.libsonnet') + {
{ ['kube-state-metrics-' + name]: kp.kubeStateMetrics[name] for name in std.objectFields(kp.kubeStateMetrics) } +
{ ['alertmanager-' + name]: kp.alertmanager[name] for name in std.objectFields(kp.alertmanager) } +
{ ['prometheus-' + name]: kp.prometheus[name] for name in std.objectFields(kp.prometheus) } +
+{ ['prometheus-adapter-' + name]: kp.prometheusAdapter[name] for name in std.objectFields(kp.prometheusAdapter) } +
{ ['grafana-' + name]: kp.grafana[name] for name in std.objectFields(kp.grafana) }
```
diff --git a/docs/developing-prometheus-rules-and-grafana-dashboards.md b/docs/developing-prometheus-rules-and-grafana-dashboards.md
index 1eb4f15a..72deb0e3 100644
--- a/docs/developing-prometheus-rules-and-grafana-dashboards.md
+++ b/docs/developing-prometheus-rules-and-grafana-dashboards.md
@@ -22,6 +22,7 @@ local kp = (import 'kube-prometheus/kube-prometheus.libsonnet') + {
{ ['kube-state-metrics-' + name]: kp.kubeStateMetrics[name] for name in std.objectFields(kp.kubeStateMetrics) } +
{ ['alertmanager-' + name]: kp.alertmanager[name] for name in std.objectFields(kp.alertmanager) } +
{ ['prometheus-' + name]: kp.prometheus[name] for name in std.objectFields(kp.prometheus) } +
+{ ['prometheus-adapter-' + name]: kp.prometheusAdapter[name] for name in std.objectFields(kp.prometheusAdapter) } +
{ ['grafana-' + name]: kp.grafana[name] for name in std.objectFields(kp.grafana) }
```
diff --git a/example.jsonnet b/example.jsonnet
index 1d36eb1f..2a10509c 100644
--- a/example.jsonnet
+++ b/example.jsonnet
@@ -10,4 +10,5 @@ local kp = (import 'kube-prometheus/kube-prometheus.libsonnet') + {
{ ['kube-state-metrics-' + name]: kp.kubeStateMetrics[name] for name in std.objectFields(kp.kubeStateMetrics) } +
{ ['alertmanager-' + name]: kp.alertmanager[name] for name in std.objectFields(kp.alertmanager) } +
{ ['prometheus-' + name]: kp.prometheus[name] for name in std.objectFields(kp.prometheus) } +
+{ ['prometheus-adapter-' + name]: kp.prometheusAdapter[name] for name in std.objectFields(kp.prometheusAdapter) } +
{ ['grafana-' + name]: kp.grafana[name] for name in std.objectFields(kp.grafana) }
diff --git a/jsonnet/kube-prometheus/kube-prometheus.libsonnet b/jsonnet/kube-prometheus/kube-prometheus.libsonnet
index 4402ca96..2dd32b50 100644
--- a/jsonnet/kube-prometheus/kube-prometheus.libsonnet
+++ b/jsonnet/kube-prometheus/kube-prometheus.libsonnet
@@ -7,6 +7,7 @@ local configMapList = k.core.v1.configMapList;
(import 'alertmanager/alertmanager.libsonnet') +
(import 'prometheus-operator/prometheus-operator.libsonnet') +
(import 'prometheus/prometheus.libsonnet') +
+(import 'prometheus-adapter/prometheus-adapter.libsonnet') +
(import 'kubernetes-mixin/mixin.libsonnet') +
(import 'alerts/alerts.libsonnet') +
(import 'rules/rules.libsonnet') + {
diff --git a/jsonnet/kube-prometheus/prometheus-adapter/prometheus-adapter.libsonnet b/jsonnet/kube-prometheus/prometheus-adapter/prometheus-adapter.libsonnet
new file mode 100644
index 00000000..177fb197
--- /dev/null
+++ b/jsonnet/kube-prometheus/prometheus-adapter/prometheus-adapter.libsonnet
@@ -0,0 +1,198 @@
+local k = import 'ksonnet/ksonnet.beta.3/k.libsonnet';
+
+{
+ _config+:: {
+ namespace: 'default',
+
+ versions+:: {
+ prometheusAdapter: 'v0.3.0',
+ },
+
+ imageRepos+:: {
+ prometheusAdapter: 'quay.io/coreos/k8s-prometheus-adapter-amd64',
+ },
+
+ prometheusAdapter+:: {
+ name: 'prometheus-adapter',
+ labels: { name: $._config.prometheusAdapter.name },
+ config: |||
+ resourceRules:
+ cpu:
+ containerQuery: sum(rate(container_cpu_usage_seconds_total{<<.LabelMatchers>>}[1m])) by (<<.GroupBy>>)
+ nodeQuery: sum(rate(container_cpu_usage_seconds_total{<<.LabelMatchers>>, id='/'}[1m])) by (<<.GroupBy>>)
+ resources:
+ overrides:
+ node:
+ resource: node
+ namespace:
+ resource: namespace
+ pod_name:
+ resource: pod
+ containerLabel: container_name
+ memory:
+ containerQuery: sum(container_memory_working_set_bytes{<<.LabelMatchers>>}) by (<<.GroupBy>>)
+ nodeQuery: sum(container_memory_working_set_bytes{<<.LabelMatchers>>,id='/'}) by (<<.GroupBy>>)
+ resources:
+ overrides:
+ node:
+ resource: node
+ namespace:
+ resource: namespace
+ pod_name:
+ resource: pod
+ containerLabel: container_name
+ window: 1m
+ |||,
+ },
+ },
+
+ prometheusAdapter+:: {
+ apiService:
+ {
+ apiVersion: 'apiregistration.k8s.io/v1beta1',
+ kind: 'APIService',
+ metadata: {
+ name: 'v1beta1.metrics.k8s.io',
+ },
+ spec: {
+ service: {
+ name: $.prometheusAdapter.service.metadata.name,
+ namespace: $._config.namespace,
+ },
+ group: 'metrics.k8s.io',
+ version: 'v1beta1',
+ insecureSkipTLSVerify: true,
+ groupPriorityMinimum: 100,
+ versionPriority: 100,
+ },
+ },
+
+ configMap:
+ local configmap = k.core.v1.configMap;
+
+ configmap.new('adapter-config', { 'config.yaml': $._config.prometheusAdapter.config }) +
+ configmap.mixin.metadata.withNamespace($._config.namespace),
+
+ service:
+ local service = k.core.v1.service;
+ local servicePort = k.core.v1.service.mixin.spec.portsType;
+
+ service.new(
+ $._config.prometheusAdapter.name,
+ $._config.prometheusAdapter.labels,
+ servicePort.newNamed('https', 443, 6443),
+ ) +
+ service.mixin.metadata.withNamespace($._config.namespace) +
+ service.mixin.metadata.withLabels($._config.prometheusAdapter.labels),
+
+ deployment:
+ local deployment = k.apps.v1beta2.deployment;
+ local volume = deployment.mixin.spec.template.spec.volumesType;
+ local container = deployment.mixin.spec.template.spec.containersType;
+ local containerVolumeMount = container.volumeMountsType;
+
+ local c =
+ container.new($._config.prometheusAdapter.name, $._config.imageRepos.prometheusAdapter + ':' + $._config.versions.prometheusAdapter) +
+ container.withArgs([
+ '--cert-dir=/var/run/serving-cert',
+ '--config=/etc/adapter/config.yaml',
+ '--logtostderr=true',
+ '--metrics-relist-interval=1m',
+ '--prometheus-url=http://prometheus-' + $._config.prometheus.name + '.' + $._config.namespace + '.svc:9090/',
+ '--secure-port=6443',
+ ]) +
+ container.withPorts([{ containerPort: 6443 }]) +
+ container.withVolumeMounts([
+ containerVolumeMount.new('volume-serving-cert', '/var/run/serving-cert'),
+ containerVolumeMount.new('config', '/etc/adapter'),
+ ],);
+
+ deployment.new($._config.prometheusAdapter.name, 1, c, $._config.prometheusAdapter.labels) +
+ deployment.mixin.metadata.withNamespace($._config.namespace) +
+ deployment.mixin.spec.selector.withMatchLabels($._config.prometheusAdapter.labels) +
+ deployment.mixin.spec.template.spec.withServiceAccountName($.prometheusAdapter.serviceAccount.metadata.name) +
+ deployment.mixin.spec.template.spec.withVolumes([
+ // volume.fromSecret('volume-serving-cert', 'cm-adapter-serving-certs'),
+ volume.fromEmptyDir(name='volume-serving-cert'),
+ { name: 'config', configMap: { name: 'adapter-config' } },
+ ]),
+
+ serviceAccount:
+ local serviceAccount = k.core.v1.serviceAccount;
+
+ serviceAccount.new($._config.prometheusAdapter.name) +
+ serviceAccount.mixin.metadata.withNamespace($._config.namespace),
+
+ clusterRole:
+ local clusterRole = k.rbac.v1.clusterRole;
+ local policyRule = clusterRole.rulesType;
+
+ local rules =
+ policyRule.new() +
+ policyRule.withApiGroups(['']) +
+ policyRule.withResources(['nodes', 'namespaces', 'pods', 'services']) +
+ policyRule.withVerbs(['get', 'list', 'watch']);
+
+ clusterRole.new() +
+ clusterRole.mixin.metadata.withName($._config.prometheusAdapter.name) +
+ clusterRole.withRules(rules),
+
+ clusterRoleBinding:
+ local clusterRoleBinding = k.rbac.v1.clusterRoleBinding;
+
+ clusterRoleBinding.new() +
+ clusterRoleBinding.mixin.metadata.withName($._config.prometheusAdapter.name) +
+ clusterRoleBinding.mixin.metadata.withNamespace($._config.namespace) +
+ clusterRoleBinding.mixin.roleRef.withApiGroup('rbac.authorization.k8s.io') +
+ clusterRoleBinding.mixin.roleRef.withName($.prometheusAdapter.clusterRole.metadata.name) +
+ clusterRoleBinding.mixin.roleRef.mixinInstance({ kind: 'ClusterRole' }) +
+ clusterRoleBinding.withSubjects([{
+ kind: 'ServiceAccount',
+ name: $.prometheusAdapter.serviceAccount.metadata.name,
+ namespace: $._config.namespace,
+ }]),
+
+ clusterRoleBindingDelegator:
+ local clusterRoleBinding = k.rbac.v1.clusterRoleBinding;
+
+ clusterRoleBinding.new() +
+ clusterRoleBinding.mixin.metadata.withName('resource-metrics:system:auth-delegator') +
+ clusterRoleBinding.mixin.roleRef.withApiGroup('rbac.authorization.k8s.io') +
+ clusterRoleBinding.mixin.roleRef.withName('system:auth-delegator') +
+ clusterRoleBinding.mixin.roleRef.mixinInstance({ kind: 'ClusterRole' }) +
+ clusterRoleBinding.withSubjects([{
+ kind: 'ServiceAccount',
+ name: $.prometheusAdapter.serviceAccount.metadata.name,
+ namespace: $._config.namespace,
+ }]),
+
+ clusterRoleServerResources:
+ local clusterRole = k.rbac.v1.clusterRole;
+ local policyRule = clusterRole.rulesType;
+
+ local rules =
+ policyRule.new() +
+ policyRule.withApiGroups(['metrics.k8s.io']) +
+ policyRule.withResources(['*']) +
+ policyRule.withVerbs(['*']);
+
+ clusterRole.new() +
+ clusterRole.mixin.metadata.withName('resource-metrics-server-resources') +
+ clusterRole.withRules(rules),
+
+ roleBindingAuthReader:
+ local roleBinding = k.rbac.v1.roleBinding;
+
+ roleBinding.new() +
+ roleBinding.mixin.metadata.withName('resource-metrics-auth-reader') +
+ roleBinding.mixin.metadata.withNamespace('kube-system') +
+ roleBinding.mixin.roleRef.withApiGroup('rbac.authorization.k8s.io') +
+ roleBinding.mixin.roleRef.withName('extension-apiserver-authentication-reader') +
+ roleBinding.mixin.roleRef.mixinInstance({ kind: 'Role' }) +
+ roleBinding.withSubjects([{
+ kind: 'ServiceAccount',
+ name: $.prometheusAdapter.serviceAccount.metadata.name,
+ namespace: $._config.namespace,
+ }]),
+ },
+}
diff --git a/jsonnetfile.lock.json b/jsonnetfile.lock.json
index 195e58e0..e874fc68 100644
--- a/jsonnetfile.lock.json
+++ b/jsonnetfile.lock.json
@@ -8,7 +8,7 @@
"subdir": "contrib/kube-prometheus/jsonnet/kube-prometheus"
}
},
- "version": "fa0a0ae33a16a23845da8ab9973dd4eed50a20df"
+ "version": "d00e8996976492005174b6412a9194421548b247"
},
{
"name": "ksonnet",
diff --git a/manifests/prometheus-adapter-apiService.yaml b/manifests/prometheus-adapter-apiService.yaml
new file mode 100644
index 00000000..95d5c32d
--- /dev/null
+++ b/manifests/prometheus-adapter-apiService.yaml
@@ -0,0 +1,13 @@
+apiVersion: apiregistration.k8s.io/v1beta1
+kind: APIService
+metadata:
+ name: v1beta1.metrics.k8s.io
+spec:
+ group: metrics.k8s.io
+ groupPriorityMinimum: 100
+ insecureSkipTLSVerify: true
+ service:
+ name: prometheus-adapter
+ namespace: monitoring
+ version: v1beta1
+ versionPriority: 100
diff --git a/manifests/prometheus-adapter-clusterRole.yaml b/manifests/prometheus-adapter-clusterRole.yaml
new file mode 100644
index 00000000..a02d2bb0
--- /dev/null
+++ b/manifests/prometheus-adapter-clusterRole.yaml
@@ -0,0 +1,16 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+ name: prometheus-adapter
+rules:
+- apiGroups:
+ - ""
+ resources:
+ - nodes
+ - namespaces
+ - pods
+ - services
+ verbs:
+ - get
+ - list
+ - watch
diff --git a/manifests/prometheus-adapter-clusterRoleBinding.yaml b/manifests/prometheus-adapter-clusterRoleBinding.yaml
new file mode 100644
index 00000000..29fa9176
--- /dev/null
+++ b/manifests/prometheus-adapter-clusterRoleBinding.yaml
@@ -0,0 +1,13 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+ name: prometheus-adapter
+ namespace: monitoring
+roleRef:
+ apiGroup: rbac.authorization.k8s.io
+ kind: ClusterRole
+ name: prometheus-adapter
+subjects:
+- kind: ServiceAccount
+ name: prometheus-adapter
+ namespace: monitoring
diff --git a/manifests/prometheus-adapter-clusterRoleBindingDelegator.yaml b/manifests/prometheus-adapter-clusterRoleBindingDelegator.yaml
new file mode 100644
index 00000000..4295b50f
--- /dev/null
+++ b/manifests/prometheus-adapter-clusterRoleBindingDelegator.yaml
@@ -0,0 +1,12 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+ name: resource-metrics:system:auth-delegator
+roleRef:
+ apiGroup: rbac.authorization.k8s.io
+ kind: ClusterRole
+ name: system:auth-delegator
+subjects:
+- kind: ServiceAccount
+ name: prometheus-adapter
+ namespace: monitoring
diff --git a/manifests/prometheus-adapter-clusterRoleServerResources.yaml b/manifests/prometheus-adapter-clusterRoleServerResources.yaml
new file mode 100644
index 00000000..fcb914c3
--- /dev/null
+++ b/manifests/prometheus-adapter-clusterRoleServerResources.yaml
@@ -0,0 +1,11 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+ name: resource-metrics-server-resources
+rules:
+- apiGroups:
+ - metrics.k8s.io
+ resources:
+ - '*'
+ verbs:
+ - '*'
diff --git a/manifests/prometheus-adapter-configMap.yaml b/manifests/prometheus-adapter-configMap.yaml
new file mode 100644
index 00000000..a231de36
--- /dev/null
+++ b/manifests/prometheus-adapter-configMap.yaml
@@ -0,0 +1,33 @@
+apiVersion: v1
+data:
+ config.yaml: |
+ resourceRules:
+ cpu:
+ containerQuery: sum(rate(container_cpu_usage_seconds_total{<<.LabelMatchers>>}[1m])) by (<<.GroupBy>>)
+ nodeQuery: sum(rate(container_cpu_usage_seconds_total{<<.LabelMatchers>>, id='/'}[1m])) by (<<.GroupBy>>)
+ resources:
+ overrides:
+ node:
+ resource: node
+ namespace:
+ resource: namespace
+ pod_name:
+ resource: pod
+ containerLabel: container_name
+ memory:
+ containerQuery: sum(container_memory_working_set_bytes{<<.LabelMatchers>>}) by (<<.GroupBy>>)
+ nodeQuery: sum(container_memory_working_set_bytes{<<.LabelMatchers>>,id='/'}) by (<<.GroupBy>>)
+ resources:
+ overrides:
+ node:
+ resource: node
+ namespace:
+ resource: namespace
+ pod_name:
+ resource: pod
+ containerLabel: container_name
+ window: 1m
+kind: ConfigMap
+metadata:
+ name: adapter-config
+ namespace: monitoring
diff --git a/manifests/prometheus-adapter-deployment.yaml b/manifests/prometheus-adapter-deployment.yaml
new file mode 100644
index 00000000..63360499
--- /dev/null
+++ b/manifests/prometheus-adapter-deployment.yaml
@@ -0,0 +1,41 @@
+apiVersion: apps/v1beta2
+kind: Deployment
+metadata:
+ name: prometheus-adapter
+ namespace: monitoring
+spec:
+ replicas: 1
+ selector:
+ matchLabels:
+ name: prometheus-adapter
+ template:
+ metadata:
+ labels:
+ name: prometheus-adapter
+ spec:
+ containers:
+ - args:
+ - --cert-dir=/var/run/serving-cert
+ - --config=/etc/adapter/config.yaml
+ - --logtostderr=true
+ - --metrics-relist-interval=1m
+ - --prometheus-url=http://prometheus-k8s.monitoring.svc:9090/
+ - --secure-port=6443
+ image: directxman12/k8s-prometheus-adapter-amd64:latest
+ name: prometheus-adapter
+ ports:
+ - containerPort: 6443
+ volumeMounts:
+ - mountPath: /var/run/serving-cert
+ name: volume-serving-cert
+ readOnly: false
+ - mountPath: /etc/adapter
+ name: config
+ readOnly: false
+ serviceAccountName: prometheus-adapter
+ volumes:
+ - emptyDir: {}
+ name: volume-serving-cert
+ - configMap:
+ name: adapter-config
+ name: config
diff --git a/manifests/prometheus-adapter-roleBindingAuthReader.yaml b/manifests/prometheus-adapter-roleBindingAuthReader.yaml
new file mode 100644
index 00000000..48c8f325
--- /dev/null
+++ b/manifests/prometheus-adapter-roleBindingAuthReader.yaml
@@ -0,0 +1,13 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+ name: resource-metrics-auth-reader
+ namespace: kube-system
+roleRef:
+ apiGroup: rbac.authorization.k8s.io
+ kind: Role
+ name: extension-apiserver-authentication-reader
+subjects:
+- kind: ServiceAccount
+ name: prometheus-adapter
+ namespace: monitoring
diff --git a/manifests/prometheus-adapter-service.yaml b/manifests/prometheus-adapter-service.yaml
new file mode 100644
index 00000000..e786e01c
--- /dev/null
+++ b/manifests/prometheus-adapter-service.yaml
@@ -0,0 +1,14 @@
+apiVersion: v1
+kind: Service
+metadata:
+ labels:
+ name: prometheus-adapter
+ name: prometheus-adapter
+ namespace: monitoring
+spec:
+ ports:
+ - name: https
+ port: 443
+ targetPort: 6443
+ selector:
+ name: prometheus-adapter
diff --git a/manifests/prometheus-adapter-serviceAccount.yaml b/manifests/prometheus-adapter-serviceAccount.yaml
new file mode 100644
index 00000000..d7e70503
--- /dev/null
+++ b/manifests/prometheus-adapter-serviceAccount.yaml
@@ -0,0 +1,5 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+ name: prometheus-adapter
+ namespace: monitoring
--
GitLab