diff --git a/README.md b/README.md index 69d402f3d7e059b941bd41c044e479f648736cf6..b035aceb33c7336de336da5de6acb0cdf501ce42 100644 --- a/README.md +++ b/README.md @@ -45,13 +45,16 @@ This stack is meant for cluster monitoring, so it is pre-configured to collect m ## Prerequisites -You will need a Kubernetes cluster, that's it! By default it is assumed, that the kubelet uses token authN and authZ, as otherwise Prometheus needs a client certificate, which gives it full access to the kubelet, rather than just the metrics. Token authN and authZ allows more fine grained and easier access control. +You will need a Kubernetes cluster, that's it! By default it is assumed, that the kubelet uses token authentication and authorization, as otherwise Prometheus needs a client certificate, which gives it full access to the kubelet, rather than just the metrics. Token authentication and authorization allows more fine grained and easier access control. This means the kubelet configuration must contain these flags: * `--authentication-token-webhook=true` This flag enables, that a `ServiceAccount` token can be used to authenticate against the kubelet(s). * `--authorization-mode=Webhook` This flag enables, that the kubelet will perform an RBAC request with the API to determine, whether the requesting entity (Prometheus in this case) is allow to access a resource, in specific for this project the `/metrics` endpoint. +This stack provides [resource metrics](https://github.com/kubernetes/metrics#resource-metrics-api) by deploying the [Prometheus Adapter](https://github.com/DirectXMan12/k8s-prometheus-adapter/). +This adapter is an Extension API Server and Kubernetes needs to be have this feature enabled, otherwise the adapter has no effect, but is still deployed. + ### minikube In order to just try out this stack, start minikube with the following command: @@ -155,6 +158,7 @@ local kp = (import 'kube-prometheus/kube-prometheus.libsonnet') + { { ['kube-state-metrics-' + name]: kp.kubeStateMetrics[name] for name in std.objectFields(kp.kubeStateMetrics) } + { ['alertmanager-' + name]: kp.alertmanager[name] for name in std.objectFields(kp.alertmanager) } + { ['prometheus-' + name]: kp.prometheus[name] for name in std.objectFields(kp.prometheus) } + +{ ['prometheus-adapter-' + name]: kp.prometheusAdapter[name] for name in std.objectFields(kp.prometheusAdapter) } + { ['grafana-' + name]: kp.grafana[name] for name in std.objectFields(kp.grafana) } ``` diff --git a/docs/developing-prometheus-rules-and-grafana-dashboards.md b/docs/developing-prometheus-rules-and-grafana-dashboards.md index 1eb4f15ad8f932009d74f6b39daa329761aa54c9..72deb0e3ebb7c66ff9ee714557efd4e41fce949d 100644 --- a/docs/developing-prometheus-rules-and-grafana-dashboards.md +++ b/docs/developing-prometheus-rules-and-grafana-dashboards.md @@ -22,6 +22,7 @@ local kp = (import 'kube-prometheus/kube-prometheus.libsonnet') + { { ['kube-state-metrics-' + name]: kp.kubeStateMetrics[name] for name in std.objectFields(kp.kubeStateMetrics) } + { ['alertmanager-' + name]: kp.alertmanager[name] for name in std.objectFields(kp.alertmanager) } + { ['prometheus-' + name]: kp.prometheus[name] for name in std.objectFields(kp.prometheus) } + +{ ['prometheus-adapter-' + name]: kp.prometheusAdapter[name] for name in std.objectFields(kp.prometheusAdapter) } + { ['grafana-' + name]: kp.grafana[name] for name in std.objectFields(kp.grafana) } ``` diff --git a/example.jsonnet b/example.jsonnet index 1d36eb1fc147d094849c33069534e08085271b1c..2a10509c18e5087cf16939eaa9e9534932a50008 100644 --- a/example.jsonnet +++ b/example.jsonnet @@ -10,4 +10,5 @@ local kp = (import 'kube-prometheus/kube-prometheus.libsonnet') + { { ['kube-state-metrics-' + name]: kp.kubeStateMetrics[name] for name in std.objectFields(kp.kubeStateMetrics) } + { ['alertmanager-' + name]: kp.alertmanager[name] for name in std.objectFields(kp.alertmanager) } + { ['prometheus-' + name]: kp.prometheus[name] for name in std.objectFields(kp.prometheus) } + +{ ['prometheus-adapter-' + name]: kp.prometheusAdapter[name] for name in std.objectFields(kp.prometheusAdapter) } + { ['grafana-' + name]: kp.grafana[name] for name in std.objectFields(kp.grafana) } diff --git a/jsonnet/kube-prometheus/kube-prometheus.libsonnet b/jsonnet/kube-prometheus/kube-prometheus.libsonnet index 4402ca964127d8d991b688aa7325589bdb9d11bd..2dd32b506a0b48c1aafa1fe9c828bad33e2a7cf4 100644 --- a/jsonnet/kube-prometheus/kube-prometheus.libsonnet +++ b/jsonnet/kube-prometheus/kube-prometheus.libsonnet @@ -7,6 +7,7 @@ local configMapList = k.core.v1.configMapList; (import 'alertmanager/alertmanager.libsonnet') + (import 'prometheus-operator/prometheus-operator.libsonnet') + (import 'prometheus/prometheus.libsonnet') + +(import 'prometheus-adapter/prometheus-adapter.libsonnet') + (import 'kubernetes-mixin/mixin.libsonnet') + (import 'alerts/alerts.libsonnet') + (import 'rules/rules.libsonnet') + { diff --git a/jsonnet/kube-prometheus/prometheus-adapter/prometheus-adapter.libsonnet b/jsonnet/kube-prometheus/prometheus-adapter/prometheus-adapter.libsonnet new file mode 100644 index 0000000000000000000000000000000000000000..177fb197172db4efdab83f51a1f374819332b0d4 --- /dev/null +++ b/jsonnet/kube-prometheus/prometheus-adapter/prometheus-adapter.libsonnet @@ -0,0 +1,198 @@ +local k = import 'ksonnet/ksonnet.beta.3/k.libsonnet'; + +{ + _config+:: { + namespace: 'default', + + versions+:: { + prometheusAdapter: 'v0.3.0', + }, + + imageRepos+:: { + prometheusAdapter: 'quay.io/coreos/k8s-prometheus-adapter-amd64', + }, + + prometheusAdapter+:: { + name: 'prometheus-adapter', + labels: { name: $._config.prometheusAdapter.name }, + config: ||| + resourceRules: + cpu: + containerQuery: sum(rate(container_cpu_usage_seconds_total{<<.LabelMatchers>>}[1m])) by (<<.GroupBy>>) + nodeQuery: sum(rate(container_cpu_usage_seconds_total{<<.LabelMatchers>>, id='/'}[1m])) by (<<.GroupBy>>) + resources: + overrides: + node: + resource: node + namespace: + resource: namespace + pod_name: + resource: pod + containerLabel: container_name + memory: + containerQuery: sum(container_memory_working_set_bytes{<<.LabelMatchers>>}) by (<<.GroupBy>>) + nodeQuery: sum(container_memory_working_set_bytes{<<.LabelMatchers>>,id='/'}) by (<<.GroupBy>>) + resources: + overrides: + node: + resource: node + namespace: + resource: namespace + pod_name: + resource: pod + containerLabel: container_name + window: 1m + |||, + }, + }, + + prometheusAdapter+:: { + apiService: + { + apiVersion: 'apiregistration.k8s.io/v1beta1', + kind: 'APIService', + metadata: { + name: 'v1beta1.metrics.k8s.io', + }, + spec: { + service: { + name: $.prometheusAdapter.service.metadata.name, + namespace: $._config.namespace, + }, + group: 'metrics.k8s.io', + version: 'v1beta1', + insecureSkipTLSVerify: true, + groupPriorityMinimum: 100, + versionPriority: 100, + }, + }, + + configMap: + local configmap = k.core.v1.configMap; + + configmap.new('adapter-config', { 'config.yaml': $._config.prometheusAdapter.config }) + + configmap.mixin.metadata.withNamespace($._config.namespace), + + service: + local service = k.core.v1.service; + local servicePort = k.core.v1.service.mixin.spec.portsType; + + service.new( + $._config.prometheusAdapter.name, + $._config.prometheusAdapter.labels, + servicePort.newNamed('https', 443, 6443), + ) + + service.mixin.metadata.withNamespace($._config.namespace) + + service.mixin.metadata.withLabels($._config.prometheusAdapter.labels), + + deployment: + local deployment = k.apps.v1beta2.deployment; + local volume = deployment.mixin.spec.template.spec.volumesType; + local container = deployment.mixin.spec.template.spec.containersType; + local containerVolumeMount = container.volumeMountsType; + + local c = + container.new($._config.prometheusAdapter.name, $._config.imageRepos.prometheusAdapter + ':' + $._config.versions.prometheusAdapter) + + container.withArgs([ + '--cert-dir=/var/run/serving-cert', + '--config=/etc/adapter/config.yaml', + '--logtostderr=true', + '--metrics-relist-interval=1m', + '--prometheus-url=http://prometheus-' + $._config.prometheus.name + '.' + $._config.namespace + '.svc:9090/', + '--secure-port=6443', + ]) + + container.withPorts([{ containerPort: 6443 }]) + + container.withVolumeMounts([ + containerVolumeMount.new('volume-serving-cert', '/var/run/serving-cert'), + containerVolumeMount.new('config', '/etc/adapter'), + ],); + + deployment.new($._config.prometheusAdapter.name, 1, c, $._config.prometheusAdapter.labels) + + deployment.mixin.metadata.withNamespace($._config.namespace) + + deployment.mixin.spec.selector.withMatchLabels($._config.prometheusAdapter.labels) + + deployment.mixin.spec.template.spec.withServiceAccountName($.prometheusAdapter.serviceAccount.metadata.name) + + deployment.mixin.spec.template.spec.withVolumes([ + // volume.fromSecret('volume-serving-cert', 'cm-adapter-serving-certs'), + volume.fromEmptyDir(name='volume-serving-cert'), + { name: 'config', configMap: { name: 'adapter-config' } }, + ]), + + serviceAccount: + local serviceAccount = k.core.v1.serviceAccount; + + serviceAccount.new($._config.prometheusAdapter.name) + + serviceAccount.mixin.metadata.withNamespace($._config.namespace), + + clusterRole: + local clusterRole = k.rbac.v1.clusterRole; + local policyRule = clusterRole.rulesType; + + local rules = + policyRule.new() + + policyRule.withApiGroups(['']) + + policyRule.withResources(['nodes', 'namespaces', 'pods', 'services']) + + policyRule.withVerbs(['get', 'list', 'watch']); + + clusterRole.new() + + clusterRole.mixin.metadata.withName($._config.prometheusAdapter.name) + + clusterRole.withRules(rules), + + clusterRoleBinding: + local clusterRoleBinding = k.rbac.v1.clusterRoleBinding; + + clusterRoleBinding.new() + + clusterRoleBinding.mixin.metadata.withName($._config.prometheusAdapter.name) + + clusterRoleBinding.mixin.metadata.withNamespace($._config.namespace) + + clusterRoleBinding.mixin.roleRef.withApiGroup('rbac.authorization.k8s.io') + + clusterRoleBinding.mixin.roleRef.withName($.prometheusAdapter.clusterRole.metadata.name) + + clusterRoleBinding.mixin.roleRef.mixinInstance({ kind: 'ClusterRole' }) + + clusterRoleBinding.withSubjects([{ + kind: 'ServiceAccount', + name: $.prometheusAdapter.serviceAccount.metadata.name, + namespace: $._config.namespace, + }]), + + clusterRoleBindingDelegator: + local clusterRoleBinding = k.rbac.v1.clusterRoleBinding; + + clusterRoleBinding.new() + + clusterRoleBinding.mixin.metadata.withName('resource-metrics:system:auth-delegator') + + clusterRoleBinding.mixin.roleRef.withApiGroup('rbac.authorization.k8s.io') + + clusterRoleBinding.mixin.roleRef.withName('system:auth-delegator') + + clusterRoleBinding.mixin.roleRef.mixinInstance({ kind: 'ClusterRole' }) + + clusterRoleBinding.withSubjects([{ + kind: 'ServiceAccount', + name: $.prometheusAdapter.serviceAccount.metadata.name, + namespace: $._config.namespace, + }]), + + clusterRoleServerResources: + local clusterRole = k.rbac.v1.clusterRole; + local policyRule = clusterRole.rulesType; + + local rules = + policyRule.new() + + policyRule.withApiGroups(['metrics.k8s.io']) + + policyRule.withResources(['*']) + + policyRule.withVerbs(['*']); + + clusterRole.new() + + clusterRole.mixin.metadata.withName('resource-metrics-server-resources') + + clusterRole.withRules(rules), + + roleBindingAuthReader: + local roleBinding = k.rbac.v1.roleBinding; + + roleBinding.new() + + roleBinding.mixin.metadata.withName('resource-metrics-auth-reader') + + roleBinding.mixin.metadata.withNamespace('kube-system') + + roleBinding.mixin.roleRef.withApiGroup('rbac.authorization.k8s.io') + + roleBinding.mixin.roleRef.withName('extension-apiserver-authentication-reader') + + roleBinding.mixin.roleRef.mixinInstance({ kind: 'Role' }) + + roleBinding.withSubjects([{ + kind: 'ServiceAccount', + name: $.prometheusAdapter.serviceAccount.metadata.name, + namespace: $._config.namespace, + }]), + }, +} diff --git a/jsonnetfile.lock.json b/jsonnetfile.lock.json index 195e58e036139aa8b4207b9c60e3f8e7bf21daee..e874fc68319f1f612a31ec12f565928933e50b33 100644 --- a/jsonnetfile.lock.json +++ b/jsonnetfile.lock.json @@ -8,7 +8,7 @@ "subdir": "contrib/kube-prometheus/jsonnet/kube-prometheus" } }, - "version": "fa0a0ae33a16a23845da8ab9973dd4eed50a20df" + "version": "d00e8996976492005174b6412a9194421548b247" }, { "name": "ksonnet", diff --git a/manifests/prometheus-adapter-apiService.yaml b/manifests/prometheus-adapter-apiService.yaml new file mode 100644 index 0000000000000000000000000000000000000000..95d5c32dbedb5b39c1aad09d0493f7516de6acda --- /dev/null +++ b/manifests/prometheus-adapter-apiService.yaml @@ -0,0 +1,13 @@ +apiVersion: apiregistration.k8s.io/v1beta1 +kind: APIService +metadata: + name: v1beta1.metrics.k8s.io +spec: + group: metrics.k8s.io + groupPriorityMinimum: 100 + insecureSkipTLSVerify: true + service: + name: prometheus-adapter + namespace: monitoring + version: v1beta1 + versionPriority: 100 diff --git a/manifests/prometheus-adapter-clusterRole.yaml b/manifests/prometheus-adapter-clusterRole.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a02d2bb050365e23d178dca0bd580ed6f6c94584 --- /dev/null +++ b/manifests/prometheus-adapter-clusterRole.yaml @@ -0,0 +1,16 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: prometheus-adapter +rules: +- apiGroups: + - "" + resources: + - nodes + - namespaces + - pods + - services + verbs: + - get + - list + - watch diff --git a/manifests/prometheus-adapter-clusterRoleBinding.yaml b/manifests/prometheus-adapter-clusterRoleBinding.yaml new file mode 100644 index 0000000000000000000000000000000000000000..29fa917642efd7515d30a9428d3ea0eb821cb65d --- /dev/null +++ b/manifests/prometheus-adapter-clusterRoleBinding.yaml @@ -0,0 +1,13 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: prometheus-adapter + namespace: monitoring +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: prometheus-adapter +subjects: +- kind: ServiceAccount + name: prometheus-adapter + namespace: monitoring diff --git a/manifests/prometheus-adapter-clusterRoleBindingDelegator.yaml b/manifests/prometheus-adapter-clusterRoleBindingDelegator.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4295b50f008f4a7ce5841445ab71f150dc8c4334 --- /dev/null +++ b/manifests/prometheus-adapter-clusterRoleBindingDelegator.yaml @@ -0,0 +1,12 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: resource-metrics:system:auth-delegator +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:auth-delegator +subjects: +- kind: ServiceAccount + name: prometheus-adapter + namespace: monitoring diff --git a/manifests/prometheus-adapter-clusterRoleServerResources.yaml b/manifests/prometheus-adapter-clusterRoleServerResources.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fcb914c364dbdef9bf57defdccfe2f0ead709eca --- /dev/null +++ b/manifests/prometheus-adapter-clusterRoleServerResources.yaml @@ -0,0 +1,11 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: resource-metrics-server-resources +rules: +- apiGroups: + - metrics.k8s.io + resources: + - '*' + verbs: + - '*' diff --git a/manifests/prometheus-adapter-configMap.yaml b/manifests/prometheus-adapter-configMap.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a231de36ad87f90ea78e03c3917540698cdf7890 --- /dev/null +++ b/manifests/prometheus-adapter-configMap.yaml @@ -0,0 +1,33 @@ +apiVersion: v1 +data: + config.yaml: | + resourceRules: + cpu: + containerQuery: sum(rate(container_cpu_usage_seconds_total{<<.LabelMatchers>>}[1m])) by (<<.GroupBy>>) + nodeQuery: sum(rate(container_cpu_usage_seconds_total{<<.LabelMatchers>>, id='/'}[1m])) by (<<.GroupBy>>) + resources: + overrides: + node: + resource: node + namespace: + resource: namespace + pod_name: + resource: pod + containerLabel: container_name + memory: + containerQuery: sum(container_memory_working_set_bytes{<<.LabelMatchers>>}) by (<<.GroupBy>>) + nodeQuery: sum(container_memory_working_set_bytes{<<.LabelMatchers>>,id='/'}) by (<<.GroupBy>>) + resources: + overrides: + node: + resource: node + namespace: + resource: namespace + pod_name: + resource: pod + containerLabel: container_name + window: 1m +kind: ConfigMap +metadata: + name: adapter-config + namespace: monitoring diff --git a/manifests/prometheus-adapter-deployment.yaml b/manifests/prometheus-adapter-deployment.yaml new file mode 100644 index 0000000000000000000000000000000000000000..633604999cbc3ee00f736d24b5c35bfc72ad1ef6 --- /dev/null +++ b/manifests/prometheus-adapter-deployment.yaml @@ -0,0 +1,41 @@ +apiVersion: apps/v1beta2 +kind: Deployment +metadata: + name: prometheus-adapter + namespace: monitoring +spec: + replicas: 1 + selector: + matchLabels: + name: prometheus-adapter + template: + metadata: + labels: + name: prometheus-adapter + spec: + containers: + - args: + - --cert-dir=/var/run/serving-cert + - --config=/etc/adapter/config.yaml + - --logtostderr=true + - --metrics-relist-interval=1m + - --prometheus-url=http://prometheus-k8s.monitoring.svc:9090/ + - --secure-port=6443 + image: directxman12/k8s-prometheus-adapter-amd64:latest + name: prometheus-adapter + ports: + - containerPort: 6443 + volumeMounts: + - mountPath: /var/run/serving-cert + name: volume-serving-cert + readOnly: false + - mountPath: /etc/adapter + name: config + readOnly: false + serviceAccountName: prometheus-adapter + volumes: + - emptyDir: {} + name: volume-serving-cert + - configMap: + name: adapter-config + name: config diff --git a/manifests/prometheus-adapter-roleBindingAuthReader.yaml b/manifests/prometheus-adapter-roleBindingAuthReader.yaml new file mode 100644 index 0000000000000000000000000000000000000000..48c8f3253d484a15d96df4fc1265e03e6c273390 --- /dev/null +++ b/manifests/prometheus-adapter-roleBindingAuthReader.yaml @@ -0,0 +1,13 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: resource-metrics-auth-reader + namespace: kube-system +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: extension-apiserver-authentication-reader +subjects: +- kind: ServiceAccount + name: prometheus-adapter + namespace: monitoring diff --git a/manifests/prometheus-adapter-service.yaml b/manifests/prometheus-adapter-service.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e786e01c4aff3ce0e9ca3c3266b1dda817ca8935 --- /dev/null +++ b/manifests/prometheus-adapter-service.yaml @@ -0,0 +1,14 @@ +apiVersion: v1 +kind: Service +metadata: + labels: + name: prometheus-adapter + name: prometheus-adapter + namespace: monitoring +spec: + ports: + - name: https + port: 443 + targetPort: 6443 + selector: + name: prometheus-adapter diff --git a/manifests/prometheus-adapter-serviceAccount.yaml b/manifests/prometheus-adapter-serviceAccount.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d7e7050391eecfc48a05e98fcd95d5b50f88a18e --- /dev/null +++ b/manifests/prometheus-adapter-serviceAccount.yaml @@ -0,0 +1,5 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: prometheus-adapter + namespace: monitoring