Convert custom-metrics into an addon

README.md

@@ -198,6 +198,7 @@ local kp =
   // (import 'kube-prometheus/kube-prometheus-node-ports.libsonnet') +
   // (import 'kube-prometheus/kube-prometheus-static-etcd.libsonnet') +
   // (import 'kube-prometheus/kube-prometheus-thanos-sidecar.libsonnet') +
+  // (import 'kube-prometheus/kube-prometheus-custom-metrics.libsonnet') +
   {
     _config+:: {
       namespace: 'monitoring',

docs/developing-prometheus-rules-and-grafana-dashboards.md

@@ -18,6 +18,7 @@ local kp =
   // (import 'kube-prometheus/kube-prometheus-node-ports.libsonnet') +
   // (import 'kube-prometheus/kube-prometheus-static-etcd.libsonnet') +
   // (import 'kube-prometheus/kube-prometheus-thanos-sidecar.libsonnet') +
+  // (import 'kube-prometheus/kube-prometheus-custom-metrics.libsonnet') +
   {
     _config+:: {
       namespace: 'monitoring',

example.jsonnet

@@ -6,6 +6,7 @@ local kp =
   // (import 'kube-prometheus/kube-prometheus-node-ports.libsonnet') +
   // (import 'kube-prometheus/kube-prometheus-static-etcd.libsonnet') +
   // (import 'kube-prometheus/kube-prometheus-thanos-sidecar.libsonnet') +
+  // (import 'kube-prometheus/kube-prometheus-custom-metrics.libsonnet') +
   {
     _config+:: {
       namespace: 'monitoring',

experimental/custom-metrics-api/.gitignore

-apiserver-key.pem
-apiserver.csr
-apiserver.pem
-metrics-ca-config.json
-metrics-ca.crt
-metrics-ca.key
-cm-adapter-serving-certs.yaml

experimental/custom-metrics-api/README.md

-# Custom Metrics API
-
-The custom metrics API allows the HPA v2 to scale based on arbirary metrics.
-
-This directory contains an example deployment which extends the Prometheus Adapter, deployed with kube-prometheus, serve the [Custom Metrics API](https://github.com/kubernetes/community/blob/master/contributors/design-proposals/instrumentation/custom-metrics-api.md) by talking to Prometheus running inside the cluster.
-
-Make sure you have the Prometheus Adapter up and running in the `monitoring` namespace.
-
-You can deploy everything in the `monitoring` namespace using `./deploy.sh`.
-
-When you're done, you can teardown using the `./teardown.sh` script.
-
-### Sample App
-
-Additionally, this directory contains a sample app that uses the [Horizontal Pod Autoscaler](https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/) to scale the Deployment's replicas of Pods up and down as needed.  
-Deploy this app by running `kubectl apply -f sample-app.yaml`. 
-Make the app accessible on your system, for example by using `kubectl port-forward svc/sample-app 8080`. Next you need to put some load on its http endpoints. 
-
-A tool like [hey](https://github.com/rakyll/hey) is helpful for doing so: `hey -c 20 -n 100000000 http://localhost:8080/metrics`
-
-There is an even more detailed information on this sample app at [luxas/kubeadm-workshop](https://github.com/luxas/kubeadm-workshop#deploying-the-prometheus-operator-for-monitoring-services-in-the-cluster).

experimental/custom-metrics-api/custom-metrics-apiserver-resource-reader-cluster-role-binding.yaml

-apiVersion: rbac.authorization.k8s.io/v1beta1
-kind: ClusterRoleBinding
-metadata:
-  name: custom-metrics-server-resources
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: ClusterRole
-  name: custom-metrics-server-resources
-subjects:
-- kind: ServiceAccount
-  name: prometheus-adapter
-  namespace: monitoring

experimental/custom-metrics-api/custom-metrics-apiservice.yaml

-apiVersion: apiregistration.k8s.io/v1beta1
-kind: APIService
-metadata:
-  name: v1beta1.custom.metrics.k8s.io
-spec:
-  service:
-    name: prometheus-adapter
-    namespace: monitoring
-  group: custom.metrics.k8s.io
-  version: v1beta1
-  insecureSkipTLSVerify: true
-  groupPriorityMinimum: 100
-  versionPriority: 100

experimental/custom-metrics-api/custom-metrics-cluster-role.yaml

-apiVersion: rbac.authorization.k8s.io/v1beta1
-kind: ClusterRole
-metadata:
-  name: custom-metrics-server-resources
-rules:
-- apiGroups:
-  - custom.metrics.k8s.io
-  resources: ["*"]
-  verbs: ["*"]

experimental/custom-metrics-api/custom-metrics-configmap.yaml

-apiVersion: v1
-kind: ConfigMap
-metadata:
-  name: adapter-config
-  namespace: monitoring
-data:
-  config.yaml: |
-    rules:
-    - seriesQuery: '{__name__=~"^container_.*",container!="POD",namespace!="",pod!=""}'
-      seriesFilters: []
-      resources:
-        overrides:
-          namespace:
-            resource: namespace
-          pod:
-            resource: pod
-      name:
-        matches: ^container_(.*)_seconds_total$
-        as: ""
-      metricsQuery: sum(rate(<<.Series>>{<<.LabelMatchers>>,container!="POD"}[1m])) by (<<.GroupBy>>)
-    - seriesQuery: '{__name__=~"^container_.*",container!="POD",namespace!="",pod!=""}'
-      seriesFilters:
-      - isNot: ^container_.*_seconds_total$
-      resources:
-        overrides:
-          namespace:
-            resource: namespace
-          pod:
-            resource: pod
-      name:
-        matches: ^container_(.*)_total$
-        as: ""
-      metricsQuery: sum(rate(<<.Series>>{<<.LabelMatchers>>,container!="POD"}[1m])) by (<<.GroupBy>>)
-    - seriesQuery: '{__name__=~"^container_.*",container!="POD",namespace!="",pod!=""}'
-      seriesFilters:
-      - isNot: ^container_.*_total$
-      resources:
-        overrides:
-          namespace:
-            resource: namespace
-          pod:
-            resource: pod
-      name:
-        matches: ^container_(.*)$
-        as: ""
-      metricsQuery: sum(<<.Series>>{<<.LabelMatchers>>,container!="POD"}) by (<<.GroupBy>>)
-    - seriesQuery: '{namespace!="",__name__!~"^container_.*"}'
-      seriesFilters:
-      - isNot: .*_total$
-      resources:
-        template: <<.Resource>>
-      name:
-        matches: ""
-        as: ""
-      metricsQuery: sum(<<.Series>>{<<.LabelMatchers>>}) by (<<.GroupBy>>)
-    - seriesQuery: '{namespace!="",__name__!~"^container_.*"}'
-      seriesFilters:
-      - isNot: .*_seconds_total
-      resources:
-        template: <<.Resource>>
-      name:
-        matches: ^(.*)_total$
-        as: ""
-      metricsQuery: sum(rate(<<.Series>>{<<.LabelMatchers>>}[1m])) by (<<.GroupBy>>)
-    - seriesQuery: '{namespace!="",__name__!~"^container_.*"}'
-      seriesFilters: []
-      resources:
-        template: <<.Resource>>
-      name:
-        matches: ^(.*)_seconds_total$
-        as: ""
-      metricsQuery: sum(rate(<<.Series>>{<<.LabelMatchers>>}[1m])) by (<<.GroupBy>>)
-    resourceRules:
-      cpu:
-        containerQuery: sum(rate(container_cpu_usage_seconds_total{<<.LabelMatchers>>}[1m])) by (<<.GroupBy>>)
-        nodeQuery: sum(rate(container_cpu_usage_seconds_total{<<.LabelMatchers>>, id='/'}[1m])) by (<<.GroupBy>>)
-        resources:
-          overrides:
-            node:
-              resource: node
-            namespace:
-              resource: namespace
-            pod:
-              resource: pod
-        containerLabel: container
-      memory:
-        containerQuery: sum(container_memory_working_set_bytes{<<.LabelMatchers>>}) by (<<.GroupBy>>)
-        nodeQuery: sum(container_memory_working_set_bytes{<<.LabelMatchers>>,id='/'}) by (<<.GroupBy>>)
-        resources:
-          overrides:
-            node:
-              resource: node
-            namespace:
-              resource: namespace
-            pod:
-              resource: pod
-        containerLabel: container
-      window: 1m

experimental/custom-metrics-api/deploy.sh

-#!/usr/bin/env bash
-
-kubectl apply -n monitoring -f custom-metrics-apiserver-resource-reader-cluster-role-binding.yaml
-kubectl apply -n monitoring -f custom-metrics-apiservice.yaml
-kubectl apply -n monitoring -f custom-metrics-cluster-role.yaml
-kubectl apply -n monitoring -f custom-metrics-configmap.yaml
-kubectl apply -n monitoring -f hpa-custom-metrics-cluster-role-binding.yaml

experimental/custom-metrics-api/hpa-custom-metrics-cluster-role-binding.yaml

-apiVersion: rbac.authorization.k8s.io/v1beta1
-kind: ClusterRoleBinding
-metadata:
-  name: hpa-controller-custom-metrics
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: ClusterRole
-  name: custom-metrics-server-resources
-subjects:
-- kind: ServiceAccount
-  name: horizontal-pod-autoscaler
-  namespace: kube-system

experimental/custom-metrics-api/sample-app.yaml

-kind: ServiceMonitor
-apiVersion: monitoring.coreos.com/v1
-metadata:
-  name: sample-app
-  labels:
-    app: sample-app
-spec:
-  selector:
-    matchLabels:
-      app: sample-app
-  endpoints: 
-  - port: http
-    interval: 5s
----
-apiVersion: v1
-kind: Service
-metadata:
-  name: sample-app
-  labels:
-    app: sample-app
-spec:
-  ports:
-  - name: http
-    port: 8080
-    targetPort: 8080
-  selector:
-    app: sample-app
----
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: sample-app
-  labels:
-    app: sample-app
-spec:
-  replicas: 1
-  selector:
-    matchLabels:
-      app: sample-app
-  template:
-    metadata:
-      labels:
-        app: sample-app
-    spec:
-      containers:
-      - image: luxas/autoscale-demo:v0.1.2
-        name: metrics-provider
-        ports:
-        - name: http
-          containerPort: 8080
----
-kind: HorizontalPodAutoscaler
-apiVersion: autoscaling/v2beta1
-metadata:
-  name: sample-app
-spec:
-  scaleTargetRef:
-    apiVersion: apps/v1
-    kind: Deployment
-    name: sample-app
-  minReplicas: 1
-  maxReplicas: 10
-  metrics:
-  - type: Pods
-    pods:
-      metricName: http_requests
-      targetAverageValue: 500m

experimental/custom-metrics-api/teardown.sh

-#!/usr/bin/env bash
-
-kubectl delete -n monitoring -f custom-metrics-apiserver-resource-reader-cluster-role-binding.yaml
-kubectl delete -n monitoring -f custom-metrics-apiservice.yaml
-kubectl delete -n monitoring -f custom-metrics-cluster-role.yaml
-kubectl delete -n monitoring -f custom-metrics-configmap.yaml
-kubectl delete -n monitoring -f hpa-custom-metrics-cluster-role-binding.yaml

jsonnet/kube-prometheus/kube-prometheus-custom-metrics.libsonnet

+local k = import 'ksonnet/ksonnet.beta.4/k.libsonnet';
+
+// Custom metrics API allows the HPA v2 to scale based on arbirary metrics.
+// For more details on usage visit https://github.com/DirectXMan12/k8s-prometheus-adapter#quick-links
+
+{
+  _config+:: {
+    prometheusAdapter+:: {
+      // Rules for custom-metrics
+      config+:: {
+        rules+: [
+          {
+            seriesQuery: '{__name__=~"^container_.*",container!="POD",namespace!="",pod!=""}',
+            seriesFilters: [],
+            resources: {
+              overrides: {
+                namespace: {
+                  resource: 'namespace'
+                },
+                pod: {
+                  resource: 'pod'
+                }
+              },
+            },
+            name: {
+              matches: '^container_(.*)_seconds_total$',
+              as: ""
+            },
+            metricsQuery: 'sum(rate(<<.Series>>{<<.LabelMatchers>>,container!="POD"}[1m])) by (<<.GroupBy>>)'
+          },
+          {
+            seriesQuery: '{__name__=~"^container_.*",container!="POD",namespace!="",pod!=""}',
+            seriesFilters: [
+              { isNot: '^container_.*_seconds_total$' },
+            ],
+            resources: {
+              overrides: {
+                namespace: {
+                  resource: 'namespace'
+                },
+                pod: {
+                  resource: 'pod'
+                }
+              },
+            },
+            name: {
+              matches: '^container_(.*)_total$',
+              as: ''
+            },
+            metricsQuery: 'sum(rate(<<.Series>>{<<.LabelMatchers>>,container!="POD"}[1m])) by (<<.GroupBy>>)'
+          },
+          {
+            seriesQuery: '{__name__=~"^container_.*",container!="POD",namespace!="",pod!=""}',
+            seriesFilters: [
+              { isNot: '^container_.*_total$' },
+            ],
+            resources: {
+              overrides: {
+                namespace: {
+                  resource: 'namespace'
+                },
+                pod: {
+                  resource: 'pod'
+                }
+              },
+            },
+            name: {
+              matches: '^container_(.*)$',
+              as: ''
+            },
+            metricsQuery: 'sum(<<.Series>>{<<.LabelMatchers>>,container!="POD"}) by (<<.GroupBy>>)'
+          },
+          {
+            seriesQuery: '{namespace!="",__name__!~"^container_.*"}',
+            seriesFilters: [
+              { isNot: '.*_total$' },
+            ],
+            resources: {
+              template: '<<.Resource>>'
+            },
+            name: {
+              matches: '',
+              as: ''
+            },
+            metricsQuery: 'sum(<<.Series>>{<<.LabelMatchers>>}) by (<<.GroupBy>>)'
+          },
+          {
+            seriesQuery: '{namespace!="",__name__!~"^container_.*"}',
+            seriesFilters: [
+              { isNot: '.*_seconds_total' },
+            ],
+            resources: {
+              template: '<<.Resource>>'
+            },
+            name: {
+              matches: '^(.*)_total$',
+              as: ''
+            },
+            metricsQuery: 'sum(rate(<<.Series>>{<<.LabelMatchers>>}[1m])) by (<<.GroupBy>>)'
+          },
+          {
+            seriesQuery: '{namespace!="",__name__!~"^container_.*"}',
+            seriesFilters: [],
+            resources: {
+              template: '<<.Resource>>'
+            },
+            name: {
+              matches: '^(.*)_seconds_total$',
+              as: ''
+            },
+            metricsQuery: 'sum(rate(<<.Series>>{<<.LabelMatchers>>}[1m])) by (<<.GroupBy>>)'
+          }
+        ],
+      },
+    },
+  },
+
+  prometheusAdapter+:: {
+    customMetricsApiService: {
+      apiVersion: 'apiregistration.k8s.io/v1',
+      kind: 'APIService',
+      metadata: {
+        name: 'v1beta1.custom.metrics.k8s.io',
+      },
+      spec: {
+        service: {
+          name: $.prometheusAdapter.service.metadata.name,
+          namespace: $._config.namespace,
+        },
+        group: 'custom.metrics.k8s.io',
+        version: 'v1beta1',
+        insecureSkipTLSVerify: true,
+        groupPriorityMinimum: 100,
+        versionPriority: 100,
+      },
+    },
+    customMetricsClusterRoleServerResources:
+      local clusterRole = k.rbac.v1.clusterRole;
+      local policyRule = clusterRole.rulesType;
+
+      local rules =
+        policyRule.new() +
+        policyRule.withApiGroups(['custom.metrics.k8s.io']) +
+        policyRule.withResources(['*']) +
+        policyRule.withVerbs(['*']);
+
+      clusterRole.new() +
+      clusterRole.mixin.metadata.withName('custom-metrics-server-resources') +
+      clusterRole.withRules(rules),
+
+    customMetricsClusterRoleBindingServerResources:
+      local clusterRoleBinding = k.rbac.v1.clusterRoleBinding;
+
+      clusterRoleBinding.new() +
+      clusterRoleBinding.mixin.metadata.withName('custom-metrics-server-resources') +
+      clusterRoleBinding.mixin.roleRef.withApiGroup('rbac.authorization.k8s.io') +
+      clusterRoleBinding.mixin.roleRef.withName('custom-metrics-server-resources') +
+      clusterRoleBinding.mixin.roleRef.mixinInstance({ kind: 'ClusterRole' }) +
+      clusterRoleBinding.withSubjects([{
+        kind: 'ServiceAccount',
+        name: $.prometheusAdapter.serviceAccount.metadata.name,
+        namespace: $._config.namespace,
+      }]),
+
+    customMetricsClusterRoleBindingHPA:
+      local clusterRoleBinding = k.rbac.v1.clusterRoleBinding;
+
+      clusterRoleBinding.new() +
+      clusterRoleBinding.mixin.metadata.withName('hpa-controller-custom-metrics') +
+      clusterRoleBinding.mixin.roleRef.withApiGroup('rbac.authorization.k8s.io') +
+      clusterRoleBinding.mixin.roleRef.withName('custom-metrics-server-resources') +
+      clusterRoleBinding.mixin.roleRef.mixinInstance({ kind: 'ClusterRole' }) +
+      clusterRoleBinding.withSubjects([{
+        kind: 'ServiceAccount',
+        name: 'horizontal-pod-autoscaler',
+        namespace: 'kube-system',
+      }]),
+  }
+}