From b65ed3ba6018633faae7d1bc2ea502e9cfbcdc99 Mon Sep 17 00:00:00 2001
From: Frederic Branczyk <fbranczyk@gmail.com>
Date: Fri, 10 Nov 2017 10:33:32 +0100
Subject: [PATCH] kube-prometheus: add custom-metrics-api example

---
 manifests/custom-metrics-api/.gitignore       |  7 ++++
 manifests/custom-metrics-api/README.md        | 11 +++++
 ...r-auth-delegator-cluster-role-binding.yaml | 12 ++++++
 ...cs-apiserver-auth-reader-role-binding.yaml | 13 ++++++
 .../custom-metrics-apiserver-deployment.yaml  | 41 +++++++++++++++++++
 ...-resource-reader-cluster-role-binding.yaml | 12 ++++++
 ...tom-metrics-apiserver-service-account.yaml |  4 ++
 .../custom-metrics-apiserver-service.yaml     | 10 +++++
 .../custom-metrics-apiservice.yaml            | 13 ++++++
 .../custom-metrics-cluster-role.yaml          |  9 ++++
 ...-metrics-resource-reader-cluster-role.yaml | 14 +++++++
 manifests/custom-metrics-api/deploy.sh        | 13 ++++++
 manifests/custom-metrics-api/gencerts.sh      | 21 ++++++++++
 ...a-custom-metrics-cluster-role-binding.yaml | 12 ++++++
 manifests/custom-metrics-api/teardown.sh      | 13 ++++++
 15 files changed, 205 insertions(+)
 create mode 100644 manifests/custom-metrics-api/.gitignore
 create mode 100644 manifests/custom-metrics-api/README.md
 create mode 100644 manifests/custom-metrics-api/custom-metrics-apiserver-auth-delegator-cluster-role-binding.yaml
 create mode 100644 manifests/custom-metrics-api/custom-metrics-apiserver-auth-reader-role-binding.yaml
 create mode 100644 manifests/custom-metrics-api/custom-metrics-apiserver-deployment.yaml
 create mode 100644 manifests/custom-metrics-api/custom-metrics-apiserver-resource-reader-cluster-role-binding.yaml
 create mode 100644 manifests/custom-metrics-api/custom-metrics-apiserver-service-account.yaml
 create mode 100644 manifests/custom-metrics-api/custom-metrics-apiserver-service.yaml
 create mode 100644 manifests/custom-metrics-api/custom-metrics-apiservice.yaml
 create mode 100644 manifests/custom-metrics-api/custom-metrics-cluster-role.yaml
 create mode 100644 manifests/custom-metrics-api/custom-metrics-resource-reader-cluster-role.yaml
 create mode 100755 manifests/custom-metrics-api/deploy.sh
 create mode 100755 manifests/custom-metrics-api/gencerts.sh
 create mode 100644 manifests/custom-metrics-api/hpa-custom-metrics-cluster-role-binding.yaml
 create mode 100755 manifests/custom-metrics-api/teardown.sh

diff --git a/manifests/custom-metrics-api/.gitignore b/manifests/custom-metrics-api/.gitignore
new file mode 100644
index 00000000..794c008c
--- /dev/null
+++ b/manifests/custom-metrics-api/.gitignore
@@ -0,0 +1,7 @@
+apiserver-key.pem
+apiserver.csr
+apiserver.pem
+metrics-ca-config.json
+metrics-ca.crt
+metrics-ca.key
+cm-adapter-serving-certs.yaml
diff --git a/manifests/custom-metrics-api/README.md b/manifests/custom-metrics-api/README.md
new file mode 100644
index 00000000..91375a42
--- /dev/null
+++ b/manifests/custom-metrics-api/README.md
@@ -0,0 +1,11 @@
+# Custom Metrics API
+
+The custom metrics API allows the HPA v2 to scale on arbirary metrics.
+
+This directory contains an example deployment of the custom metrics API adapter using Prometheus as the backing monitoring system.
+
+In order to deploy the custom metrics adapter for Prometheus you need to generate TLS certficates used to serve the API. An example of how these could be generated can be found in `./gencerts.sh`, note that this is _not_ recommended to be used in production. You need to employ a secure PKI strategy, this is merely an example to get started and try it out quickly.
+
+Once the generated `Secret` with the certificates is in place, you can deploy everything in the `monitoring` namespace using `./deploy.sh`.
+
+When you're done, you can teardown using the `./teardown.sh` script.
diff --git a/manifests/custom-metrics-api/custom-metrics-apiserver-auth-delegator-cluster-role-binding.yaml b/manifests/custom-metrics-api/custom-metrics-apiserver-auth-delegator-cluster-role-binding.yaml
new file mode 100644
index 00000000..8853bc1f
--- /dev/null
+++ b/manifests/custom-metrics-api/custom-metrics-apiserver-auth-delegator-cluster-role-binding.yaml
@@ -0,0 +1,12 @@
+apiVersion: rbac.authorization.k8s.io/v1beta1
+kind: ClusterRoleBinding
+metadata:
+  name: custom-metrics:system:auth-delegator
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: system:auth-delegator
+subjects:
+- kind: ServiceAccount
+  name: custom-metrics-apiserver
+  namespace: monitoring
diff --git a/manifests/custom-metrics-api/custom-metrics-apiserver-auth-reader-role-binding.yaml b/manifests/custom-metrics-api/custom-metrics-apiserver-auth-reader-role-binding.yaml
new file mode 100644
index 00000000..682143cf
--- /dev/null
+++ b/manifests/custom-metrics-api/custom-metrics-apiserver-auth-reader-role-binding.yaml
@@ -0,0 +1,13 @@
+apiVersion: rbac.authorization.k8s.io/v1beta1
+kind: RoleBinding
+metadata:
+  name: custom-metrics-auth-reader
+  namespace: kube-system
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: extension-apiserver-authentication-reader
+subjects:
+- kind: ServiceAccount
+  name: custom-metrics-apiserver
+  namespace: monitoring
diff --git a/manifests/custom-metrics-api/custom-metrics-apiserver-deployment.yaml b/manifests/custom-metrics-api/custom-metrics-apiserver-deployment.yaml
new file mode 100644
index 00000000..e5b4beea
--- /dev/null
+++ b/manifests/custom-metrics-api/custom-metrics-apiserver-deployment.yaml
@@ -0,0 +1,41 @@
+apiVersion: extensions/v1beta1
+kind: Deployment
+metadata:
+  labels:
+    app: custom-metrics-apiserver
+  name: custom-metrics-apiserver
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: custom-metrics-apiserver
+  template:
+    metadata:
+      labels:
+        app: custom-metrics-apiserver
+      name: custom-metrics-apiserver
+    spec:
+      serviceAccountName: custom-metrics-apiserver
+      containers:
+      - name: custom-metrics-apiserver
+        image: quay.io/coreos/k8s-prometheus-adapter-amd64:v0.2.0
+        args:
+        - /adapter
+        - --secure-port=6443
+        - --tls-cert-file=/var/run/serving-cert/serving.crt
+        - --tls-private-key-file=/var/run/serving-cert/serving.key
+        - --logtostderr=true
+        - --prometheus-url=http://prometheus-k8s.monitoring.svc:9090/
+        - --metrics-relist-interval=30s
+        - --rate-interval=5m
+        - --v=10
+        ports:
+        - containerPort: 6443
+        volumeMounts:
+        - mountPath: /var/run/serving-cert
+          name: volume-serving-cert
+          readOnly: true
+      volumes:
+      - name: volume-serving-cert
+        secret:
+          secretName: cm-adapter-serving-certs
diff --git a/manifests/custom-metrics-api/custom-metrics-apiserver-resource-reader-cluster-role-binding.yaml b/manifests/custom-metrics-api/custom-metrics-apiserver-resource-reader-cluster-role-binding.yaml
new file mode 100644
index 00000000..0335c177
--- /dev/null
+++ b/manifests/custom-metrics-api/custom-metrics-apiserver-resource-reader-cluster-role-binding.yaml
@@ -0,0 +1,12 @@
+apiVersion: rbac.authorization.k8s.io/v1beta1
+kind: ClusterRoleBinding
+metadata:
+  name: custom-metrics-resource-reader
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: custom-metrics-resource-reader
+subjects:
+- kind: ServiceAccount
+  name: custom-metrics-apiserver
+  namespace: monitoring
diff --git a/manifests/custom-metrics-api/custom-metrics-apiserver-service-account.yaml b/manifests/custom-metrics-api/custom-metrics-apiserver-service-account.yaml
new file mode 100644
index 00000000..29359409
--- /dev/null
+++ b/manifests/custom-metrics-api/custom-metrics-apiserver-service-account.yaml
@@ -0,0 +1,4 @@
+kind: ServiceAccount
+apiVersion: v1
+metadata:
+  name: custom-metrics-apiserver
diff --git a/manifests/custom-metrics-api/custom-metrics-apiserver-service.yaml b/manifests/custom-metrics-api/custom-metrics-apiserver-service.yaml
new file mode 100644
index 00000000..fb0addcb
--- /dev/null
+++ b/manifests/custom-metrics-api/custom-metrics-apiserver-service.yaml
@@ -0,0 +1,10 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: custom-metrics-apiserver
+spec:
+  ports:
+  - port: 443
+    targetPort: 6443
+  selector:
+    app: custom-metrics-apiserver
diff --git a/manifests/custom-metrics-api/custom-metrics-apiservice.yaml b/manifests/custom-metrics-api/custom-metrics-apiservice.yaml
new file mode 100644
index 00000000..cfc2ee63
--- /dev/null
+++ b/manifests/custom-metrics-api/custom-metrics-apiservice.yaml
@@ -0,0 +1,13 @@
+apiVersion: apiregistration.k8s.io/v1beta1
+kind: APIService
+metadata:
+  name: v1beta1.custom.metrics.k8s.io
+spec:
+  service:
+    name: custom-metrics-apiserver
+    namespace: monitoring
+  group: custom.metrics.k8s.io
+  version: v1beta1
+  insecureSkipTLSVerify: true
+  groupPriorityMinimum: 100
+  versionPriority: 100
diff --git a/manifests/custom-metrics-api/custom-metrics-cluster-role.yaml b/manifests/custom-metrics-api/custom-metrics-cluster-role.yaml
new file mode 100644
index 00000000..003f0bf1
--- /dev/null
+++ b/manifests/custom-metrics-api/custom-metrics-cluster-role.yaml
@@ -0,0 +1,9 @@
+apiVersion: rbac.authorization.k8s.io/v1beta1
+kind: ClusterRole
+metadata:
+  name: custom-metrics-server-resources
+rules:
+- apiGroups:
+  - custom.metrics.k8s.io
+  resources: ["*"]
+  verbs: ["*"]
diff --git a/manifests/custom-metrics-api/custom-metrics-resource-reader-cluster-role.yaml b/manifests/custom-metrics-api/custom-metrics-resource-reader-cluster-role.yaml
new file mode 100644
index 00000000..a5ad7604
--- /dev/null
+++ b/manifests/custom-metrics-api/custom-metrics-resource-reader-cluster-role.yaml
@@ -0,0 +1,14 @@
+apiVersion: rbac.authorization.k8s.io/v1beta1
+kind: ClusterRole
+metadata:
+  name: custom-metrics-resource-reader
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - namespaces
+  - pods
+  - services
+  verbs:
+  - get
+  - list
diff --git a/manifests/custom-metrics-api/deploy.sh b/manifests/custom-metrics-api/deploy.sh
new file mode 100755
index 00000000..2255c7fd
--- /dev/null
+++ b/manifests/custom-metrics-api/deploy.sh
@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+
+kubectl create -f custom-metrics-apiserver-auth-delegator-cluster-role-binding.yaml
+kubectl create -f custom-metrics-apiserver-auth-reader-role-binding.yaml
+kubectl -n monitoring create -f cm-adapter-serving-certs.yaml
+kubectl -n monitoring create -f custom-metrics-apiserver-deployment.yaml
+kubectl create -f custom-metrics-apiserver-resource-reader-cluster-role-binding.yaml
+kubectl -n monitoring create -f custom-metrics-apiserver-service-account.yaml
+kubectl -n monitoring create -f custom-metrics-apiserver-service.yaml
+kubectl create -f custom-metrics-apiservice.yaml
+kubectl create -f custom-metrics-cluster-role.yaml
+kubectl create -f custom-metrics-resource-reader-cluster-role.yaml
+kubectl create -f hpa-custom-metrics-cluster-role-binding.yaml
diff --git a/manifests/custom-metrics-api/gencerts.sh b/manifests/custom-metrics-api/gencerts.sh
new file mode 100755
index 00000000..312ce74c
--- /dev/null
+++ b/manifests/custom-metrics-api/gencerts.sh
@@ -0,0 +1,21 @@
+#!/usr/bin/env bash
+
+go get -v -u github.com/cloudflare/cfssl/cmd/...
+
+export PURPOSE=metrics
+openssl req -x509 -sha256 -new -nodes -days 365 -newkey rsa:2048 -keyout ${PURPOSE}-ca.key -out ${PURPOSE}-ca.crt -subj "/CN=ca"
+echo '{"signing":{"default":{"expiry":"43800h","usages":["signing","key encipherment","'${PURPOSE}'"]}}}' > "${PURPOSE}-ca-config.json"
+
+export SERVICE_NAME=custom-metrics-apiserver
+export ALT_NAMES='"custom-metrics-apiserver.monitoring","custom-metrics-apiserver.monitoring.svc"'
+echo '{"CN":"'${SERVICE_NAME}'","hosts":['${ALT_NAMES}'],"key":{"algo":"rsa","size":2048}}' | cfssl gencert -ca=metrics-ca.crt -ca-key=metrics-ca.key -config=metrics-ca-config.json - | cfssljson -bare apiserver
+
+cat <<-EOF > cm-adapter-serving-certs.yaml
+apiVersion: v1
+kind: Secret
+metadata:
+  name: cm-adapter-serving-certs
+data:
+  serving.crt: $(cat apiserver.pem | base64 --wrap=0)
+  serving.key: $(cat apiserver-key.pem | base64 --wrap=0)
+EOF
diff --git a/manifests/custom-metrics-api/hpa-custom-metrics-cluster-role-binding.yaml b/manifests/custom-metrics-api/hpa-custom-metrics-cluster-role-binding.yaml
new file mode 100644
index 00000000..530ebea5
--- /dev/null
+++ b/manifests/custom-metrics-api/hpa-custom-metrics-cluster-role-binding.yaml
@@ -0,0 +1,12 @@
+apiVersion: rbac.authorization.k8s.io/v1beta1
+kind: ClusterRoleBinding
+metadata:
+  name: hpa-controller-custom-metrics
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: custom-metrics-server-resources
+subjects:
+- kind: ServiceAccount
+  name: horizontal-pod-autoscaler
+  namespace: kube-system
diff --git a/manifests/custom-metrics-api/teardown.sh b/manifests/custom-metrics-api/teardown.sh
new file mode 100755
index 00000000..4797de1c
--- /dev/null
+++ b/manifests/custom-metrics-api/teardown.sh
@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+
+kubectl delete -f custom-metrics-apiserver-auth-delegator-cluster-role-binding.yaml
+kubectl delete -f custom-metrics-apiserver-auth-reader-role-binding.yaml
+kubectl -n monitoring delete -f cm-adapter-serving-certs.yaml
+kubectl -n monitoring delete -f custom-metrics-apiserver-deployment.yaml
+kubectl delete -f custom-metrics-apiserver-resource-reader-cluster-role-binding.yaml
+kubectl -n monitoring delete -f custom-metrics-apiserver-service-account.yaml
+kubectl -n monitoring delete -f custom-metrics-apiserver-service.yaml
+kubectl delete -f custom-metrics-apiservice.yaml
+kubectl delete -f custom-metrics-cluster-role.yaml
+kubectl delete -f custom-metrics-resource-reader-cluster-role.yaml
+kubectl delete -f hpa-custom-metrics-cluster-role-binding.yaml
-- 
GitLab