From 7b92c0289ad1391124a958ad889db4af815e2bcd Mon Sep 17 00:00:00 2001
From: Fabian Reinartz <fab.reinartz@gmail.com>
Date: Mon, 24 Oct 2016 11:55:51 +0200
Subject: [PATCH] Add roadmap, fix manifests, improve hack scripts

---
 README.md                                     | 13 ++++++++---
 hack/cluster-monitoring/deploy                | 23 +++++++++++++------
 hack/cluster-monitoring/teardown              | 20 +++++++++-------
 hack/example-service-monitoring/teardown      |  2 +-
 .../controllers/alertmanager-controller.yaml  |  5 +---
 .../controllers/prometheus-controller.yaml    |  5 +---
 .../example-app/prometheus-frontend-svc.yaml  |  3 ++-
 manifests/grafana/grafana-depl.yaml           |  2 --
 manifests/grafana/grafana-svc.yaml            |  4 ++--
 manifests/prometheus/prometheus-k8s-cm.yaml   |  4 ----
 manifests/prometheus/prometheus-k8s-svc.yaml  |  3 ++-
 11 files changed, 47 insertions(+), 37 deletions(-)

diff --git a/README.md b/README.md
index 8ea3c718..3bea1c65 100644
--- a/README.md
+++ b/README.md
@@ -24,7 +24,6 @@ controller, the [node_exporter](https://github.com/prometheus/node_exporter),
 
 ```bash
 export KUBECONFIG=<path>          # defaults to "~/.kube/config"
-export KUBE_NAMESPACE=<ns>        # defaults to "default"
 hack/cluster-monitoring/deploy
 ```
 
@@ -39,9 +38,9 @@ To tear it all down again, run:
 hack/cluster-monitoring/teardown
 ```
 
-__All services in the manifest still contain the `prometheus.io/scrape = true` annotations. It is not
+*All services in the manifest still contain the `prometheus.io/scrape = true` annotations. It is not
 used by the Prometheus controller. They remain for convential deployments as in
-[this example configuration](https://github.com/prometheus/prometheus/blob/master/documentation/examples/prometheus-kubernetes.yml).__
+[this example configuration](https://github.com/prometheus/prometheus/blob/master/documentation/examples/prometheus-kubernetes.yml).*
 
 ## Monitoring custom services
 
@@ -64,4 +63,12 @@ Teardown:
 hack/example-service-monitoring/teardown
 ```
 
+## Roadmap
+
+* Incorporate [Alertmanager controller](https://github.com/coreos/kube-alertmanager-controller)
+* Grafana controller that dynamically discovers and deploys dashboards from ConfigMaps
+* Collection of base alerting for cluster monitoring
+* KPM/Helm packages to easily provide production-ready cluster-monitoring setup (essentially contents of `hack/cluster-monitoring`)
+* Add meta-monitoring to default cluster monitoring setup
+
 
diff --git a/hack/cluster-monitoring/deploy b/hack/cluster-monitoring/deploy
index 4e337430..49aa10d7 100755
--- a/hack/cluster-monitoring/deploy
+++ b/hack/cluster-monitoring/deploy
@@ -1,14 +1,23 @@
 #!/usr/bin/env bash
 
+set -o xtrace
+
 if [ -z "${KUBECONFIG}" ]; then
     KUBECONFIG=~/.kube/config
 fi
 
-if [ -z "${KUBE_NAMESPACE}" ]; then
-    KUBE_NAMESPACE=default
-fi  
+kubectl --kubeconfig="$KUBECONFIG" create namespace monitoring
+
+kctl() {
+    kubectl --kubeconfig="$KUBECONFIG" -n "monitoring" "$@"
+}
+
+kctl apply -f manifests/controllers/prometheus-controller.yaml
+
+# Wait for TPRs to be ready.
+until kctl get servicemonitor; do sleep 0.1; done
+until kctl get prometheus; do sleep 0.1; done
 
-kubectl --kubeconfig="$KUBECONFIG" -n "$KUBE_NAMESPACE" create -f manifests/controllers/prometheus-controller.yaml
-kubectl --kubeconfig="$KUBECONFIG" -n "$KUBE_NAMESPACE" create -f manifests/exporters
-kubectl --kubeconfig="$KUBECONFIG" -n "$KUBE_NAMESPACE" create -f manifests/grafana
-kubectl --kubeconfig="$KUBECONFIG" -n "$KUBE_NAMESPACE" create -f manifests/prometheus
+kctl apply -f manifests/exporters
+kctl apply -f manifests/grafana
+kctl apply -f manifests/prometheus
diff --git a/hack/cluster-monitoring/teardown b/hack/cluster-monitoring/teardown
index bf1e88ef..54b35ada 100755
--- a/hack/cluster-monitoring/teardown
+++ b/hack/cluster-monitoring/teardown
@@ -4,11 +4,15 @@ if [ -z "${KUBECONFIG}" ]; then
     KUBECONFIG=~/.kube/config
 fi
 
-if [ -z "${KUBE_NAMESPACE}" ]; then
-    KUBE_NAMESPACE=default
-fi  
-
-kubectl --kubeconfig="$KUBECONFIG" -n "$KUBE_NAMESPACE" delete -f manifests/exporters
-kubectl --kubeconfig="$KUBECONFIG" -n "$KUBE_NAMESPACE" delete -f manifests/grafana
-kubectl --kubeconfig="$KUBECONFIG" -n "$KUBE_NAMESPACE" delete -f manifests/prometheus
-kubectl --kubeconfig="$KUBECONFIG" -n "$KUBE_NAMESPACE" delete -f manifests/controllers/prometheus-controller.yaml
+kctl() {
+    kubectl --kubeconfig="$KUBECONFIG" -n "monitoring" "$@"
+}
+
+kctl delete -f manifests/exporters
+kctl delete -f manifests/grafana
+kctl delete -f manifests/prometheus
+
+# Hack: wait a bit to let the controller delete the deployed Prometheus server.
+sleep 5
+
+kctl delete -f manifests/controllers/prometheus-controller.yaml
diff --git a/hack/example-service-monitoring/teardown b/hack/example-service-monitoring/teardown
index 2be7dcfd..448ff8a8 100755
--- a/hack/example-service-monitoring/teardown
+++ b/hack/example-service-monitoring/teardown
@@ -4,4 +4,4 @@ if [ -z "${KUBECONFIG}" ]; then
     KUBECONFIG=~/.kube/config
 fi
 
-kubectl --kubeconfig="$KUBECONFIG" -n "$KUBE_NAMESPACE" delete -f manifests/examples/example-app
\ No newline at end of file
+kubectl --kubeconfig="$KUBECONFIG" delete -f manifests/examples/example-app
\ No newline at end of file
diff --git a/manifests/controllers/alertmanager-controller.yaml b/manifests/controllers/alertmanager-controller.yaml
index 7f5eb7ee..65c71674 100644
--- a/manifests/controllers/alertmanager-controller.yaml
+++ b/manifests/controllers/alertmanager-controller.yaml
@@ -20,7 +20,4 @@ spec:
              memory: 50Mi
            limits:
              cpu: 200m
-             memory: 100Mi
-      imagePullSecrets:
-      - name: coreos-quay
-
+             memory: 100Mi
\ No newline at end of file
diff --git a/manifests/controllers/prometheus-controller.yaml b/manifests/controllers/prometheus-controller.yaml
index 7e62e605..0b4092ae 100644
--- a/manifests/controllers/prometheus-controller.yaml
+++ b/manifests/controllers/prometheus-controller.yaml
@@ -20,7 +20,4 @@ spec:
              memory: 50Mi
            limits:
              cpu: 200m
-             memory: 100Mi
-      imagePullSecrets:
-      - name: coreos-quay
-      
+             memory: 100Mi
\ No newline at end of file
diff --git a/manifests/examples/example-app/prometheus-frontend-svc.yaml b/manifests/examples/example-app/prometheus-frontend-svc.yaml
index 157d2f99..8295915b 100644
--- a/manifests/examples/example-app/prometheus-frontend-svc.yaml
+++ b/manifests/examples/example-app/prometheus-frontend-svc.yaml
@@ -12,4 +12,5 @@ spec:
     protocol: TCP
     targetPort: web
   selector:
-    prometheus.coreos.com: prometheus-frontend
+    prometheus.coreos.com/type: prometheus
+    prometheus.coreos.com/name: prometheus-frontend
diff --git a/manifests/grafana/grafana-depl.yaml b/manifests/grafana/grafana-depl.yaml
index b422cf01..25d2d209 100644
--- a/manifests/grafana/grafana-depl.yaml
+++ b/manifests/grafana/grafana-depl.yaml
@@ -54,6 +54,4 @@ spec:
       - name: grafana-dashboards
         configMap:
           name: grafana-dashboards
-      imagePullSecrets:
-      - name: coreos-quay
 
diff --git a/manifests/grafana/grafana-svc.yaml b/manifests/grafana/grafana-svc.yaml
index 11a12fc1..0fd4e87b 100644
--- a/manifests/grafana/grafana-svc.yaml
+++ b/manifests/grafana/grafana-svc.yaml
@@ -7,11 +7,11 @@ metadata:
   annotations:
     prometheus.io/scrape: 'true'
 spec:
+  type: NodePort
   ports:
   - name: web
     port: 3000
     protocol: TCP
     nodePort: 30902
   selector:
-    app: grafana
-  type: NodePort
\ No newline at end of file
+    app: grafana
\ No newline at end of file
diff --git a/manifests/prometheus/prometheus-k8s-cm.yaml b/manifests/prometheus/prometheus-k8s-cm.yaml
index 0ced5576..09b8e0b5 100644
--- a/manifests/prometheus/prometheus-k8s-cm.yaml
+++ b/manifests/prometheus/prometheus-k8s-cm.yaml
@@ -7,10 +7,6 @@ data:
     global:
       evaluation_interval: 30s
 
-    # Add your etcd scrape config here. We cannot default here as etcd is a
-    # prerequisite for Kubernetes.
-    # TODO(fabxc): potentially make this configurable via KPM in the future.
-
     scrape_configs:
     - job_name: kubelets
       scrape_interval: 20s
diff --git a/manifests/prometheus/prometheus-k8s-svc.yaml b/manifests/prometheus/prometheus-k8s-svc.yaml
index c5d79d02..d738d254 100644
--- a/manifests/prometheus/prometheus-k8s-svc.yaml
+++ b/manifests/prometheus/prometheus-k8s-svc.yaml
@@ -11,4 +11,5 @@ spec:
     protocol: TCP
     targetPort: web
   selector:
-    prometheus.coreos.com: prometheus-k8s
+    prometheus.coreos.com/type: prometheus
+    prometheus.coreos.com/name: prometheus-k8s
-- 
GitLab