From c8273cf9e9cda472760fd2167fb7425124e34d84 Mon Sep 17 00:00:00 2001
From: Paul Gier <pgier@redhat.com>
Date: Wed, 23 Oct 2019 01:38:31 -0500
Subject: [PATCH] Scripts and readme (#258)

* Avoid race condition when deploying quickstart example

The namespace and CRD creation must happen before any dependent objects
are created.  So we can put these in a separate directory (manifest/setup)
so they can be created before the other objects.

Some minor updates to the README and added a couple of scripts
for the quickstarts

Update travis script to avoid race condition

Signed-off-by: Paul Gier <pgier@redhat.com>

* simplify the example quickstart script and improve readme

Signed-off-by: Paul Gier <pgier@redhat.com>

* increase minikube memory to 6g for quickstart example
---
 README.md                                     | 38 +++++++++++--------
 build.sh                                      |  2 +-
 ...prometheus-rules-and-grafana-dashboards.md |  9 ++++-
 example.jsonnet                               |  9 ++++-
 examples/kustomize.jsonnet                    |  9 ++++-
 kustomization.yaml                            | 24 ++++++------
 ...> prometheus-operator-serviceMonitor.yaml} |  0
 .../0namespace-namespace.yaml}                |  0
 ...alertmanagerCustomResourceDefinition.yaml} |  0
 ...-0podmonitorCustomResourceDefinition.yaml} |  0
 ...-0prometheusCustomResourceDefinition.yaml} |  0
 ...ometheusruleCustomResourceDefinition.yaml} |  0
 ...rvicemonitorCustomResourceDefinition.yaml} |  0
 .../prometheus-operator-clusterRole.yaml}     |  0
 ...ometheus-operator-clusterRoleBinding.yaml} |  0
 .../prometheus-operator-deployment.yaml}      |  0
 .../prometheus-operator-service.yaml}         |  0
 .../prometheus-operator-serviceAccount.yaml}  |  0
 scripts/minikube-start-kvm.sh                 | 12 ++++++
 scripts/minikube-start.sh                     | 11 ++++++
 scripts/monitoring-deploy.sh                  | 11 ++++++
 tests/e2e/travis-e2e.sh                       | 14 +++----
 22 files changed, 98 insertions(+), 41 deletions(-)
 rename manifests/{0prometheus-operator-serviceMonitor.yaml => prometheus-operator-serviceMonitor.yaml} (100%)
 rename manifests/{00namespace-namespace.yaml => setup/0namespace-namespace.yaml} (100%)
 rename manifests/{0prometheus-operator-0alertmanagerCustomResourceDefinition.yaml => setup/prometheus-operator-0alertmanagerCustomResourceDefinition.yaml} (100%)
 rename manifests/{0prometheus-operator-0podmonitorCustomResourceDefinition.yaml => setup/prometheus-operator-0podmonitorCustomResourceDefinition.yaml} (100%)
 rename manifests/{0prometheus-operator-0prometheusCustomResourceDefinition.yaml => setup/prometheus-operator-0prometheusCustomResourceDefinition.yaml} (100%)
 rename manifests/{0prometheus-operator-0prometheusruleCustomResourceDefinition.yaml => setup/prometheus-operator-0prometheusruleCustomResourceDefinition.yaml} (100%)
 rename manifests/{0prometheus-operator-0servicemonitorCustomResourceDefinition.yaml => setup/prometheus-operator-0servicemonitorCustomResourceDefinition.yaml} (100%)
 rename manifests/{0prometheus-operator-clusterRole.yaml => setup/prometheus-operator-clusterRole.yaml} (100%)
 rename manifests/{0prometheus-operator-clusterRoleBinding.yaml => setup/prometheus-operator-clusterRoleBinding.yaml} (100%)
 rename manifests/{0prometheus-operator-deployment.yaml => setup/prometheus-operator-deployment.yaml} (100%)
 rename manifests/{0prometheus-operator-service.yaml => setup/prometheus-operator-service.yaml} (100%)
 rename manifests/{0prometheus-operator-serviceAccount.yaml => setup/prometheus-operator-serviceAccount.yaml} (100%)
 create mode 100755 scripts/minikube-start-kvm.sh
 create mode 100755 scripts/minikube-start.sh
 create mode 100755 scripts/monitoring-deploy.sh

diff --git a/README.md b/README.md
index 6fcdb3c4..2ca8757a 100644
--- a/README.md
+++ b/README.md
@@ -71,13 +71,13 @@ This adapter is an Extension API Server and Kubernetes needs to be have this fea
 
 ### minikube
 
-In order to just try out this stack, start [minikube](https://github.com/kubernetes/minikube) with the following command:
+To try out this stack, start [minikube](https://github.com/kubernetes/minikube) with the following command:
 
 ```shell
-$ minikube delete && minikube start --kubernetes-version=v1.14.4 --memory=4096 --bootstrapper=kubeadm --extra-config=kubelet.authentication-token-webhook=true --extra-config=kubelet.authorization-mode=Webhook --extra-config=scheduler.address=0.0.0.0 --extra-config=controller-manager.address=0.0.0.0
+$ minikube delete && minikube start --kubernetes-version=v1.16.0 --memory=6g --bootstrapper=kubeadm --extra-config=kubelet.authentication-token-webhook=true --extra-config=kubelet.authorization-mode=Webhook --extra-config=scheduler.address=0.0.0.0 --extra-config=controller-manager.address=0.0.0.0
 ```
 
-The kube-prometheus stack includes a resource metrics API server, like the metrics-server does. So ensure the metrics-server plugin is disabled on minikube:
+The kube-prometheus stack includes a resource metrics API server, so the metrics-server addon is not necessary. Ensure the metrics-server addon is disabled on minikube:
 
 ```shell
 $ minikube addons disable metrics-server
@@ -90,20 +90,23 @@ $ minikube addons disable metrics-server
 This project is intended to be used as a library (i.e. the intent is not for you to create your own modified copy of this repository).
 
 Though for a quickstart a compiled version of the Kubernetes [manifests](manifests) generated with this library (specifically with `example.jsonnet`) is checked into this repository in order to try the content out quickly. To try out the stack un-customized run:
- * Simply create the stack:
-```shell
-$ kubectl create -f manifests/
-
-# It can take a few seconds for the above 'create manifests' command to fully create the following resources, so verify the resources are ready before proceeding.
-$ until kubectl get customresourcedefinitions servicemonitors.monitoring.coreos.com ; do date; sleep 1; echo ""; done
-$ until kubectl get servicemonitors --all-namespaces ; do date; sleep 1; echo ""; done
+ * Create the monitoring stack using the config in the `manifests` directory:
 
-$ kubectl apply -f manifests/ # This command sometimes may need to be done twice (to workaround a race condition).
+```shell
+# Create the namespace and CRDs, and then wait for them to be availble before creating the remaining resources
+kubectl create -f manifests/setup
+until kubectl get servicemonitors --all-namespaces ; do date; sleep 1; echo ""; done
+kubectl create -f manifests/
 ```
 
+We create the namespace and CustomResourceDefinitions first to avoid race conditions when deploying the monitoring components.
+Alternatively, the resources in both folders can be applied with a single command 
+`kubectl create -f manifests/setup -f manifests`, but it may be necessary to run the command multiple times for all components to
+be created successfullly.
+
  * And to teardown the stack:
 ```shell
-$ kubectl delete -f manifests/
+kubectl delete --ignore-not-found=true -f manifests/ -f manifests/setup
 ```
 
 ### Access the dashboards
@@ -187,8 +190,13 @@ local kp =
     },
   };
 
-{ ['00namespace-' + name]: kp.kubePrometheus[name] for name in std.objectFields(kp.kubePrometheus) } +
-{ ['0prometheus-operator-' + name]: kp.prometheusOperator[name] for name in std.objectFields(kp.prometheusOperator) } +
+{ ['setup/0namespace-' + name]: kp.kubePrometheus[name] for name in std.objectFields(kp.kubePrometheus) } +
+{
+  ['setup/prometheus-operator-' + name]: kp.prometheusOperator[name]
+  for name in std.filter((function(name) name != 'serviceMonitor'), std.objectFields(kp.prometheusOperator))
+} +
+// serviceMonitor is separated so that it can be created after the CRDs are ready
+{ 'prometheus-operator-serviceMonitor': kp.prometheusOperator.serviceMonitor } +
 { ['node-exporter-' + name]: kp.nodeExporter[name] for name in std.objectFields(kp.nodeExporter) } +
 { ['kube-state-metrics-' + name]: kp.kubeStateMetrics[name] for name in std.objectFields(kp.kubeStateMetrics) } +
 { ['alertmanager-' + name]: kp.alertmanager[name] for name in std.objectFields(kp.alertmanager) } +
@@ -212,7 +220,7 @@ set -o pipefail
 
 # Make sure to start with a clean 'manifests' dir
 rm -rf manifests
-mkdir manifests
+mkdir -p manifests/setup
 
                                                # optional, but we would like to generate yaml, not json
 jsonnet -J vendor -m manifests "${1-example.jsonnet}" | xargs -I{} sh -c 'cat {} | gojsontoyaml > {}.yaml; rm -f {}' -- {}
diff --git a/build.sh b/build.sh
index f68cd447..0663889d 100755
--- a/build.sh
+++ b/build.sh
@@ -9,7 +9,7 @@ set -o pipefail
 
 # Make sure to start with a clean 'manifests' dir
 rm -rf manifests
-mkdir manifests
+mkdir -p manifests/setup
 
                                                # optional, but we would like to generate yaml, not json
 jsonnet -J vendor -m manifests "${1-example.jsonnet}" | xargs -I{} sh -c 'cat {} | gojsontoyaml > {}.yaml; rm -f {}' -- {}
diff --git a/docs/developing-prometheus-rules-and-grafana-dashboards.md b/docs/developing-prometheus-rules-and-grafana-dashboards.md
index aa1283ec..34801a88 100644
--- a/docs/developing-prometheus-rules-and-grafana-dashboards.md
+++ b/docs/developing-prometheus-rules-and-grafana-dashboards.md
@@ -24,8 +24,13 @@ local kp =
     },
   };
 
-{ ['00namespace-' + name]: kp.kubePrometheus[name] for name in std.objectFields(kp.kubePrometheus) } +
-{ ['0prometheus-operator-' + name]: kp.prometheusOperator[name] for name in std.objectFields(kp.prometheusOperator) } +
+{ ['setup/0namespace-' + name]: kp.kubePrometheus[name] for name in std.objectFields(kp.kubePrometheus) } +
+{
+  ['setup/prometheus-operator-' + name]: kp.prometheusOperator[name]
+  for name in std.filter((function(name) name != 'serviceMonitor'), std.objectFields(kp.prometheusOperator))
+} +
+// serviceMonitor is separated so that it can be created after the CRDs are ready
+{ 'prometheus-operator-serviceMonitor': kp.prometheusOperator.serviceMonitor } +
 { ['node-exporter-' + name]: kp.nodeExporter[name] for name in std.objectFields(kp.nodeExporter) } +
 { ['kube-state-metrics-' + name]: kp.kubeStateMetrics[name] for name in std.objectFields(kp.kubeStateMetrics) } +
 { ['alertmanager-' + name]: kp.alertmanager[name] for name in std.objectFields(kp.alertmanager) } +
diff --git a/example.jsonnet b/example.jsonnet
index 7002eff2..77864e09 100644
--- a/example.jsonnet
+++ b/example.jsonnet
@@ -12,8 +12,13 @@ local kp =
     },
   };
 
-{ ['00namespace-' + name]: kp.kubePrometheus[name] for name in std.objectFields(kp.kubePrometheus) } +
-{ ['0prometheus-operator-' + name]: kp.prometheusOperator[name] for name in std.objectFields(kp.prometheusOperator) } +
+{ ['setup/0namespace-' + name]: kp.kubePrometheus[name] for name in std.objectFields(kp.kubePrometheus) } +
+{
+  ['setup/prometheus-operator-' + name]: kp.prometheusOperator[name]
+  for name in std.filter((function(name) name != 'serviceMonitor'), std.objectFields(kp.prometheusOperator))
+} +
+// serviceMonitor is separated so that it can be created after the CRDs are ready
+{ 'prometheus-operator-serviceMonitor': kp.prometheusOperator.serviceMonitor } +
 { ['node-exporter-' + name]: kp.nodeExporter[name] for name in std.objectFields(kp.nodeExporter) } +
 { ['kube-state-metrics-' + name]: kp.kubeStateMetrics[name] for name in std.objectFields(kp.kubeStateMetrics) } +
 { ['alertmanager-' + name]: kp.alertmanager[name] for name in std.objectFields(kp.alertmanager) } +
diff --git a/examples/kustomize.jsonnet b/examples/kustomize.jsonnet
index db3ba344..38dd6c89 100644
--- a/examples/kustomize.jsonnet
+++ b/examples/kustomize.jsonnet
@@ -8,8 +8,13 @@ local kp =
 local manifests =
   // Uncomment line below to enable vertical auto scaling of kube-state-metrics
   //{ ['ksm-autoscaler-' + name]: kp.ksmAutoscaler[name] for name in std.objectFields(kp.ksmAutoscaler) } +
-  { ['00namespace-' + name]: kp.kubePrometheus[name] for name in std.objectFields(kp.kubePrometheus) } +
-  { ['0prometheus-operator-' + name]: kp.prometheusOperator[name] for name in std.objectFields(kp.prometheusOperator) } +
+  { ['setup/0namespace-' + name]: kp.kubePrometheus[name] for name in std.objectFields(kp.kubePrometheus) } +
+  {
+    ['setup/prometheus-operator-' + name]: kp.prometheusOperator[name]
+    for name in std.filter((function(name) name != 'serviceMonitor'), std.objectFields(kp.prometheusOperator))
+  } +
+  // serviceMonitor is separated so that it can be created after the CRDs are ready
+  { 'prometheus-operator-serviceMonitor': kp.prometheusOperator.serviceMonitor } +
   { ['node-exporter-' + name]: kp.nodeExporter[name] for name in std.objectFields(kp.nodeExporter) } +
   { ['kube-state-metrics-' + name]: kp.kubeStateMetrics[name] for name in std.objectFields(kp.kubeStateMetrics) } +
   { ['alertmanager-' + name]: kp.alertmanager[name] for name in std.objectFields(kp.alertmanager) } +
diff --git a/kustomization.yaml b/kustomization.yaml
index a580ed8e..bd03a83e 100644
--- a/kustomization.yaml
+++ b/kustomization.yaml
@@ -1,18 +1,6 @@
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
-- ./manifests/00namespace-namespace.yaml
-- ./manifests/0prometheus-operator-0alertmanagerCustomResourceDefinition.yaml
-- ./manifests/0prometheus-operator-0podmonitorCustomResourceDefinition.yaml
-- ./manifests/0prometheus-operator-0prometheusCustomResourceDefinition.yaml
-- ./manifests/0prometheus-operator-0prometheusruleCustomResourceDefinition.yaml
-- ./manifests/0prometheus-operator-0servicemonitorCustomResourceDefinition.yaml
-- ./manifests/0prometheus-operator-clusterRole.yaml
-- ./manifests/0prometheus-operator-clusterRoleBinding.yaml
-- ./manifests/0prometheus-operator-deployment.yaml
-- ./manifests/0prometheus-operator-service.yaml
-- ./manifests/0prometheus-operator-serviceAccount.yaml
-- ./manifests/0prometheus-operator-serviceMonitor.yaml
 - ./manifests/alertmanager-alertmanager.yaml
 - ./manifests/alertmanager-secret.yaml
 - ./manifests/alertmanager-service.yaml
@@ -52,6 +40,7 @@ resources:
 - ./manifests/prometheus-adapter-serviceAccount.yaml
 - ./manifests/prometheus-clusterRole.yaml
 - ./manifests/prometheus-clusterRoleBinding.yaml
+- ./manifests/prometheus-operator-serviceMonitor.yaml
 - ./manifests/prometheus-prometheus.yaml
 - ./manifests/prometheus-roleBindingConfig.yaml
 - ./manifests/prometheus-roleBindingSpecificNamespaces.yaml
@@ -66,3 +55,14 @@ resources:
 - ./manifests/prometheus-serviceMonitorKubeControllerManager.yaml
 - ./manifests/prometheus-serviceMonitorKubeScheduler.yaml
 - ./manifests/prometheus-serviceMonitorKubelet.yaml
+- ./manifests/setup/0namespace-namespace.yaml
+- ./manifests/setup/prometheus-operator-0alertmanagerCustomResourceDefinition.yaml
+- ./manifests/setup/prometheus-operator-0podmonitorCustomResourceDefinition.yaml
+- ./manifests/setup/prometheus-operator-0prometheusCustomResourceDefinition.yaml
+- ./manifests/setup/prometheus-operator-0prometheusruleCustomResourceDefinition.yaml
+- ./manifests/setup/prometheus-operator-0servicemonitorCustomResourceDefinition.yaml
+- ./manifests/setup/prometheus-operator-clusterRole.yaml
+- ./manifests/setup/prometheus-operator-clusterRoleBinding.yaml
+- ./manifests/setup/prometheus-operator-deployment.yaml
+- ./manifests/setup/prometheus-operator-service.yaml
+- ./manifests/setup/prometheus-operator-serviceAccount.yaml
diff --git a/manifests/0prometheus-operator-serviceMonitor.yaml b/manifests/prometheus-operator-serviceMonitor.yaml
similarity index 100%
rename from manifests/0prometheus-operator-serviceMonitor.yaml
rename to manifests/prometheus-operator-serviceMonitor.yaml
diff --git a/manifests/00namespace-namespace.yaml b/manifests/setup/0namespace-namespace.yaml
similarity index 100%
rename from manifests/00namespace-namespace.yaml
rename to manifests/setup/0namespace-namespace.yaml
diff --git a/manifests/0prometheus-operator-0alertmanagerCustomResourceDefinition.yaml b/manifests/setup/prometheus-operator-0alertmanagerCustomResourceDefinition.yaml
similarity index 100%
rename from manifests/0prometheus-operator-0alertmanagerCustomResourceDefinition.yaml
rename to manifests/setup/prometheus-operator-0alertmanagerCustomResourceDefinition.yaml
diff --git a/manifests/0prometheus-operator-0podmonitorCustomResourceDefinition.yaml b/manifests/setup/prometheus-operator-0podmonitorCustomResourceDefinition.yaml
similarity index 100%
rename from manifests/0prometheus-operator-0podmonitorCustomResourceDefinition.yaml
rename to manifests/setup/prometheus-operator-0podmonitorCustomResourceDefinition.yaml
diff --git a/manifests/0prometheus-operator-0prometheusCustomResourceDefinition.yaml b/manifests/setup/prometheus-operator-0prometheusCustomResourceDefinition.yaml
similarity index 100%
rename from manifests/0prometheus-operator-0prometheusCustomResourceDefinition.yaml
rename to manifests/setup/prometheus-operator-0prometheusCustomResourceDefinition.yaml
diff --git a/manifests/0prometheus-operator-0prometheusruleCustomResourceDefinition.yaml b/manifests/setup/prometheus-operator-0prometheusruleCustomResourceDefinition.yaml
similarity index 100%
rename from manifests/0prometheus-operator-0prometheusruleCustomResourceDefinition.yaml
rename to manifests/setup/prometheus-operator-0prometheusruleCustomResourceDefinition.yaml
diff --git a/manifests/0prometheus-operator-0servicemonitorCustomResourceDefinition.yaml b/manifests/setup/prometheus-operator-0servicemonitorCustomResourceDefinition.yaml
similarity index 100%
rename from manifests/0prometheus-operator-0servicemonitorCustomResourceDefinition.yaml
rename to manifests/setup/prometheus-operator-0servicemonitorCustomResourceDefinition.yaml
diff --git a/manifests/0prometheus-operator-clusterRole.yaml b/manifests/setup/prometheus-operator-clusterRole.yaml
similarity index 100%
rename from manifests/0prometheus-operator-clusterRole.yaml
rename to manifests/setup/prometheus-operator-clusterRole.yaml
diff --git a/manifests/0prometheus-operator-clusterRoleBinding.yaml b/manifests/setup/prometheus-operator-clusterRoleBinding.yaml
similarity index 100%
rename from manifests/0prometheus-operator-clusterRoleBinding.yaml
rename to manifests/setup/prometheus-operator-clusterRoleBinding.yaml
diff --git a/manifests/0prometheus-operator-deployment.yaml b/manifests/setup/prometheus-operator-deployment.yaml
similarity index 100%
rename from manifests/0prometheus-operator-deployment.yaml
rename to manifests/setup/prometheus-operator-deployment.yaml
diff --git a/manifests/0prometheus-operator-service.yaml b/manifests/setup/prometheus-operator-service.yaml
similarity index 100%
rename from manifests/0prometheus-operator-service.yaml
rename to manifests/setup/prometheus-operator-service.yaml
diff --git a/manifests/0prometheus-operator-serviceAccount.yaml b/manifests/setup/prometheus-operator-serviceAccount.yaml
similarity index 100%
rename from manifests/0prometheus-operator-serviceAccount.yaml
rename to manifests/setup/prometheus-operator-serviceAccount.yaml
diff --git a/scripts/minikube-start-kvm.sh b/scripts/minikube-start-kvm.sh
new file mode 100755
index 00000000..da82a95c
--- /dev/null
+++ b/scripts/minikube-start-kvm.sh
@@ -0,0 +1,12 @@
+#!/bin/bash
+minikube delete
+minikube addons disable metrics-server
+minikube start \
+    --vm-driver=kvm2 \
+    --kubernetes-version=v1.16.0 \
+    --memory=6g \
+    --bootstrapper=kubeadm \
+    --extra-config=kubelet.authentication-token-webhook=true \
+    --extra-config=kubelet.authorization-mode=Webhook \
+    --extra-config=scheduler.address=0.0.0.0 \
+    --extra-config=controller-manager.address=0.0.0.0
diff --git a/scripts/minikube-start.sh b/scripts/minikube-start.sh
new file mode 100755
index 00000000..eb29e860
--- /dev/null
+++ b/scripts/minikube-start.sh
@@ -0,0 +1,11 @@
+#!/bin/bash
+minikube delete
+minikube addons disable metrics-server
+minikube start \
+    --kubernetes-version=v1.16.0 \
+    --memory=6g \
+    --bootstrapper=kubeadm \
+    --extra-config=kubelet.authentication-token-webhook=true \
+    --extra-config=kubelet.authorization-mode=Webhook \
+    --extra-config=scheduler.address=0.0.0.0 \
+    --extra-config=controller-manager.address=0.0.0.0
diff --git a/scripts/monitoring-deploy.sh b/scripts/monitoring-deploy.sh
new file mode 100755
index 00000000..7bc20905
--- /dev/null
+++ b/scripts/monitoring-deploy.sh
@@ -0,0 +1,11 @@
+#!/bin/bash
+
+# create namespace and CRDs
+kubectl create -f manifests/setup
+
+# wait for CRD creation to complete
+until kubectl get servicemonitors --all-namespaces ; do date; sleep 1; echo ""; done
+
+# create monitoring components
+kubectl create -f manifests/
+
diff --git a/tests/e2e/travis-e2e.sh b/tests/e2e/travis-e2e.sh
index aa32d668..4efb2bd1 100755
--- a/tests/e2e/travis-e2e.sh
+++ b/tests/e2e/travis-e2e.sh
@@ -16,13 +16,13 @@ chmod +x kind
 ./kind create cluster
 export KUBECONFIG="$(./kind get kubeconfig-path)"
 
-./kubectl apply -f manifests/0prometheus-operator-0alertmanagerCustomResourceDefinition.yaml
-./kubectl apply -f manifests/0prometheus-operator-0prometheusCustomResourceDefinition.yaml
-./kubectl apply -f manifests/0prometheus-operator-0prometheusruleCustomResourceDefinition.yaml
-./kubectl apply -f manifests/0prometheus-operator-0servicemonitorCustomResourceDefinition.yaml
+# create namespace, permissions, and CRDs
+./kubectl create -f manifests/setup
 
-# Wait for CRDs to be successfully registered
-sleep 10
+# wait for CRD creation to complete
+until ./kubectl get servicemonitors --all-namespaces ; do date; sleep 1; echo ""; done
+
+# create monitoring components
+./kubectl create -f manifests/
 
-./kubectl apply -f manifests
 make test-e2e
-- 
GitLab