diff --git a/README.md b/README.md index 6fcdb3c46ab69a6ca831580bdd3b50efaabaf739..2ca8757ace60a38f0e287296f7c2e5cd19ee9a39 100644 --- a/README.md +++ b/README.md @@ -71,13 +71,13 @@ This adapter is an Extension API Server and Kubernetes needs to be have this fea ### minikube -In order to just try out this stack, start [minikube](https://github.com/kubernetes/minikube) with the following command: +To try out this stack, start [minikube](https://github.com/kubernetes/minikube) with the following command: ```shell -$ minikube delete && minikube start --kubernetes-version=v1.14.4 --memory=4096 --bootstrapper=kubeadm --extra-config=kubelet.authentication-token-webhook=true --extra-config=kubelet.authorization-mode=Webhook --extra-config=scheduler.address=0.0.0.0 --extra-config=controller-manager.address=0.0.0.0 +$ minikube delete && minikube start --kubernetes-version=v1.16.0 --memory=6g --bootstrapper=kubeadm --extra-config=kubelet.authentication-token-webhook=true --extra-config=kubelet.authorization-mode=Webhook --extra-config=scheduler.address=0.0.0.0 --extra-config=controller-manager.address=0.0.0.0 ``` -The kube-prometheus stack includes a resource metrics API server, like the metrics-server does. So ensure the metrics-server plugin is disabled on minikube: +The kube-prometheus stack includes a resource metrics API server, so the metrics-server addon is not necessary. Ensure the metrics-server addon is disabled on minikube: ```shell $ minikube addons disable metrics-server @@ -90,20 +90,23 @@ $ minikube addons disable metrics-server This project is intended to be used as a library (i.e. the intent is not for you to create your own modified copy of this repository). Though for a quickstart a compiled version of the Kubernetes [manifests](manifests) generated with this library (specifically with `example.jsonnet`) is checked into this repository in order to try the content out quickly. To try out the stack un-customized run: - * Simply create the stack: -```shell -$ kubectl create -f manifests/ - -# It can take a few seconds for the above 'create manifests' command to fully create the following resources, so verify the resources are ready before proceeding. -$ until kubectl get customresourcedefinitions servicemonitors.monitoring.coreos.com ; do date; sleep 1; echo ""; done -$ until kubectl get servicemonitors --all-namespaces ; do date; sleep 1; echo ""; done + * Create the monitoring stack using the config in the `manifests` directory: -$ kubectl apply -f manifests/ # This command sometimes may need to be done twice (to workaround a race condition). +```shell +# Create the namespace and CRDs, and then wait for them to be availble before creating the remaining resources +kubectl create -f manifests/setup +until kubectl get servicemonitors --all-namespaces ; do date; sleep 1; echo ""; done +kubectl create -f manifests/ ``` +We create the namespace and CustomResourceDefinitions first to avoid race conditions when deploying the monitoring components. +Alternatively, the resources in both folders can be applied with a single command +`kubectl create -f manifests/setup -f manifests`, but it may be necessary to run the command multiple times for all components to +be created successfullly. + * And to teardown the stack: ```shell -$ kubectl delete -f manifests/ +kubectl delete --ignore-not-found=true -f manifests/ -f manifests/setup ``` ### Access the dashboards @@ -187,8 +190,13 @@ local kp = }, }; -{ ['00namespace-' + name]: kp.kubePrometheus[name] for name in std.objectFields(kp.kubePrometheus) } + -{ ['0prometheus-operator-' + name]: kp.prometheusOperator[name] for name in std.objectFields(kp.prometheusOperator) } + +{ ['setup/0namespace-' + name]: kp.kubePrometheus[name] for name in std.objectFields(kp.kubePrometheus) } + +{ + ['setup/prometheus-operator-' + name]: kp.prometheusOperator[name] + for name in std.filter((function(name) name != 'serviceMonitor'), std.objectFields(kp.prometheusOperator)) +} + +// serviceMonitor is separated so that it can be created after the CRDs are ready +{ 'prometheus-operator-serviceMonitor': kp.prometheusOperator.serviceMonitor } + { ['node-exporter-' + name]: kp.nodeExporter[name] for name in std.objectFields(kp.nodeExporter) } + { ['kube-state-metrics-' + name]: kp.kubeStateMetrics[name] for name in std.objectFields(kp.kubeStateMetrics) } + { ['alertmanager-' + name]: kp.alertmanager[name] for name in std.objectFields(kp.alertmanager) } + @@ -212,7 +220,7 @@ set -o pipefail # Make sure to start with a clean 'manifests' dir rm -rf manifests -mkdir manifests +mkdir -p manifests/setup # optional, but we would like to generate yaml, not json jsonnet -J vendor -m manifests "${1-example.jsonnet}" | xargs -I{} sh -c 'cat {} | gojsontoyaml > {}.yaml; rm -f {}' -- {} diff --git a/build.sh b/build.sh index f68cd4475df21548ce6f7b26eff574929f3043e7..0663889d83038e825061fcfa3e332b51df1a4711 100755 --- a/build.sh +++ b/build.sh @@ -9,7 +9,7 @@ set -o pipefail # Make sure to start with a clean 'manifests' dir rm -rf manifests -mkdir manifests +mkdir -p manifests/setup # optional, but we would like to generate yaml, not json jsonnet -J vendor -m manifests "${1-example.jsonnet}" | xargs -I{} sh -c 'cat {} | gojsontoyaml > {}.yaml; rm -f {}' -- {} diff --git a/docs/developing-prometheus-rules-and-grafana-dashboards.md b/docs/developing-prometheus-rules-and-grafana-dashboards.md index aa1283ecea847551d3aaaefc6ad22f73c603f7f4..34801a88ac8f3092aa2515f8e5c969bc87db0a51 100644 --- a/docs/developing-prometheus-rules-and-grafana-dashboards.md +++ b/docs/developing-prometheus-rules-and-grafana-dashboards.md @@ -24,8 +24,13 @@ local kp = }, }; -{ ['00namespace-' + name]: kp.kubePrometheus[name] for name in std.objectFields(kp.kubePrometheus) } + -{ ['0prometheus-operator-' + name]: kp.prometheusOperator[name] for name in std.objectFields(kp.prometheusOperator) } + +{ ['setup/0namespace-' + name]: kp.kubePrometheus[name] for name in std.objectFields(kp.kubePrometheus) } + +{ + ['setup/prometheus-operator-' + name]: kp.prometheusOperator[name] + for name in std.filter((function(name) name != 'serviceMonitor'), std.objectFields(kp.prometheusOperator)) +} + +// serviceMonitor is separated so that it can be created after the CRDs are ready +{ 'prometheus-operator-serviceMonitor': kp.prometheusOperator.serviceMonitor } + { ['node-exporter-' + name]: kp.nodeExporter[name] for name in std.objectFields(kp.nodeExporter) } + { ['kube-state-metrics-' + name]: kp.kubeStateMetrics[name] for name in std.objectFields(kp.kubeStateMetrics) } + { ['alertmanager-' + name]: kp.alertmanager[name] for name in std.objectFields(kp.alertmanager) } + diff --git a/example.jsonnet b/example.jsonnet index 7002eff26b6619042086f891c02dd85e67e85ee3..77864e09a87d14f1dd7befbe65c7921eb03af326 100644 --- a/example.jsonnet +++ b/example.jsonnet @@ -12,8 +12,13 @@ local kp = }, }; -{ ['00namespace-' + name]: kp.kubePrometheus[name] for name in std.objectFields(kp.kubePrometheus) } + -{ ['0prometheus-operator-' + name]: kp.prometheusOperator[name] for name in std.objectFields(kp.prometheusOperator) } + +{ ['setup/0namespace-' + name]: kp.kubePrometheus[name] for name in std.objectFields(kp.kubePrometheus) } + +{ + ['setup/prometheus-operator-' + name]: kp.prometheusOperator[name] + for name in std.filter((function(name) name != 'serviceMonitor'), std.objectFields(kp.prometheusOperator)) +} + +// serviceMonitor is separated so that it can be created after the CRDs are ready +{ 'prometheus-operator-serviceMonitor': kp.prometheusOperator.serviceMonitor } + { ['node-exporter-' + name]: kp.nodeExporter[name] for name in std.objectFields(kp.nodeExporter) } + { ['kube-state-metrics-' + name]: kp.kubeStateMetrics[name] for name in std.objectFields(kp.kubeStateMetrics) } + { ['alertmanager-' + name]: kp.alertmanager[name] for name in std.objectFields(kp.alertmanager) } + diff --git a/examples/kustomize.jsonnet b/examples/kustomize.jsonnet index db3ba344278cbaac7e8cda89fc542eb2e3d736f3..38dd6c89d66559beff0c949545c2f1e5b40cccbf 100644 --- a/examples/kustomize.jsonnet +++ b/examples/kustomize.jsonnet @@ -8,8 +8,13 @@ local kp = local manifests = // Uncomment line below to enable vertical auto scaling of kube-state-metrics //{ ['ksm-autoscaler-' + name]: kp.ksmAutoscaler[name] for name in std.objectFields(kp.ksmAutoscaler) } + - { ['00namespace-' + name]: kp.kubePrometheus[name] for name in std.objectFields(kp.kubePrometheus) } + - { ['0prometheus-operator-' + name]: kp.prometheusOperator[name] for name in std.objectFields(kp.prometheusOperator) } + + { ['setup/0namespace-' + name]: kp.kubePrometheus[name] for name in std.objectFields(kp.kubePrometheus) } + + { + ['setup/prometheus-operator-' + name]: kp.prometheusOperator[name] + for name in std.filter((function(name) name != 'serviceMonitor'), std.objectFields(kp.prometheusOperator)) + } + + // serviceMonitor is separated so that it can be created after the CRDs are ready + { 'prometheus-operator-serviceMonitor': kp.prometheusOperator.serviceMonitor } + { ['node-exporter-' + name]: kp.nodeExporter[name] for name in std.objectFields(kp.nodeExporter) } + { ['kube-state-metrics-' + name]: kp.kubeStateMetrics[name] for name in std.objectFields(kp.kubeStateMetrics) } + { ['alertmanager-' + name]: kp.alertmanager[name] for name in std.objectFields(kp.alertmanager) } + diff --git a/kustomization.yaml b/kustomization.yaml index a580ed8e6602627dd362095bc07356b0dc55d801..bd03a83e0008b4dc8340f71f5c1963a05b381f17 100644 --- a/kustomization.yaml +++ b/kustomization.yaml @@ -1,18 +1,6 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: -- ./manifests/00namespace-namespace.yaml -- ./manifests/0prometheus-operator-0alertmanagerCustomResourceDefinition.yaml -- ./manifests/0prometheus-operator-0podmonitorCustomResourceDefinition.yaml -- ./manifests/0prometheus-operator-0prometheusCustomResourceDefinition.yaml -- ./manifests/0prometheus-operator-0prometheusruleCustomResourceDefinition.yaml -- ./manifests/0prometheus-operator-0servicemonitorCustomResourceDefinition.yaml -- ./manifests/0prometheus-operator-clusterRole.yaml -- ./manifests/0prometheus-operator-clusterRoleBinding.yaml -- ./manifests/0prometheus-operator-deployment.yaml -- ./manifests/0prometheus-operator-service.yaml -- ./manifests/0prometheus-operator-serviceAccount.yaml -- ./manifests/0prometheus-operator-serviceMonitor.yaml - ./manifests/alertmanager-alertmanager.yaml - ./manifests/alertmanager-secret.yaml - ./manifests/alertmanager-service.yaml @@ -52,6 +40,7 @@ resources: - ./manifests/prometheus-adapter-serviceAccount.yaml - ./manifests/prometheus-clusterRole.yaml - ./manifests/prometheus-clusterRoleBinding.yaml +- ./manifests/prometheus-operator-serviceMonitor.yaml - ./manifests/prometheus-prometheus.yaml - ./manifests/prometheus-roleBindingConfig.yaml - ./manifests/prometheus-roleBindingSpecificNamespaces.yaml @@ -66,3 +55,14 @@ resources: - ./manifests/prometheus-serviceMonitorKubeControllerManager.yaml - ./manifests/prometheus-serviceMonitorKubeScheduler.yaml - ./manifests/prometheus-serviceMonitorKubelet.yaml +- ./manifests/setup/0namespace-namespace.yaml +- ./manifests/setup/prometheus-operator-0alertmanagerCustomResourceDefinition.yaml +- ./manifests/setup/prometheus-operator-0podmonitorCustomResourceDefinition.yaml +- ./manifests/setup/prometheus-operator-0prometheusCustomResourceDefinition.yaml +- ./manifests/setup/prometheus-operator-0prometheusruleCustomResourceDefinition.yaml +- ./manifests/setup/prometheus-operator-0servicemonitorCustomResourceDefinition.yaml +- ./manifests/setup/prometheus-operator-clusterRole.yaml +- ./manifests/setup/prometheus-operator-clusterRoleBinding.yaml +- ./manifests/setup/prometheus-operator-deployment.yaml +- ./manifests/setup/prometheus-operator-service.yaml +- ./manifests/setup/prometheus-operator-serviceAccount.yaml diff --git a/manifests/0prometheus-operator-serviceMonitor.yaml b/manifests/prometheus-operator-serviceMonitor.yaml similarity index 100% rename from manifests/0prometheus-operator-serviceMonitor.yaml rename to manifests/prometheus-operator-serviceMonitor.yaml diff --git a/manifests/00namespace-namespace.yaml b/manifests/setup/0namespace-namespace.yaml similarity index 100% rename from manifests/00namespace-namespace.yaml rename to manifests/setup/0namespace-namespace.yaml diff --git a/manifests/0prometheus-operator-0alertmanagerCustomResourceDefinition.yaml b/manifests/setup/prometheus-operator-0alertmanagerCustomResourceDefinition.yaml similarity index 100% rename from manifests/0prometheus-operator-0alertmanagerCustomResourceDefinition.yaml rename to manifests/setup/prometheus-operator-0alertmanagerCustomResourceDefinition.yaml diff --git a/manifests/0prometheus-operator-0podmonitorCustomResourceDefinition.yaml b/manifests/setup/prometheus-operator-0podmonitorCustomResourceDefinition.yaml similarity index 100% rename from manifests/0prometheus-operator-0podmonitorCustomResourceDefinition.yaml rename to manifests/setup/prometheus-operator-0podmonitorCustomResourceDefinition.yaml diff --git a/manifests/0prometheus-operator-0prometheusCustomResourceDefinition.yaml b/manifests/setup/prometheus-operator-0prometheusCustomResourceDefinition.yaml similarity index 100% rename from manifests/0prometheus-operator-0prometheusCustomResourceDefinition.yaml rename to manifests/setup/prometheus-operator-0prometheusCustomResourceDefinition.yaml diff --git a/manifests/0prometheus-operator-0prometheusruleCustomResourceDefinition.yaml b/manifests/setup/prometheus-operator-0prometheusruleCustomResourceDefinition.yaml similarity index 100% rename from manifests/0prometheus-operator-0prometheusruleCustomResourceDefinition.yaml rename to manifests/setup/prometheus-operator-0prometheusruleCustomResourceDefinition.yaml diff --git a/manifests/0prometheus-operator-0servicemonitorCustomResourceDefinition.yaml b/manifests/setup/prometheus-operator-0servicemonitorCustomResourceDefinition.yaml similarity index 100% rename from manifests/0prometheus-operator-0servicemonitorCustomResourceDefinition.yaml rename to manifests/setup/prometheus-operator-0servicemonitorCustomResourceDefinition.yaml diff --git a/manifests/0prometheus-operator-clusterRole.yaml b/manifests/setup/prometheus-operator-clusterRole.yaml similarity index 100% rename from manifests/0prometheus-operator-clusterRole.yaml rename to manifests/setup/prometheus-operator-clusterRole.yaml diff --git a/manifests/0prometheus-operator-clusterRoleBinding.yaml b/manifests/setup/prometheus-operator-clusterRoleBinding.yaml similarity index 100% rename from manifests/0prometheus-operator-clusterRoleBinding.yaml rename to manifests/setup/prometheus-operator-clusterRoleBinding.yaml diff --git a/manifests/0prometheus-operator-deployment.yaml b/manifests/setup/prometheus-operator-deployment.yaml similarity index 100% rename from manifests/0prometheus-operator-deployment.yaml rename to manifests/setup/prometheus-operator-deployment.yaml diff --git a/manifests/0prometheus-operator-service.yaml b/manifests/setup/prometheus-operator-service.yaml similarity index 100% rename from manifests/0prometheus-operator-service.yaml rename to manifests/setup/prometheus-operator-service.yaml diff --git a/manifests/0prometheus-operator-serviceAccount.yaml b/manifests/setup/prometheus-operator-serviceAccount.yaml similarity index 100% rename from manifests/0prometheus-operator-serviceAccount.yaml rename to manifests/setup/prometheus-operator-serviceAccount.yaml diff --git a/scripts/minikube-start-kvm.sh b/scripts/minikube-start-kvm.sh new file mode 100755 index 0000000000000000000000000000000000000000..da82a95cb1960567a69657dc0751f9bacc76510d --- /dev/null +++ b/scripts/minikube-start-kvm.sh @@ -0,0 +1,12 @@ +#!/bin/bash +minikube delete +minikube addons disable metrics-server +minikube start \ + --vm-driver=kvm2 \ + --kubernetes-version=v1.16.0 \ + --memory=6g \ + --bootstrapper=kubeadm \ + --extra-config=kubelet.authentication-token-webhook=true \ + --extra-config=kubelet.authorization-mode=Webhook \ + --extra-config=scheduler.address=0.0.0.0 \ + --extra-config=controller-manager.address=0.0.0.0 diff --git a/scripts/minikube-start.sh b/scripts/minikube-start.sh new file mode 100755 index 0000000000000000000000000000000000000000..eb29e860cf2a8419ea04e91b92707b12e5ed7e9d --- /dev/null +++ b/scripts/minikube-start.sh @@ -0,0 +1,11 @@ +#!/bin/bash +minikube delete +minikube addons disable metrics-server +minikube start \ + --kubernetes-version=v1.16.0 \ + --memory=6g \ + --bootstrapper=kubeadm \ + --extra-config=kubelet.authentication-token-webhook=true \ + --extra-config=kubelet.authorization-mode=Webhook \ + --extra-config=scheduler.address=0.0.0.0 \ + --extra-config=controller-manager.address=0.0.0.0 diff --git a/scripts/monitoring-deploy.sh b/scripts/monitoring-deploy.sh new file mode 100755 index 0000000000000000000000000000000000000000..7bc20905c658802d568c6535af4f80f784b656f3 --- /dev/null +++ b/scripts/monitoring-deploy.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +# create namespace and CRDs +kubectl create -f manifests/setup + +# wait for CRD creation to complete +until kubectl get servicemonitors --all-namespaces ; do date; sleep 1; echo ""; done + +# create monitoring components +kubectl create -f manifests/ + diff --git a/tests/e2e/travis-e2e.sh b/tests/e2e/travis-e2e.sh index aa32d668c2e8cf39524d9471ddd7be9bd29b7ff4..4efb2bd17d9f6abd6969f9e5089244f8adcf59f1 100755 --- a/tests/e2e/travis-e2e.sh +++ b/tests/e2e/travis-e2e.sh @@ -16,13 +16,13 @@ chmod +x kind ./kind create cluster export KUBECONFIG="$(./kind get kubeconfig-path)" -./kubectl apply -f manifests/0prometheus-operator-0alertmanagerCustomResourceDefinition.yaml -./kubectl apply -f manifests/0prometheus-operator-0prometheusCustomResourceDefinition.yaml -./kubectl apply -f manifests/0prometheus-operator-0prometheusruleCustomResourceDefinition.yaml -./kubectl apply -f manifests/0prometheus-operator-0servicemonitorCustomResourceDefinition.yaml +# create namespace, permissions, and CRDs +./kubectl create -f manifests/setup -# Wait for CRDs to be successfully registered -sleep 10 +# wait for CRD creation to complete +until ./kubectl get servicemonitors --all-namespaces ; do date; sleep 1; echo ""; done + +# create monitoring components +./kubectl create -f manifests/ -./kubectl apply -f manifests make test-e2e