diff --git a/hack/grafana-dashboards-configmap-generator/bin/grafana_dashboards_generate.sh b/hack/grafana-dashboards-configmap-generator/bin/grafana_dashboards_generate.sh index 1173a998d5e289aaa1b1ca799b55c42dc4cf18ec..91bcd4288c36002765299596242bbaf95111820d 100755 --- a/hack/grafana-dashboards-configmap-generator/bin/grafana_dashboards_generate.sh +++ b/hack/grafana-dashboards-configmap-generator/bin/grafana_dashboards_generate.sh @@ -16,8 +16,12 @@ set -u # Based on a configurable size limit, the tool will create 1 or N configmaps # to allocate the .json resources (bin packing) +# Update: 20170914 +# The tool also generates a grafana deployment manifest (-g option) + # parameters # -o, --output-file +# -g, --grafana-manifest-file # -i, --input-dir # -s, --size-limit # -x, --apply-configmap : true or false (default = false) @@ -66,15 +70,18 @@ DATE_EXEC="$(date "+%Y-%m-%d-%H%M%S")" BIN_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" TOOL_HOME="$(dirname $BIN_DIR)" SCRIPT_BASE=`basename $0 | sed "s/\.[Ss][Hh]//"` +CONFIGMAP_DASHBOARD_PREFIX="grafana-dashboards" TEMPLATES_DIR="$TOOL_HOME/templates" DASHBOARD_HEADER_FILE="$TEMPLATES_DIR/dashboard.header" DASHBOARD_FOOT_FILE="$TEMPLATES_DIR/dashboard.foot" CONFIGMAP_HEADER="$TEMPLATES_DIR/ConfigMap.header" +GRAFANA_DEPLOYMENT_TEMPLATE="$TEMPLATES_DIR/grafana-deployment-template.yaml" OUTPUT_BASE_DIR="$TOOL_HOME/output" # Some default values OUTPUT_FILE="$OUTPUT_BASE_DIR/grafana-dashboards-configMap-$DATE_EXEC.yaml" +GRAFANA_OUTPUT_FILE="$OUTPUT_BASE_DIR/grafana-deployment-$DATE_EXEC.yaml" DASHBOARDS_DIR="$TEMPLATES_DIR/grafana-dashboards" APPLY_CONFIGMAP="false" @@ -89,6 +96,10 @@ while (( "$#" )); do OUTPUT_FILE="$2" shift ;; + "-g" | "--grafana-output-file") + GRAFANA_OUTPUT_FILE="$2" + shift + ;; "-i" | "--input-dir") DASHBOARDS_DIR="$2" shift @@ -125,6 +136,26 @@ while (( "$#" )); do shift done +# +# Auxiliary Functions +# +indentMultiLineString() { + # Indent a given string (in one line including multiple \n) + test "$#" -eq 2 || { echo "INTERNAL ERROR: wrong call to function indentMultiLineString"; exit 1; } + local indent_number="$1" + local string="$2" + + test "$indent_number" -ge 0 || { echo "INTERNAL ERROR: wrong indent number parameter: $indent_number"; exit 1; } + + # prepare indentation text + local indent_string="" + for (( c=0; c<$indent_number; c++ )); do + indent_string="$indent_string " + done + + echo "$string" | sed -e "s#^#$indent_string#" -e "s#\\\n#\\\n$indent_string#g" +} + # # Main Functions # @@ -135,7 +166,7 @@ addConfigMapHeader() { test "$#" -eq 1 && local id="$1" || local id="" if [ "$id" ]; then - cat "$CONFIGMAP_HEADER" | sed "s/name: grafana-dashboards/name: grafana-dashboards-$id/" + cat "$CONFIGMAP_HEADER" | sed "s/name: $CONFIGMAP_DASHBOARD_PREFIX/name: $CONFIGMAP_DASHBOARD_PREFIX-$id/" else cat "$CONFIGMAP_HEADER" fi @@ -247,12 +278,23 @@ bin-pack-files() { IFS=$OLDIFS } +# prepareGrafanaDeploymentManifest() { +# local num_configmaps="$1" +# +# for (( i=0; i<$total_configmaps_created; i++ )); do +# echo "Creating deployment for $CONFIGMAP_DASHBOARD_PREFIX-$i" +# +# done +# } + + # Some variables checks... test ! -d "$TEMPLATES_DIR" && { echo "ERROR: missing templates directory $TEMPLATES_DIR"; exit 1; } test -f "$DASHBOARD_FOOT_FILE" || { echo "Template $DASHBOARD_FOOT_FILE not found"; exit 1; } test -f "$DASHBOARD_HEADER_FILE" || { echo "Template $DASHBOARD_HEADER_FILE not found"; exit 1; } test -f "$CONFIGMAP_HEADER" || { echo "Template $CONFIGMAP_HEADER not found"; exit 1; } +test -f "$GRAFANA_DEPLOYMENT_TEMPLATE" || { echo "Template $GRAFANA_DEPLOYMENT_TEMPLATE not found"; exit 1; } test ! -d "$OUTPUT_BASE_DIR" && { echo "ERROR: missing directory $OUTPUT_BASE_DIR"; exit 1; } @@ -260,7 +302,9 @@ test ! -d "$OUTPUT_BASE_DIR" && { echo "ERROR: missing directory $OUTPUT_BASE_DI test -d "$DASHBOARDS_DIR" || { echo "ERROR: Dashboards directory not found: $DASHBOARDS_DIR"; echoSyntax; exit 1; } test -f "$OUTPUT_FILE" && { echo "ERROR: Output file already exists: $OUTPUT_FILE"; exit 1; } +test -f "$GRAFANA_OUTPUT_FILE" && { echo "ERROR: Output file already exists: $GRAFANA_OUTPUT_FILE"; exit 1; } touch $OUTPUT_FILE || { echo "ERROR: Unable to create or modify $OUTPUT_FILE"; exit 1; } +touch $GRAFANA_OUTPUT_FILE || { echo "ERROR: Unable to create or modify $GRAFANA_OUTPUT_FILE"; exit 1; } # Main code start @@ -269,6 +313,8 @@ echo "# Configured size limit: $DATA_SIZE_LIMIT bytes" echo "# Grafna input dashboards and datasources will be read from: $DASHBOARDS_DIR" echo "# Grafana Dashboards ConfigMap will be created into file:" echo "$OUTPUT_FILE" +echo "# Grafana Deployment manifest will be created into file:" +echo "$GRAFANA_OUTPUT_FILE" echo # Loop variables initialization @@ -286,7 +332,7 @@ if [ "$to_process" ]; then echo echo "# Size limit not reached ($bytes_to_process). Adding all files into basic configmap" echo - addConfigMapHeader >> $OUTPUT_FILE || { echo "ERROR in call to addConfigMapHeader function"; exit 1; } + addConfigMapHeader $n >> $OUTPUT_FILE || { echo "ERROR in call to addConfigMapHeader function"; exit 1; } else echo echo "# Size limit not reached ($bytes_to_process). Adding remaining files into configmap with id $n" @@ -302,24 +348,50 @@ echo "# Process completed, configmap created: $(basename $OUTPUT_FILE)" echo "# Summary" echo "# Total files processed: $total_files_processed" echo "# Total amount of ConfigMaps inside the manifest: $total_configmaps_created" +echo +# Grafana deployment Processing (for every configmap) +#prepareGrafanaDeploymentManifest "$total_configmaps_created" +VOLUMES="" +VOLUME_MOUNTS="" +WATCH_DIR="" +for (( i=0; i<$total_configmaps_created; i++ )); do + configmap="$CONFIGMAP_DASHBOARD_PREFIX-$i" + echo "# Preparing grafana deployment to support configmap: $configmap" + + test "$VOLUME_MOUNTS" && VOLUME_MOUNTS="$VOLUME_MOUNTS\n- name: $configmap\n mountPath: /var/$configmap" || VOLUME_MOUNTS="- name: $configmap\n mountPath: /var/$configmap" + test "$VOLUMES" && VOLUMES="$VOLUMES\n- name: $configmap\n configMap:\n name: $configmap" || VOLUMES="- name: $configmap\n configMap:\n name: $configmap" + test "$WATCH_DIR" && WATCH_DIR="$WATCH_DIR\n- '--watch-dir=/var/$configmap'" || WATCH_DIR="- '--watch-dir=/var/$configmap'" + # echo "DEBUG:" + # echo "VOLUMES: $VOLUMES" + # echo "VOLUME_MOUNTS: $VOLUME_MOUNTS" + # echo "WATCH_DIR: $WATCH_DIR" + echo +done + +echo "# Processing grafana deployment template into $GRAFANA_OUTPUT_FILE" +sed -e "s#XXX_VOLUMES_XXX#$(indentMultiLineString 6 "$VOLUMES")#" \ + -e "s#XXX_VOLUME_MOUNTS_XXX#$(indentMultiLineString 8 "$VOLUME_MOUNTS")#" \ + -e "s#XXX_WATCH_DIR_XXX#$(indentMultiLineString 10 "$WATCH_DIR")#" \ + $GRAFANA_DEPLOYMENT_TEMPLATE > $GRAFANA_OUTPUT_FILE # If output file is empty we can delete it and exit test ! -s "$OUTPUT_FILE" && { echo "# Configmap empty, deleting file"; rm $OUTPUT_FILE; exit 0; } +test ! -s "$GRAFANA_OUTPUT_FILE" && { echo "# Configmap empty, deleting file"; rm $GRAFANA_OUTPUT_FILE; exit 0; } if [ "$APPLY_CONFIGMAP" = "true" ]; then test -x "$(which kubectl)" || { echo "ERROR: kubectl command not available. Apply configmap not possible"; exit 1; } - echo + echo "# Applying configuration with $APPLY_TYPE method on namespace $NAMESPACE" if kubectl -n $NAMESPACE $APPLY_TYPE -f "$OUTPUT_FILE"; then echo - echo "# ConfigMap updated. Wait until grafana-watcher applies the changes and reloads the dashboards." + echo "# ConfigMap updated. Updating grafana deployment" + kubectl -n $NAMESPACE $APPLY_TYPE -f "$GRAFANA_OUTPUT_FILE" || { echo "Error applying Grafana deployment. Check yaml file: $GRAFANA_OUTPUT_FILE"; exit 1; } else - echo - echo "ERROR APPLYING CONFIGURATION. Check yaml file" - echo "$OUTPUT_FILE" + echo "Error applying Configmap. Check yaml file: $OUTPUT_FILE" fi else echo echo "# To apply the new configMap to your k8s system do something like:" - echo "kubectl -n monitoring $APPLY_TYPE -f $(basename $OUTPUT_FILE)" + echo "kubectl -n monitoring $APPLY_TYPE -f $OUTPUT_FILE" + echo "kubectl -n monitoring $APPLY_TYPE -f $GRAFANA_OUTPUT_FILE" echo fi diff --git a/hack/grafana-dashboards-configmap-generator/output/README.md b/hack/grafana-dashboards-configmap-generator/output/README.md index 47e06b894315e239dea8c86adea2906f591383da..d2e3c8a66a3c8d261716abfea0fd5f3f8bef0312 100644 --- a/hack/grafana-dashboards-configmap-generator/output/README.md +++ b/hack/grafana-dashboards-configmap-generator/output/README.md @@ -1,2 +1 @@ -### By default, this directory will include all generated manifests (if -o is not used) -### With -o, --output-file option we can force the tool to generate a file wherever we want +### This directory will include all generated manifests if no specific options are given diff --git a/hack/grafana-dashboards-configmap-generator/templates/grafana-deployment-original.yaml b/hack/grafana-dashboards-configmap-generator/templates/grafana-deployment-original.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1666deaee1141a489eab265a274830f51d14559f --- /dev/null +++ b/hack/grafana-dashboards-configmap-generator/templates/grafana-deployment-original.yaml @@ -0,0 +1,74 @@ +apiVersion: apps/v1beta1 +kind: Deployment +metadata: + name: grafana +spec: + replicas: 1 + template: + metadata: + labels: + app: grafana + spec: + containers: + - name: grafana + image: grafana/grafana:4.4.1 + env: + - name: GF_AUTH_BASIC_ENABLED + value: "true" + - name: GF_AUTH_ANONYMOUS_ENABLED + value: "true" + - name: GF_SECURITY_ADMIN_USER + valueFrom: + secretKeyRef: + name: grafana-credentials + key: user + - name: GF_SECURITY_ADMIN_PASSWORD + valueFrom: + secretKeyRef: + name: grafana-credentials + key: password + volumeMounts: + - name: grafana-storage + mountPath: /var/grafana-storage + ports: + - name: web + containerPort: 3000 + resources: + requests: + memory: 100Mi + cpu: 100m + limits: + memory: 200Mi + cpu: 200m + - name: grafana-watcher + image: quay.io/coreos/grafana-watcher:v0.0.8 + args: + - '--watch-dir=/var/grafana-dashboards' + - '--grafana-url=http://localhost:3000' + env: + - name: GRAFANA_USER + valueFrom: + secretKeyRef: + name: grafana-credentials + key: user + - name: GRAFANA_PASSWORD + valueFrom: + secretKeyRef: + name: grafana-credentials + key: password + resources: + requests: + memory: "16Mi" + cpu: "50m" + limits: + memory: "32Mi" + cpu: "100m" + volumeMounts: + - name: grafana-dashboards + mountPath: /var/grafana-dashboards + volumes: + - name: grafana-storage + emptyDir: {} + - name: grafana-dashboards + configMap: + name: grafana-dashboards diff --git a/hack/grafana-dashboards-configmap-generator/templates/grafana-deployment-template.yaml b/hack/grafana-dashboards-configmap-generator/templates/grafana-deployment-template.yaml new file mode 100644 index 0000000000000000000000000000000000000000..844ecbc7aa032d7760cbe158af4e9b5e7cd854b5 --- /dev/null +++ b/hack/grafana-dashboards-configmap-generator/templates/grafana-deployment-template.yaml @@ -0,0 +1,71 @@ +apiVersion: apps/v1beta1 +kind: Deployment +metadata: + name: grafana +spec: + replicas: 1 + template: + metadata: + labels: + app: grafana + spec: + containers: + - name: grafana + image: grafana/grafana:4.4.1 + env: + - name: GF_AUTH_BASIC_ENABLED + value: "true" + - name: GF_AUTH_ANONYMOUS_ENABLED + value: "true" + - name: GF_SECURITY_ADMIN_USER + valueFrom: + secretKeyRef: + name: grafana-credentials + key: user + - name: GF_SECURITY_ADMIN_PASSWORD + valueFrom: + secretKeyRef: + name: grafana-credentials + key: password + volumeMounts: + - name: grafana-storage + mountPath: /var/grafana-storage + ports: + - name: web + containerPort: 3000 + resources: + requests: + memory: 100Mi + cpu: 100m + limits: + memory: 200Mi + cpu: 200m + - name: grafana-watcher + image: quay.io/coreos/grafana-watcher:v0.0.8 + args: +XXX_WATCH_DIR_XXX + - '--grafana-url=http://localhost:3000' + env: + - name: GRAFANA_USER + valueFrom: + secretKeyRef: + name: grafana-credentials + key: user + - name: GRAFANA_PASSWORD + valueFrom: + secretKeyRef: + name: grafana-credentials + key: password + resources: + requests: + memory: "16Mi" + cpu: "50m" + limits: + memory: "32Mi" + cpu: "100m" + volumeMounts: +XXX_VOLUME_MOUNTS_XXX + volumes: + - name: grafana-storage + emptyDir: {} +XXX_VOLUMES_XXX diff --git a/hack/scripts/generate-manifests.sh b/hack/scripts/generate-manifests.sh index 64287dae2d06288ef149a131b270fb02801d80f2..bb0c0685695abf5b0a618d2a72683ea137ba0e02 100755 --- a/hack/scripts/generate-manifests.sh +++ b/hack/scripts/generate-manifests.sh @@ -10,8 +10,10 @@ hack/scripts/generate-rules-configmap.sh > manifests/prometheus/prometheus-k8s-r # Max Size per ConfigMap: 240000 # Input dir: assets/grafana # output file: manifests/grafana/grafana-dashboards.yaml +# grafana deployment output file: manifests/grafana/grafana-deployment.yaml test -f manifests/grafana/grafana-dashboards.yaml && rm -f manifests/grafana/grafana-dashboards.yaml -hack/grafana-dashboards-configmap-generator/bin/grafana_dashboards_generate.sh -s 240000 -i assets/grafana -o manifests/grafana/grafana-dashboards.yaml +test -f manifests/grafana/grafana-deployment.yaml && rm -f manifests/grafana/grafana-deployment.yaml +hack/grafana-dashboards-configmap-generator/bin/grafana_dashboards_generate.sh -s 240000 -i assets/grafana -o manifests/grafana/grafana-dashboards.yaml -g manifests/grafana/grafana-deployment.yaml # Generate Grafana Credentials Secret hack/scripts/generate-grafana-credentials-secret.sh admin admin > manifests/grafana/grafana-credentials.yaml diff --git a/manifests/grafana/grafana-dashboards.yaml b/manifests/grafana/grafana-dashboards.yaml index 38dfca6381a3bc782d0c3903f2c64935f38355c0..36ac037302e19d982cd55dbca443dc89567b118b 100644 --- a/manifests/grafana/grafana-dashboards.yaml +++ b/manifests/grafana/grafana-dashboards.yaml @@ -1,7 +1,7 @@ apiVersion: v1 kind: ConfigMap metadata: - name: grafana-dashboards + name: grafana-dashboards-0 data: deployment-dashboard.json: |+ { @@ -821,7 +821,7 @@ data: "timezone": "browser", "title": "Deployment", "version": 1 - }, + } , "inputs": [ { "name": "DS_PROMETHEUS", @@ -1882,7 +1882,7 @@ data: "timezone": "browser", "title": "Kubernetes Capacity Planing", "version": 4 - }, + } , "inputs": [ { "name": "DS_PROMETHEUS", @@ -2628,7 +2628,7 @@ data: "timezone": "", "title": "Kubernetes Cluster Health", "version": 12 - }, + } , "inputs": [ { "name": "DS_PROMETHEUS", @@ -3537,7 +3537,7 @@ data: "timezone": "", "title": "Kubernetes Cluster Status", "version": 22 - }, + } , "inputs": [ { "name": "DS_PROMETHEUS", @@ -4213,7 +4213,7 @@ data: "timezone": "", "title": "Kubernetes Control Plane Status", "version": 11 - }, + } , "inputs": [ { "name": "DS_PROMETHEUS", @@ -4660,7 +4660,7 @@ data: "timezone": "browser", "title": "Kubernetes Resource Requests", "version": 2 - }, + } , "inputs": [ { "name": "DS_PROMETHEUS", @@ -5565,7 +5565,7 @@ data: "timezone": "browser", "title": "Nodes", "version": 2 - }, + } , "inputs": [ { "name": "DS_PROMETHEUS", @@ -6010,7 +6010,7 @@ data: "timezone": "browser", "title": "Pods", "version": 1 - }, + } , "inputs": [ { "name": "DS_PROMETHEUS", diff --git a/manifests/grafana/grafana-deployment.yaml b/manifests/grafana/grafana-deployment.yaml index 1666deaee1141a489eab265a274830f51d14559f..a75724e59e1c0461900d0b8cd9a12dbfbf798c06 100644 --- a/manifests/grafana/grafana-deployment.yaml +++ b/manifests/grafana/grafana-deployment.yaml @@ -43,7 +43,7 @@ spec: - name: grafana-watcher image: quay.io/coreos/grafana-watcher:v0.0.8 args: - - '--watch-dir=/var/grafana-dashboards' + - '--watch-dir=/var/grafana-dashboards-0' - '--grafana-url=http://localhost:3000' env: - name: GRAFANA_USER @@ -64,11 +64,11 @@ spec: memory: "32Mi" cpu: "100m" volumeMounts: - - name: grafana-dashboards - mountPath: /var/grafana-dashboards + - name: grafana-dashboards-0 + mountPath: /var/grafana-dashboards-0 volumes: - name: grafana-storage emptyDir: {} - - name: grafana-dashboards + - name: grafana-dashboards-0 configMap: - name: grafana-dashboards + name: grafana-dashboards-0