From c942d6b837c652b31249957a8f7508616741700c Mon Sep 17 00:00:00 2001
From: Alok Kumar Singh <alok.singh@practo.com>
Date: Wed, 4 Mar 2020 06:47:38 +0530
Subject: [PATCH] Example with option to modify alert thresholds
Review comment: https://github.com/coreos/kube-prometheus/pull/425#discussion_r387494885
---
docs/weave-net-support.md | 37 ++++++++++++++++++++++++++++---
examples/weavenet-example.jsonnet | 34 ++++++++++++++++++++++++++--
2 files changed, 66 insertions(+), 5 deletions(-)
diff --git a/docs/weave-net-support.md b/docs/weave-net-support.md
index 8090e058..22fcfdbb 100644
--- a/docs/weave-net-support.md
+++ b/docs/weave-net-support.md
@@ -13,18 +13,49 @@ Using kube-prometheus and kubectl you will be able install the following for mon
5. [Grafana Dashboard for WeaveNet(Cluster)](https://grafana.com/grafana/dashboards/11789) This will setup the cluster level monitoring for weave net.
## Instructions
-- You can monitor weave-net CNI using kube-prometheus with:
+- You can monitor weave-net CNI using an example like below. **Please note that some alert configurations are environment specific and may require modifications of alert thresholds**. For example: The FastDP flows have never gone below 1500 for us. But if this value is say 2000 for you then you can use an example like below to update the alert. The alerts which may require threshold modifications are `WeaveNetFastDPFlowsLow` and `WeaveNetIPAMUnreachable`.
+
[embedmd]:# (../examples/weavenet-example.jsonnet)
```jsonnet
local kp = (import 'kube-prometheus/kube-prometheus.libsonnet') +
- (import 'kube-prometheus/kube-prometheus-weavenet.libsonnet');
+ (import 'kube-prometheus/kube-prometheus-weavenet.libsonnet') + {
+ _config+:: {
+ namespace: 'monitoring',
+ },
+ prometheusAlerts+:: {
+ groups: std.map(
+ function(group)
+ if group.name == 'weave-net' then
+ group {
+ rules: std.map(function(rule)
+ if rule.alert == "WeaveNetFastDPFlowsLow" then
+ rule {
+ expr: "sum(weave_flows) < 2000"
+ }
+ else if rule.alert == "WeaveNetIPAMUnreachable" then
+ rule {
+ expr: "weave_ipam_unreachable_percentage > 25"
+ }
+ else
+ rule
+ ,
+ group.rules
+ )
+ }
+ else
+ group,
+ super.groups
+ ),
+ },
+};
{ ['00namespace-' + name]: kp.kubePrometheus[name] for name in std.objectFields(kp.kubePrometheus) } +
{ ['0prometheus-operator-' + name]: kp.prometheusOperator[name] for name in std.objectFields(kp.prometheusOperator) } +
{ ['node-exporter-' + name]: kp.nodeExporter[name] for name in std.objectFields(kp.nodeExporter) } +
{ ['kube-state-metrics-' + name]: kp.kubeStateMetrics[name] for name in std.objectFields(kp.kubeStateMetrics) } +
{ ['prometheus-' + name]: kp.prometheus[name] for name in std.objectFields(kp.prometheus) } +
-{ ['prometheus-adapter-' + name]: kp.prometheusAdapter[name] for name in std.objectFields(kp.prometheusAdapter) }
+{ ['prometheus-adapter-' + name]: kp.prometheusAdapter[name] for name in std.objectFields(kp.prometheusAdapter) } +
+{ ['grafana-' + name]: kp.grafana[name] for name in std.objectFields(kp.grafana) }
```
- After you have the required yamls file please run
diff --git a/examples/weavenet-example.jsonnet b/examples/weavenet-example.jsonnet
index d501d67c..68885411 100644
--- a/examples/weavenet-example.jsonnet
+++ b/examples/weavenet-example.jsonnet
@@ -1,9 +1,39 @@
local kp = (import 'kube-prometheus/kube-prometheus.libsonnet') +
- (import 'kube-prometheus/kube-prometheus-weavenet.libsonnet');
+ (import 'kube-prometheus/kube-prometheus-weavenet.libsonnet') + {
+ _config+:: {
+ namespace: 'monitoring',
+ },
+ prometheusAlerts+:: {
+ groups: std.map(
+ function(group)
+ if group.name == 'weave-net' then
+ group {
+ rules: std.map(function(rule)
+ if rule.alert == "WeaveNetFastDPFlowsLow" then
+ rule {
+ expr: "sum(weave_flows) < 2000"
+ }
+ else if rule.alert == "WeaveNetIPAMUnreachable" then
+ rule {
+ expr: "weave_ipam_unreachable_percentage > 25"
+ }
+ else
+ rule
+ ,
+ group.rules
+ )
+ }
+ else
+ group,
+ super.groups
+ ),
+ },
+};
{ ['00namespace-' + name]: kp.kubePrometheus[name] for name in std.objectFields(kp.kubePrometheus) } +
{ ['0prometheus-operator-' + name]: kp.prometheusOperator[name] for name in std.objectFields(kp.prometheusOperator) } +
{ ['node-exporter-' + name]: kp.nodeExporter[name] for name in std.objectFields(kp.nodeExporter) } +
{ ['kube-state-metrics-' + name]: kp.kubeStateMetrics[name] for name in std.objectFields(kp.kubeStateMetrics) } +
{ ['prometheus-' + name]: kp.prometheus[name] for name in std.objectFields(kp.prometheus) } +
-{ ['prometheus-adapter-' + name]: kp.prometheusAdapter[name] for name in std.objectFields(kp.prometheusAdapter) }
+{ ['prometheus-adapter-' + name]: kp.prometheusAdapter[name] for name in std.objectFields(kp.prometheusAdapter) } +
+{ ['grafana-' + name]: kp.grafana[name] for name in std.objectFields(kp.grafana) }
--
GitLab