Merge pull request #1458 from paulfantom/grafana-mixin

jsonnet/kube-prometheus/components/grafana.libsonnet

@@ -24,6 +24,12 @@ local defaults = {
     if !std.setMember(labelName, ['app.kubernetes.io/version'])
   },
   prometheusName:: error 'must provide prometheus name',
+  mixin: {
+    ruleLabels: {},
+    _config: {
+      runbookURLPattern: 'https://runbooks.prometheus-operator.dev/runbooks/grafana/%s',
+    },
+  },
 };
 
 function(params)
@@ -40,6 +46,27 @@ function(params)
       labels: g._config.commonLabels,
     },
 
+    mixin::
+      (import 'github.com/grafana/grafana/grafana-mixin/mixin.libsonnet') +
+      (import 'github.com/kubernetes-monitoring/kubernetes-mixin/lib/add-runbook-links.libsonnet') + {
+        _config+:: g._config.mixin._config,
+      },
+
+    prometheusRule: {
+      apiVersion: 'monitoring.coreos.com/v1',
+      kind: 'PrometheusRule',
+      metadata: {
+        labels: g._config.commonLabels + g._config.mixin.ruleLabels,
+        name: g._config.name + '-rules',
+        namespace: g._config.namespace,
+      },
+      spec: {
+        local r = if std.objectHasAll(g.mixin, 'prometheusRules') then g.mixin.prometheusRules.groups else [],
+        local a = if std.objectHasAll(g.mixin, 'prometheusAlerts') then g.mixin.prometheusAlerts.groups else [],
+        groups: a + r,
+      },
+    },
+
     serviceMonitor: {
       apiVersion: 'monitoring.coreos.com/v1',
       kind: 'ServiceMonitor',

jsonnet/kube-prometheus/jsonnetfile.json

@@ -10,6 +10,16 @@
       },
       "version": "master"
     },
+    {
+      "source": {
+        "git": {
+          "remote": "https://github.com/grafana/grafana",
+          "subdir": "grafana-mixin"
+        }
+      },
+      "version": "main",
+      "name": "grafana-mixin"
+    },
     {
       "source": {
         "git": {

jsonnet/kube-prometheus/main.libsonnet

@@ -70,7 +70,12 @@ local utils = import './lib/utils.libsonnet';
       image: $.values.common.images.grafana,
       prometheusName: $.values.prometheus.name,
       // TODO(paulfantom) This should be done by iterating over all objects and looking for object.mixin.grafanaDashboards
-      dashboards: $.nodeExporter.mixin.grafanaDashboards + $.prometheus.mixin.grafanaDashboards + $.kubernetesControlPlane.mixin.grafanaDashboards + $.alertmanager.mixin.grafanaDashboards,
+      dashboards: $.nodeExporter.mixin.grafanaDashboards +
+                  $.prometheus.mixin.grafanaDashboards +
+                  $.kubernetesControlPlane.mixin.grafanaDashboards +
+                  $.alertmanager.mixin.grafanaDashboards +
+                  $.grafana.mixin.grafanaDashboards,
+      mixin+: { ruleLabels: $.values.common.ruleLabels },
     },
     kubeStateMetrics: {
       namespace: $.values.common.namespace,

jsonnetfile.lock.json

@@ -21,6 +21,16 @@
       "version": "69279532f46cb7491bb60a588be534100fdc058d",
       "sum": "cdKL5kPYfpWSpTCu4qctmh+gWQqL+4YWom6rw9qLYJU="
     },
+    {
+      "source": {
+        "git": {
+          "remote": "https://github.com/grafana/grafana.git",
+          "subdir": "grafana-mixin"
+        }
+      },
+      "version": "88bc47441f75b5956023be0b1f79c371420cdcdd",
+      "sum": "MkjR7zCgq6MUZgjDzop574tFKoTX2OBr7DTwm1K+Ofs="
+    },
     {
       "source": {
         "git": {

kustomization.yaml

@@ -20,6 +20,7 @@ resources:
 - ./manifests/grafana-dashboardDefinitions.yaml
 - ./manifests/grafana-dashboardSources.yaml
 - ./manifests/grafana-deployment.yaml
+- ./manifests/grafana-prometheusRule.yaml
 - ./manifests/grafana-service.yaml
 - ./manifests/grafana-serviceAccount.yaml
 - ./manifests/grafana-serviceMonitor.yaml

manifests/grafana-deployment.yaml

@@ -67,6 +67,9 @@ spec:
         - mountPath: /grafana-dashboard-definitions/0/controller-manager
           name: grafana-dashboard-controller-manager
           readOnly: false
+        - mountPath: /grafana-dashboard-definitions/0/grafana-overview
+          name: grafana-dashboard-grafana-overview
+          readOnly: false
         - mountPath: /grafana-dashboard-definitions/0/k8s-resources-cluster
           name: grafana-dashboard-k8s-resources-cluster
           readOnly: false
@@ -155,6 +158,9 @@ spec:
       - configMap:
           name: grafana-dashboard-controller-manager
         name: grafana-dashboard-controller-manager
+      - configMap:
+          name: grafana-dashboard-grafana-overview
+        name: grafana-dashboard-grafana-overview
       - configMap:
           name: grafana-dashboard-k8s-resources-cluster
         name: grafana-dashboard-k8s-resources-cluster

manifests/grafana-prometheusRule.yaml

+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+  labels:
+    app.kubernetes.io/component: grafana
+    app.kubernetes.io/name: grafana
+    app.kubernetes.io/part-of: kube-prometheus
+    app.kubernetes.io/version: 8.3.3
+    prometheus: k8s
+    role: alert-rules
+  name: grafana-rules
+  namespace: monitoring
+spec:
+  groups:
+  - name: GrafanaAlerts
+    rules:
+    - alert: GrafanaRequestsFailing
+      annotations:
+        message: '{{ $labels.namespace }}/{{ $labels.job }}/{{ $labels.handler }}
+          is experiencing {{ $value | humanize }}% errors'
+        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/grafana/grafanarequestsfailing
+      expr: |
+        100 * namespace_job_handler_statuscode:grafana_http_request_duration_seconds_count:rate5m{handler!~"/api/datasources/proxy/:id.*|/api/ds/query|/api/tsdb/query", status_code=~"5.."}
+        / ignoring (status_code)
+        sum without (status_code) (namespace_job_handler_statuscode:grafana_http_request_duration_seconds_count:rate5m{handler!~"/api/datasources/proxy/:id.*|/api/ds/query|/api/tsdb/query"})
+        > 50
+      for: 5m
+      labels:
+        severity: warning
+  - name: grafana_rules
+    rules:
+    - expr: |
+        sum by (namespace, job, handler, status_code) (rate(grafana_http_request_duration_seconds_count[5m]))
+      record: namespace_job_handler_statuscode:grafana_http_request_duration_seconds_count:rate5m