diff --git a/jsonnet/kube-prometheus/components/k8s-control-plane.libsonnet b/jsonnet/kube-prometheus/components/k8s-control-plane.libsonnet
index 95ceb490208262ec0cb7469e5ce42dfccc50bedc..a771e95dbe6361e0b0b96532e8cd8c4c647b7294 100644
--- a/jsonnet/kube-prometheus/components/k8s-control-plane.libsonnet
+++ b/jsonnet/kube-prometheus/components/k8s-control-plane.libsonnet
@@ -11,6 +11,7 @@ local defaults = {
mixin:: {
ruleLabels: {},
_config: {
+ showMultiCluster: true,
cadvisorSelector: 'job="kubelet", metrics_path="/metrics/cadvisor"',
kubeletSelector: 'job="kubelet", metrics_path="/metrics"',
kubeStateMetricsSelector: 'job="kube-state-metrics"',
diff --git a/jsonnet/kube-prometheus/components/mixin/alerts/general.libsonnet b/jsonnet/kube-prometheus/components/mixin/alerts/general.libsonnet
index 8e69d98f6d8933d956f5a5c5acb926c7d9b16284..c236283747c118866010a5d45ddd2a72929d4403 100644
--- a/jsonnet/kube-prometheus/components/mixin/alerts/general.libsonnet
+++ b/jsonnet/kube-prometheus/components/mixin/alerts/general.libsonnet
@@ -10,7 +10,7 @@
summary: 'One or more targets are unreachable.',
description: '{{ printf "%.4g" $value }}% of the {{ $labels.job }}/{{ $labels.service }} targets in {{ $labels.namespace }} namespace are down.',
},
- expr: '100 * (count(up == 0) BY (job, namespace, service) / count(up) BY (job, namespace, service)) > 10',
+ expr: '100 * (count(up == 0) BY (cluster, job, namespace, service) / count(up) BY (cluster, job, namespace, service)) > 10',
'for': '10m',
labels: {
severity: 'warning',
diff --git a/jsonnet/kube-prometheus/components/prometheus-operator.libsonnet b/jsonnet/kube-prometheus/components/prometheus-operator.libsonnet
index d29d41eafc5ef1cdc2c7b68dbaabaec7caf24362..5c0c96c69c930d4a9a1ad7f29d3c3be94541cf16 100644
--- a/jsonnet/kube-prometheus/components/prometheus-operator.libsonnet
+++ b/jsonnet/kube-prometheus/components/prometheus-operator.libsonnet
@@ -38,6 +38,7 @@ local defaults = {
prometheus: defaults.name,
},
_config: {
+ groupLabels: 'cluster,controller,namespace',
prometheusOperatorSelector: 'job="prometheus-operator",namespace="' + defaults.namespace + '"',
runbookURLPattern: 'https://runbooks.prometheus-operator.dev/runbooks/prometheus-operator/%s',
},
diff --git a/manifests/grafana-dashboardDefinitions.yaml b/manifests/grafana-dashboardDefinitions.yaml
index 3aa24a3fbbdc8c5bfd52d7c090ae69d3339f7b39..fa980bdce8bef06617341ea3bd3368f73de9197c 100644
--- a/manifests/grafana-dashboardDefinitions.yaml
+++ b/manifests/grafana-dashboardDefinitions.yaml
@@ -2273,7 +2273,7 @@ items:
},
"datasource": "$datasource",
- "hide": 2,
+ "hide": 0,
"includeAll": false,
"label": "cluster",
"multi": false,
@@ -4170,7 +4170,7 @@ items:
},
"datasource": "$datasource",
- "hide": 2,
+ "hide": 0,
"includeAll": false,
"label": null,
"multi": false,
@@ -5323,7 +5323,7 @@ items:
},
"datasource": "$datasource",
- "hide": 2,
+ "hide": 0,
"includeAll": false,
"label": "cluster",
"multi": false,
@@ -9051,7 +9051,7 @@ items:
"value": ""
},
"datasource": "$datasource",
- "hide": 2,
+ "hide": 0,
"includeAll": false,
"label": null,
"multi": false,
@@ -9118,7 +9118,7 @@ items:
namespace: monitoring
- apiVersion: v1
data:
- k8s-resources-namespace.json: |-
+ k8s-resources-multicluster.json: |-
{
"annotations": {
"list": [
@@ -9175,12 +9175,12 @@ items:
],
"spaceLength": 10,
- "span": 3,
+ "span": 2,
"stack": false,
"steppedLine": false,
"targets": [
{
- "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\"}) / sum(kube_pod_container_resource_requests{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"cpu\"})",
+ "expr": "cluster:node_cpu:ratio_rate5m",
"format": "time_series",
"instant": true,
"intervalFactor": 2,
@@ -9190,7 +9190,7 @@ items:
"thresholds": "70,80",
"timeFrom": null,
"timeShift": null,
- "title": "CPU Utilisation (from requests)",
+ "title": "CPU Utilisation",
"tooltip": {
"shared": false,
"sort": 2,
@@ -9262,12 +9262,12 @@ items:
],
"spaceLength": 10,
- "span": 3,
+ "span": 2,
"stack": false,
"steppedLine": false,
"targets": [
{
- "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\"}) / sum(kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"cpu\"})",
+ "expr": "sum(kube_pod_container_resource_requests{job=\"kube-state-metrics\", resource=\"cpu\"}) / sum(kube_node_status_allocatable{job=\"kube-state-metrics\", resource=\"cpu\"})",
"format": "time_series",
"instant": true,
"intervalFactor": 2,
@@ -9277,7 +9277,7 @@ items:
"thresholds": "70,80",
"timeFrom": null,
"timeShift": null,
- "title": "CPU Utilisation (from limits)",
+ "title": "CPU Requests Commitment",
"tooltip": {
"shared": false,
"sort": 2,
@@ -9349,12 +9349,12 @@ items:
],
"spaceLength": 10,
- "span": 3,
+ "span": 2,
"stack": false,
"steppedLine": false,
"targets": [
{
- "expr": "sum(container_memory_working_set_bytes{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\",container!=\"\", image!=\"\"}) / sum(kube_pod_container_resource_requests{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"memory\"})",
+ "expr": "sum(kube_pod_container_resource_limits{job=\"kube-state-metrics\", resource=\"cpu\"}) / sum(kube_node_status_allocatable{job=\"kube-state-metrics\", resource=\"cpu\"})",
"format": "time_series",
"instant": true,
"intervalFactor": 2,
@@ -9364,7 +9364,7 @@ items:
"thresholds": "70,80",
"timeFrom": null,
"timeShift": null,
- "title": "Memory Utilisation (from requests)",
+ "title": "CPU Limits Commitment",
"tooltip": {
"shared": false,
"sort": 2,
@@ -9436,12 +9436,12 @@ items:
],
"spaceLength": 10,
- "span": 3,
+ "span": 2,
"stack": false,
"steppedLine": false,
"targets": [
{
- "expr": "sum(container_memory_working_set_bytes{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\",container!=\"\", image!=\"\"}) / sum(kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"memory\"})",
+ "expr": "1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(node_memory_MemTotal_bytes{job=\"node-exporter\"})",
"format": "time_series",
"instant": true,
"intervalFactor": 2,
@@ -9451,7 +9451,181 @@ items:
"thresholds": "70,80",
"timeFrom": null,
"timeShift": null,
- "title": "Memory Utilisation (from limits)",
+ "title": "Memory Utilisation",
+ "tooltip": {
+ "shared": false,
+ "sort": 2,
+ "value_type": "individual"
+ },
+ "type": "singlestat",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": [
+
+ ]
+ },
+ "yaxes": [
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": 0,
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": false
+ }
+ ]
+ },
+ {
+ "aliasColors": {
+
+ },
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "$datasource",
+ "fill": 1,
+ "format": "percentunit",
+ "id": 5,
+ "interval": "1m",
+ "legend": {
+ "alignAsTable": true,
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "rightSide": true,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [
+
+ ],
+ "nullPointMode": "null as zero",
+ "percentage": false,
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [
+
+ ],
+ "spaceLength": 10,
+ "span": 2,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "sum(kube_pod_container_resource_requests{job=\"kube-state-metrics\", resource=\"memory\"}) / sum(kube_node_status_allocatable{job=\"kube-state-metrics\", resource=\"memory\"})",
+ "format": "time_series",
+ "instant": true,
+ "intervalFactor": 2,
+ "refId": "A"
+ }
+ ],
+ "thresholds": "70,80",
+ "timeFrom": null,
+ "timeShift": null,
+ "title": "Memory Requests Commitment",
+ "tooltip": {
+ "shared": false,
+ "sort": 2,
+ "value_type": "individual"
+ },
+ "type": "singlestat",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": [
+
+ ]
+ },
+ "yaxes": [
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": 0,
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": false
+ }
+ ]
+ },
+ {
+ "aliasColors": {
+
+ },
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "$datasource",
+ "fill": 1,
+ "format": "percentunit",
+ "id": 6,
+ "interval": "1m",
+ "legend": {
+ "alignAsTable": true,
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "rightSide": true,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [
+
+ ],
+ "nullPointMode": "null as zero",
+ "percentage": false,
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [
+
+ ],
+ "spaceLength": 10,
+ "span": 2,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "sum(kube_pod_container_resource_limits{job=\"kube-state-metrics\", resource=\"memory\"}) / sum(kube_node_status_allocatable{job=\"kube-state-metrics\", resource=\"memory\"})",
+ "format": "time_series",
+ "instant": true,
+ "intervalFactor": 2,
+ "refId": "A"
+ }
+ ],
+ "thresholds": "70,80",
+ "timeFrom": null,
+ "timeShift": null,
+ "title": "Memory Limits Commitment",
"tooltip": {
"shared": false,
"sort": 2,
@@ -9506,8 +9680,8 @@ items:
"dashLength": 10,
"dashes": false,
"datasource": "$datasource",
- "fill": 10,
- "id": 5,
+ "fill": 0,
+ "id": 7,
"interval": "1m",
"legend": {
"alignAsTable": true,
@@ -9521,7 +9695,7 @@ items:
"values": false
},
"lines": true,
- "linewidth": 0,
+ "linewidth": 2,
"links": [
],
@@ -9531,55 +9705,18 @@ items:
"points": false,
"renderer": "flot",
"seriesOverrides": [
- {
- "alias": "quota - requests",
- "color": "#F2495C",
- "dashes": true,
- "fill": 0,
- "hiddenSeries": true,
- "hideTooltip": true,
- "legend": true,
- "linewidth": 2,
- "stack": false
- },
- {
- "alias": "quota - limits",
- "color": "#FF9830",
- "dashes": true,
- "fill": 0,
- "hiddenSeries": true,
- "hideTooltip": true,
- "legend": true,
- "linewidth": 2,
- "stack": false
- }
+
],
"spaceLength": 10,
"span": 12,
- "stack": true,
+ "stack": false,
"steppedLine": false,
"targets": [
{
- "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "{{pod}}",
- "legendLink": null,
- "step": 10
- },
- {
- "expr": "scalar(kube_resourcequota{cluster=\"$cluster\", namespace=\"$namespace\", type=\"hard\",resource=\"requests.cpu\"})",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "quota - requests",
- "legendLink": null,
- "step": 10
- },
- {
- "expr": "scalar(kube_resourcequota{cluster=\"$cluster\", namespace=\"$namespace\", type=\"hard\",resource=\"limits.cpu\"})",
+ "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate) by (cluster)",
"format": "time_series",
"intervalFactor": 2,
- "legendFormat": "quota - limits",
+ "legendFormat": "{{cluster}}",
"legendLink": null,
"step": 10
}
@@ -9629,7 +9766,7 @@ items:
"repeatIteration": null,
"repeatRowId": null,
"showTitle": true,
- "title": "CPU Usage",
+ "title": "CPU",
"titleSize": "h6"
},
{
@@ -9645,7 +9782,7 @@ items:
"dashes": false,
"datasource": "$datasource",
"fill": 1,
- "id": 6,
+ "id": 8,
"interval": "1m",
"legend": {
"alignAsTable": true,
@@ -9778,7 +9915,7 @@ items:
"unit": "percentunit"
},
{
- "alias": "Pod",
+ "alias": "Cluster",
"colorMode": null,
"colors": [
@@ -9788,8 +9925,1246 @@ items:
"link": true,
"linkTargetBlank": false,
"linkTooltip": "Drill down",
- "linkUrl": "/d/6581e46e4e5c7ba40a07646395ef7b23/k8s-resources-pod?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$namespace&var-pod=$__cell",
- "pattern": "pod",
+ "linkUrl": "/d/efa86fd1d0c121a26444b636a3f509a8/k8s-resources-cluster?var-datasource=$datasource&var-cluster=$__cell",
+ "pattern": "cluster",
+ "thresholds": [
+
+ ],
+ "type": "number",
+ "unit": "short"
+ },
+ {
+ "alias": "",
+ "colorMode": null,
+ "colors": [
+
+ ],
+ "dateFormat": "YYYY-MM-DD HH:mm:ss",
+ "decimals": 2,
+ "pattern": "/.*/",
+ "thresholds": [
+
+ ],
+ "type": "string",
+ "unit": "short"
+ }
+ ],
+ "targets": [
+ {
+ "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate) by (cluster)",
+ "format": "table",
+ "instant": true,
+ "intervalFactor": 2,
+ "legendFormat": "",
+ "refId": "A",
+ "step": 10
+ },
+ {
+ "expr": "sum(kube_pod_container_resource_requests{job=\"kube-state-metrics\", resource=\"cpu\"}) by (cluster)",
+ "format": "table",
+ "instant": true,
+ "intervalFactor": 2,
+ "legendFormat": "",
+ "refId": "B",
+ "step": 10
+ },
+ {
+ "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate) by (cluster) / sum(kube_pod_container_resource_requests{job=\"kube-state-metrics\", resource=\"cpu\"}) by (cluster)",
+ "format": "table",
+ "instant": true,
+ "intervalFactor": 2,
+ "legendFormat": "",
+ "refId": "C",
+ "step": 10
+ },
+ {
+ "expr": "sum(kube_pod_container_resource_limits{job=\"kube-state-metrics\", resource=\"cpu\"}) by (cluster)",
+ "format": "table",
+ "instant": true,
+ "intervalFactor": 2,
+ "legendFormat": "",
+ "refId": "D",
+ "step": 10
+ },
+ {
+ "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate) by (cluster) / sum(kube_pod_container_resource_limits{job=\"kube-state-metrics\", resource=\"cpu\"}) by (cluster)",
+ "format": "table",
+ "instant": true,
+ "intervalFactor": 2,
+ "legendFormat": "",
+ "refId": "E",
+ "step": 10
+ }
+ ],
+ "thresholds": [
+
+ ],
+ "timeFrom": null,
+ "timeShift": null,
+ "title": "CPU Quota",
+ "tooltip": {
+ "shared": false,
+ "sort": 2,
+ "value_type": "individual"
+ },
+ "transform": "table",
+ "type": "table",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": [
+
+ ]
+ },
+ "yaxes": [
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": 0,
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": false
+ }
+ ]
+ }
+ ],
+ "repeat": null,
+ "repeatIteration": null,
+ "repeatRowId": null,
+ "showTitle": true,
+ "title": "CPU Quota",
+ "titleSize": "h6"
+ },
+ {
+ "collapse": false,
+ "height": "250px",
+ "panels": [
+ {
+ "aliasColors": {
+
+ },
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "$datasource",
+ "fill": 0,
+ "id": 9,
+ "interval": "1m",
+ "legend": {
+ "alignAsTable": true,
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "rightSide": true,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 2,
+ "links": [
+
+ ],
+ "nullPointMode": "null as zero",
+ "percentage": false,
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [
+
+ ],
+ "spaceLength": 10,
+ "span": 12,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "sum(container_memory_rss{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", container!=\"\"}) by (cluster)",
+ "format": "time_series",
+ "intervalFactor": 2,
+ "legendFormat": "{{cluster}}",
+ "legendLink": null,
+ "step": 10
+ }
+ ],
+ "thresholds": [
+
+ ],
+ "timeFrom": null,
+ "timeShift": null,
+ "title": "Memory Usage (w/o cache)",
+ "tooltip": {
+ "shared": false,
+ "sort": 2,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": [
+
+ ]
+ },
+ "yaxes": [
+ {
+ "format": "bytes",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": 0,
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": false
+ }
+ ]
+ }
+ ],
+ "repeat": null,
+ "repeatIteration": null,
+ "repeatRowId": null,
+ "showTitle": true,
+ "title": "Memory",
+ "titleSize": "h6"
+ },
+ {
+ "collapse": false,
+ "height": "250px",
+ "panels": [
+ {
+ "aliasColors": {
+
+ },
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "$datasource",
+ "fill": 1,
+ "id": 10,
+ "interval": "1m",
+ "legend": {
+ "alignAsTable": true,
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "rightSide": true,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [
+
+ ],
+ "nullPointMode": "null as zero",
+ "percentage": false,
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [
+
+ ],
+ "spaceLength": 10,
+ "span": 12,
+ "stack": false,
+ "steppedLine": false,
+ "styles": [
+ {
+ "alias": "Time",
+ "dateFormat": "YYYY-MM-DD HH:mm:ss",
+ "pattern": "Time",
+ "type": "hidden"
+ },
+ {
+ "alias": "Memory Usage",
+ "colorMode": null,
+ "colors": [
+
+ ],
+ "dateFormat": "YYYY-MM-DD HH:mm:ss",
+ "decimals": 2,
+ "link": false,
+ "linkTargetBlank": false,
+ "linkTooltip": "Drill down",
+ "linkUrl": "",
+ "pattern": "Value #A",
+ "thresholds": [
+
+ ],
+ "type": "number",
+ "unit": "bytes"
+ },
+ {
+ "alias": "Memory Requests",
+ "colorMode": null,
+ "colors": [
+
+ ],
+ "dateFormat": "YYYY-MM-DD HH:mm:ss",
+ "decimals": 2,
+ "link": false,
+ "linkTargetBlank": false,
+ "linkTooltip": "Drill down",
+ "linkUrl": "",
+ "pattern": "Value #B",
+ "thresholds": [
+
+ ],
+ "type": "number",
+ "unit": "bytes"
+ },
+ {
+ "alias": "Memory Requests %",
+ "colorMode": null,
+ "colors": [
+
+ ],
+ "dateFormat": "YYYY-MM-DD HH:mm:ss",
+ "decimals": 2,
+ "link": false,
+ "linkTargetBlank": false,
+ "linkTooltip": "Drill down",
+ "linkUrl": "",
+ "pattern": "Value #C",
+ "thresholds": [
+
+ ],
+ "type": "number",
+ "unit": "percentunit"
+ },
+ {
+ "alias": "Memory Limits",
+ "colorMode": null,
+ "colors": [
+
+ ],
+ "dateFormat": "YYYY-MM-DD HH:mm:ss",
+ "decimals": 2,
+ "link": false,
+ "linkTargetBlank": false,
+ "linkTooltip": "Drill down",
+ "linkUrl": "",
+ "pattern": "Value #D",
+ "thresholds": [
+
+ ],
+ "type": "number",
+ "unit": "bytes"
+ },
+ {
+ "alias": "Memory Limits %",
+ "colorMode": null,
+ "colors": [
+
+ ],
+ "dateFormat": "YYYY-MM-DD HH:mm:ss",
+ "decimals": 2,
+ "link": false,
+ "linkTargetBlank": false,
+ "linkTooltip": "Drill down",
+ "linkUrl": "",
+ "pattern": "Value #E",
+ "thresholds": [
+
+ ],
+ "type": "number",
+ "unit": "percentunit"
+ },
+ {
+ "alias": "Cluster",
+ "colorMode": null,
+ "colors": [
+
+ ],
+ "dateFormat": "YYYY-MM-DD HH:mm:ss",
+ "decimals": 2,
+ "link": true,
+ "linkTargetBlank": false,
+ "linkTooltip": "Drill down",
+ "linkUrl": "/d/efa86fd1d0c121a26444b636a3f509a8/k8s-resources-cluster?var-datasource=$datasource&var-cluster=$__cell",
+ "pattern": "cluster",
+ "thresholds": [
+
+ ],
+ "type": "number",
+ "unit": "short"
+ },
+ {
+ "alias": "",
+ "colorMode": null,
+ "colors": [
+
+ ],
+ "dateFormat": "YYYY-MM-DD HH:mm:ss",
+ "decimals": 2,
+ "pattern": "/.*/",
+ "thresholds": [
+
+ ],
+ "type": "string",
+ "unit": "short"
+ }
+ ],
+ "targets": [
+ {
+ "expr": "sum(container_memory_rss{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", container!=\"\"}) by (cluster)",
+ "format": "table",
+ "instant": true,
+ "intervalFactor": 2,
+ "legendFormat": "",
+ "refId": "A",
+ "step": 10
+ },
+ {
+ "expr": "sum(kube_pod_container_resource_requests{job=\"kube-state-metrics\", resource=\"memory\"}) by (cluster)",
+ "format": "table",
+ "instant": true,
+ "intervalFactor": 2,
+ "legendFormat": "",
+ "refId": "B",
+ "step": 10
+ },
+ {
+ "expr": "sum(container_memory_rss{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", container!=\"\"}) by (cluster) / sum(kube_pod_container_resource_requests{job=\"kube-state-metrics\", resource=\"memory\"}) by (cluster)",
+ "format": "table",
+ "instant": true,
+ "intervalFactor": 2,
+ "legendFormat": "",
+ "refId": "C",
+ "step": 10
+ },
+ {
+ "expr": "sum(kube_pod_container_resource_limits{job=\"kube-state-metrics\", resource=\"memory\"}) by (cluster)",
+ "format": "table",
+ "instant": true,
+ "intervalFactor": 2,
+ "legendFormat": "",
+ "refId": "D",
+ "step": 10
+ },
+ {
+ "expr": "sum(container_memory_rss{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", container!=\"\"}) by (cluster) / sum(kube_pod_container_resource_limits{job=\"kube-state-metrics\", resource=\"memory\"}) by (cluster)",
+ "format": "table",
+ "instant": true,
+ "intervalFactor": 2,
+ "legendFormat": "",
+ "refId": "E",
+ "step": 10
+ }
+ ],
+ "thresholds": [
+
+ ],
+ "timeFrom": null,
+ "timeShift": null,
+ "title": "Requests by Cluster",
+ "tooltip": {
+ "shared": false,
+ "sort": 2,
+ "value_type": "individual"
+ },
+ "transform": "table",
+ "type": "table",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": [
+
+ ]
+ },
+ "yaxes": [
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": 0,
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": false
+ }
+ ]
+ }
+ ],
+ "repeat": null,
+ "repeatIteration": null,
+ "repeatRowId": null,
+ "showTitle": true,
+ "title": "Memory Requests",
+ "titleSize": "h6"
+ }
+ ],
+ "schemaVersion": 14,
+ "style": "dark",
+ "tags": [
+ "kubernetes-mixin"
+ ],
+ "templating": {
+ "list": [
+ {
+ "current": {
+ "text": "default",
+ "value": "default"
+ },
+ "hide": 0,
+ "label": "Data Source",
+ "name": "datasource",
+ "options": [
+
+ ],
+ "query": "prometheus",
+ "refresh": 1,
+ "regex": "",
+ "type": "datasource"
+ }
+ ]
+ },
+ "time": {
+ "from": "now-1h",
+ "to": "now"
+ },
+ "timepicker": {
+ "refresh_intervals": [
+ "5s",
+ "10s",
+ "30s",
+ "1m",
+ "5m",
+ "15m",
+ "30m",
+ "1h",
+ "2h",
+ "1d"
+ ],
+ "time_options": [
+ "5m",
+ "15m",
+ "1h",
+ "6h",
+ "12h",
+ "24h",
+ "2d",
+ "7d",
+ "30d"
+ ]
+ },
+ "timezone": "UTC",
+ "title": "Kubernetes / Compute Resources / Multi-Cluster",
+ "uid": "b59e6c9f2fcbe2e16d77fc492374cc4f",
+ "version": 0
+ }
+ kind: ConfigMap
+ metadata:
+ labels:
+ app.kubernetes.io/component: grafana
+ app.kubernetes.io/name: grafana
+ app.kubernetes.io/part-of: kube-prometheus
+ app.kubernetes.io/version: 9.5.2
+ name: grafana-dashboard-k8s-resources-multicluster
+ namespace: monitoring
+- apiVersion: v1
+ data:
+ k8s-resources-namespace.json: |-
+ {
+ "annotations": {
+ "list": [
+
+ ]
+ },
+ "editable": true,
+ "gnetId": null,
+ "graphTooltip": 0,
+ "hideControls": false,
+ "links": [
+
+ ],
+ "refresh": "10s",
+ "rows": [
+ {
+ "collapse": false,
+ "height": "100px",
+ "panels": [
+ {
+ "aliasColors": {
+
+ },
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "$datasource",
+ "fill": 1,
+ "format": "percentunit",
+ "id": 1,
+ "interval": "1m",
+ "legend": {
+ "alignAsTable": true,
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "rightSide": true,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [
+
+ ],
+ "nullPointMode": "null as zero",
+ "percentage": false,
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [
+
+ ],
+ "spaceLength": 10,
+ "span": 3,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\"}) / sum(kube_pod_container_resource_requests{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"cpu\"})",
+ "format": "time_series",
+ "instant": true,
+ "intervalFactor": 2,
+ "refId": "A"
+ }
+ ],
+ "thresholds": "70,80",
+ "timeFrom": null,
+ "timeShift": null,
+ "title": "CPU Utilisation (from requests)",
+ "tooltip": {
+ "shared": false,
+ "sort": 2,
+ "value_type": "individual"
+ },
+ "type": "singlestat",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": [
+
+ ]
+ },
+ "yaxes": [
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": 0,
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": false
+ }
+ ]
+ },
+ {
+ "aliasColors": {
+
+ },
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "$datasource",
+ "fill": 1,
+ "format": "percentunit",
+ "id": 2,
+ "interval": "1m",
+ "legend": {
+ "alignAsTable": true,
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "rightSide": true,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [
+
+ ],
+ "nullPointMode": "null as zero",
+ "percentage": false,
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [
+
+ ],
+ "spaceLength": 10,
+ "span": 3,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\"}) / sum(kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"cpu\"})",
+ "format": "time_series",
+ "instant": true,
+ "intervalFactor": 2,
+ "refId": "A"
+ }
+ ],
+ "thresholds": "70,80",
+ "timeFrom": null,
+ "timeShift": null,
+ "title": "CPU Utilisation (from limits)",
+ "tooltip": {
+ "shared": false,
+ "sort": 2,
+ "value_type": "individual"
+ },
+ "type": "singlestat",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": [
+
+ ]
+ },
+ "yaxes": [
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": 0,
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": false
+ }
+ ]
+ },
+ {
+ "aliasColors": {
+
+ },
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "$datasource",
+ "fill": 1,
+ "format": "percentunit",
+ "id": 3,
+ "interval": "1m",
+ "legend": {
+ "alignAsTable": true,
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "rightSide": true,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [
+
+ ],
+ "nullPointMode": "null as zero",
+ "percentage": false,
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [
+
+ ],
+ "spaceLength": 10,
+ "span": 3,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "sum(container_memory_working_set_bytes{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\",container!=\"\", image!=\"\"}) / sum(kube_pod_container_resource_requests{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"memory\"})",
+ "format": "time_series",
+ "instant": true,
+ "intervalFactor": 2,
+ "refId": "A"
+ }
+ ],
+ "thresholds": "70,80",
+ "timeFrom": null,
+ "timeShift": null,
+ "title": "Memory Utilisation (from requests)",
+ "tooltip": {
+ "shared": false,
+ "sort": 2,
+ "value_type": "individual"
+ },
+ "type": "singlestat",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": [
+
+ ]
+ },
+ "yaxes": [
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": 0,
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": false
+ }
+ ]
+ },
+ {
+ "aliasColors": {
+
+ },
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "$datasource",
+ "fill": 1,
+ "format": "percentunit",
+ "id": 4,
+ "interval": "1m",
+ "legend": {
+ "alignAsTable": true,
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "rightSide": true,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [
+
+ ],
+ "nullPointMode": "null as zero",
+ "percentage": false,
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [
+
+ ],
+ "spaceLength": 10,
+ "span": 3,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "sum(container_memory_working_set_bytes{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\",container!=\"\", image!=\"\"}) / sum(kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"memory\"})",
+ "format": "time_series",
+ "instant": true,
+ "intervalFactor": 2,
+ "refId": "A"
+ }
+ ],
+ "thresholds": "70,80",
+ "timeFrom": null,
+ "timeShift": null,
+ "title": "Memory Utilisation (from limits)",
+ "tooltip": {
+ "shared": false,
+ "sort": 2,
+ "value_type": "individual"
+ },
+ "type": "singlestat",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": [
+
+ ]
+ },
+ "yaxes": [
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": 0,
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": false
+ }
+ ]
+ }
+ ],
+ "repeat": null,
+ "repeatIteration": null,
+ "repeatRowId": null,
+ "showTitle": false,
+ "title": "Headlines",
+ "titleSize": "h6"
+ },
+ {
+ "collapse": false,
+ "height": "250px",
+ "panels": [
+ {
+ "aliasColors": {
+
+ },
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "$datasource",
+ "fill": 10,
+ "id": 5,
+ "interval": "1m",
+ "legend": {
+ "alignAsTable": true,
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "rightSide": true,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 0,
+ "links": [
+
+ ],
+ "nullPointMode": "null as zero",
+ "percentage": false,
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [
+ {
+ "alias": "quota - requests",
+ "color": "#F2495C",
+ "dashes": true,
+ "fill": 0,
+ "hiddenSeries": true,
+ "hideTooltip": true,
+ "legend": true,
+ "linewidth": 2,
+ "stack": false
+ },
+ {
+ "alias": "quota - limits",
+ "color": "#FF9830",
+ "dashes": true,
+ "fill": 0,
+ "hiddenSeries": true,
+ "hideTooltip": true,
+ "legend": true,
+ "linewidth": 2,
+ "stack": false
+ }
+ ],
+ "spaceLength": 10,
+ "span": 12,
+ "stack": true,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)",
+ "format": "time_series",
+ "intervalFactor": 2,
+ "legendFormat": "{{pod}}",
+ "legendLink": null,
+ "step": 10
+ },
+ {
+ "expr": "scalar(kube_resourcequota{cluster=\"$cluster\", namespace=\"$namespace\", type=\"hard\",resource=\"requests.cpu\"})",
+ "format": "time_series",
+ "intervalFactor": 2,
+ "legendFormat": "quota - requests",
+ "legendLink": null,
+ "step": 10
+ },
+ {
+ "expr": "scalar(kube_resourcequota{cluster=\"$cluster\", namespace=\"$namespace\", type=\"hard\",resource=\"limits.cpu\"})",
+ "format": "time_series",
+ "intervalFactor": 2,
+ "legendFormat": "quota - limits",
+ "legendLink": null,
+ "step": 10
+ }
+ ],
+ "thresholds": [
+
+ ],
+ "timeFrom": null,
+ "timeShift": null,
+ "title": "CPU Usage",
+ "tooltip": {
+ "shared": false,
+ "sort": 2,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": [
+
+ ]
+ },
+ "yaxes": [
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": 0,
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": false
+ }
+ ]
+ }
+ ],
+ "repeat": null,
+ "repeatIteration": null,
+ "repeatRowId": null,
+ "showTitle": true,
+ "title": "CPU Usage",
+ "titleSize": "h6"
+ },
+ {
+ "collapse": false,
+ "height": "250px",
+ "panels": [
+ {
+ "aliasColors": {
+
+ },
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "$datasource",
+ "fill": 1,
+ "id": 6,
+ "interval": "1m",
+ "legend": {
+ "alignAsTable": true,
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "rightSide": true,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [
+
+ ],
+ "nullPointMode": "null as zero",
+ "percentage": false,
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [
+
+ ],
+ "spaceLength": 10,
+ "span": 12,
+ "stack": false,
+ "steppedLine": false,
+ "styles": [
+ {
+ "alias": "Time",
+ "dateFormat": "YYYY-MM-DD HH:mm:ss",
+ "pattern": "Time",
+ "type": "hidden"
+ },
+ {
+ "alias": "CPU Usage",
+ "colorMode": null,
+ "colors": [
+
+ ],
+ "dateFormat": "YYYY-MM-DD HH:mm:ss",
+ "decimals": 2,
+ "link": false,
+ "linkTargetBlank": false,
+ "linkTooltip": "Drill down",
+ "linkUrl": "",
+ "pattern": "Value #A",
+ "thresholds": [
+
+ ],
+ "type": "number",
+ "unit": "short"
+ },
+ {
+ "alias": "CPU Requests",
+ "colorMode": null,
+ "colors": [
+
+ ],
+ "dateFormat": "YYYY-MM-DD HH:mm:ss",
+ "decimals": 2,
+ "link": false,
+ "linkTargetBlank": false,
+ "linkTooltip": "Drill down",
+ "linkUrl": "",
+ "pattern": "Value #B",
+ "thresholds": [
+
+ ],
+ "type": "number",
+ "unit": "short"
+ },
+ {
+ "alias": "CPU Requests %",
+ "colorMode": null,
+ "colors": [
+
+ ],
+ "dateFormat": "YYYY-MM-DD HH:mm:ss",
+ "decimals": 2,
+ "link": false,
+ "linkTargetBlank": false,
+ "linkTooltip": "Drill down",
+ "linkUrl": "",
+ "pattern": "Value #C",
+ "thresholds": [
+
+ ],
+ "type": "number",
+ "unit": "percentunit"
+ },
+ {
+ "alias": "CPU Limits",
+ "colorMode": null,
+ "colors": [
+
+ ],
+ "dateFormat": "YYYY-MM-DD HH:mm:ss",
+ "decimals": 2,
+ "link": false,
+ "linkTargetBlank": false,
+ "linkTooltip": "Drill down",
+ "linkUrl": "",
+ "pattern": "Value #D",
+ "thresholds": [
+
+ ],
+ "type": "number",
+ "unit": "short"
+ },
+ {
+ "alias": "CPU Limits %",
+ "colorMode": null,
+ "colors": [
+
+ ],
+ "dateFormat": "YYYY-MM-DD HH:mm:ss",
+ "decimals": 2,
+ "link": false,
+ "linkTargetBlank": false,
+ "linkTooltip": "Drill down",
+ "linkUrl": "",
+ "pattern": "Value #E",
+ "thresholds": [
+
+ ],
+ "type": "number",
+ "unit": "percentunit"
+ },
+ {
+ "alias": "Pod",
+ "colorMode": null,
+ "colors": [
+
+ ],
+ "dateFormat": "YYYY-MM-DD HH:mm:ss",
+ "decimals": 2,
+ "link": true,
+ "linkTargetBlank": false,
+ "linkTooltip": "Drill down",
+ "linkUrl": "/d/6581e46e4e5c7ba40a07646395ef7b23/k8s-resources-pod?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$namespace&var-pod=$__cell",
+ "pattern": "pod",
"thresholds": [
],
@@ -11810,7 +13185,7 @@ items:
"value": ""
},
"datasource": "$datasource",
- "hide": 2,
+ "hide": 0,
"includeAll": false,
"label": null,
"multi": false,
@@ -12825,7 +14200,7 @@ items:
"value": ""
},
"datasource": "$datasource",
- "hide": 2,
+ "hide": 0,
"includeAll": false,
"label": null,
"multi": false,
@@ -15256,7 +16631,7 @@ items:
"value": ""
},
"datasource": "$datasource",
- "hide": 2,
+ "hide": 0,
"includeAll": false,
"label": null,
"multi": false,
@@ -17242,7 +18617,7 @@ items:
"value": ""
},
"datasource": "$datasource",
- "hide": 2,
+ "hide": 0,
"includeAll": false,
"label": null,
"multi": false,
@@ -19442,7 +20817,7 @@ items:
"value": ""
},
"datasource": "$datasource",
- "hide": 2,
+ "hide": 0,
"includeAll": false,
"label": null,
"multi": false,
@@ -21718,7 +23093,7 @@ items:
},
"datasource": "$datasource",
- "hide": 2,
+ "hide": 0,
"includeAll": false,
"label": "cluster",
"multi": false,
@@ -23085,7 +24460,7 @@ items:
},
"datasource": "$datasource",
- "hide": 2,
+ "hide": 0,
"includeAll": false,
"label": null,
"multi": false,
@@ -24778,7 +26153,7 @@ items:
},
"datasource": "$datasource",
- "hide": 2,
+ "hide": 0,
"includeAll": false,
"label": null,
"multi": false,
@@ -29693,7 +31068,7 @@ items:
},
"datasource": "$datasource",
- "hide": 2,
+ "hide": 0,
"includeAll": false,
"label": "cluster",
"multi": false,
@@ -30818,7 +32193,7 @@ items:
},
"datasource": "$datasource",
- "hide": 2,
+ "hide": 0,
"includeAll": false,
"label": null,
"multi": false,
@@ -35079,7 +36454,7 @@ items:
},
"datasource": "$datasource",
- "hide": 2,
+ "hide": 0,
"includeAll": false,
"label": "cluster",
"multi": false,
@@ -36180,7 +37555,7 @@ items:
},
"datasource": "$datasource",
- "hide": 2,
+ "hide": 0,
"includeAll": false,
"label": "cluster",
"multi": false,
@@ -37457,7 +38832,7 @@ items:
},
"datasource": "$datasource",
- "hide": 2,
+ "hide": 0,
"includeAll": false,
"label": null,
"multi": false,
diff --git a/manifests/grafana-deployment.yaml b/manifests/grafana-deployment.yaml
index 3adbfdbe709235ebd3affb7204015108f22321e5..c3654559d4a5114e4d80894f98f86b6d00e8ec3e 100644
--- a/manifests/grafana-deployment.yaml
+++ b/manifests/grafana-deployment.yaml
@@ -83,6 +83,9 @@ spec:
- mountPath: /grafana-dashboard-definitions/0/k8s-resources-cluster
name: grafana-dashboard-k8s-resources-cluster
readOnly: false
+ - mountPath: /grafana-dashboard-definitions/0/k8s-resources-multicluster
+ name: grafana-dashboard-k8s-resources-multicluster
+ readOnly: false
- mountPath: /grafana-dashboard-definitions/0/k8s-resources-namespace
name: grafana-dashboard-k8s-resources-namespace
readOnly: false
@@ -180,6 +183,9 @@ spec:
- configMap:
name: grafana-dashboard-k8s-resources-cluster
name: grafana-dashboard-k8s-resources-cluster
+ - configMap:
+ name: grafana-dashboard-k8s-resources-multicluster
+ name: grafana-dashboard-k8s-resources-multicluster
- configMap:
name: grafana-dashboard-k8s-resources-namespace
name: grafana-dashboard-k8s-resources-namespace
diff --git a/manifests/kubePrometheus-prometheusRule.yaml b/manifests/kubePrometheus-prometheusRule.yaml
index 0facf214fa83c8581765379cd5af59a35a86d6ff..1356ee9dc9d04f0f54db9b44e4b002e85daae65f 100644
--- a/manifests/kubePrometheus-prometheusRule.yaml
+++ b/manifests/kubePrometheus-prometheusRule.yaml
@@ -18,7 +18,7 @@ spec:
description: '{{ printf "%.4g" $value }}% of the {{ $labels.job }}/{{ $labels.service }} targets in {{ $labels.namespace }} namespace are down.'
runbook_url: https://runbooks.prometheus-operator.dev/runbooks/general/targetdown
summary: One or more targets are unreachable.
- expr: 100 * (count(up == 0) BY (job, namespace, service) / count(up) BY (job, namespace, service)) > 10
+ expr: 100 * (count(up == 0) BY (cluster, job, namespace, service) / count(up) BY (cluster, job, namespace, service)) > 10
for: 10m
labels:
severity: warning
diff --git a/manifests/kubernetesControlPlane-prometheusRule.yaml b/manifests/kubernetesControlPlane-prometheusRule.yaml
index dd508fcfb7891138aa7e8c3dd6d62a629347e80c..7e9a3827a5c6c0c7e9edf68eabc840175c0b48a9 100644
--- a/manifests/kubernetesControlPlane-prometheusRule.yaml
+++ b/manifests/kubernetesControlPlane-prometheusRule.yaml
@@ -247,50 +247,50 @@ spec:
rules:
- alert: KubeCPUOvercommit
annotations:
- description: Cluster has overcommitted CPU resource requests for Pods by {{ $value }} CPU shares and cannot tolerate node failure.
+ description: Cluster {{ $labels.cluster }} has overcommitted CPU resource requests for Pods by {{ $value }} CPU shares and cannot tolerate node failure.
runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubecpuovercommit
summary: Cluster has overcommitted CPU resource requests.
expr: |
- sum(namespace_cpu:kube_pod_container_resource_requests:sum{}) - (sum(kube_node_status_allocatable{resource="cpu", job="kube-state-metrics"}) - max(kube_node_status_allocatable{resource="cpu", job="kube-state-metrics"})) > 0
+ sum(namespace_cpu:kube_pod_container_resource_requests:sum{job="kube-state-metrics",}) by (cluster) - (sum(kube_node_status_allocatable{job="kube-state-metrics",resource="cpu"}) by (cluster) - max(kube_node_status_allocatable{job="kube-state-metrics",resource="cpu"}) by (cluster)) > 0
and
- (sum(kube_node_status_allocatable{resource="cpu", job="kube-state-metrics"}) - max(kube_node_status_allocatable{resource="cpu", job="kube-state-metrics"})) > 0
+ (sum(kube_node_status_allocatable{job="kube-state-metrics",resource="cpu"}) by (cluster) - max(kube_node_status_allocatable{job="kube-state-metrics",resource="cpu"}) by (cluster)) > 0
for: 10m
labels:
severity: warning
- alert: KubeMemoryOvercommit
annotations:
- description: Cluster has overcommitted memory resource requests for Pods by {{ $value | humanize }} bytes and cannot tolerate node failure.
+ description: Cluster {{ $labels.cluster }} has overcommitted memory resource requests for Pods by {{ $value | humanize }} bytes and cannot tolerate node failure.
runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubememoryovercommit
summary: Cluster has overcommitted memory resource requests.
expr: |
- sum(namespace_memory:kube_pod_container_resource_requests:sum{}) - (sum(kube_node_status_allocatable{resource="memory", job="kube-state-metrics"}) - max(kube_node_status_allocatable{resource="memory", job="kube-state-metrics"})) > 0
+ sum(namespace_memory:kube_pod_container_resource_requests:sum{}) by (cluster) - (sum(kube_node_status_allocatable{resource="memory", job="kube-state-metrics"}) by (cluster) - max(kube_node_status_allocatable{resource="memory", job="kube-state-metrics"}) by (cluster)) > 0
and
- (sum(kube_node_status_allocatable{resource="memory", job="kube-state-metrics"}) - max(kube_node_status_allocatable{resource="memory", job="kube-state-metrics"})) > 0
+ (sum(kube_node_status_allocatable{resource="memory", job="kube-state-metrics"}) by (cluster) - max(kube_node_status_allocatable{resource="memory", job="kube-state-metrics"}) by (cluster)) > 0
for: 10m
labels:
severity: warning
- alert: KubeCPUQuotaOvercommit
annotations:
- description: Cluster has overcommitted CPU resource requests for Namespaces.
+ description: Cluster {{ $labels.cluster }} has overcommitted CPU resource requests for Namespaces.
runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubecpuquotaovercommit
summary: Cluster has overcommitted CPU resource requests.
expr: |
- sum(min without(resource) (kube_resourcequota{job="kube-state-metrics", type="hard", resource=~"(cpu|requests.cpu)"}))
+ sum(min without(resource) (kube_resourcequota{job="kube-state-metrics", type="hard", resource=~"(cpu|requests.cpu)"})) by (cluster)
/
- sum(kube_node_status_allocatable{resource="cpu", job="kube-state-metrics"})
+ sum(kube_node_status_allocatable{resource="cpu", job="kube-state-metrics"}) by (cluster)
> 1.5
for: 5m
labels:
severity: warning
- alert: KubeMemoryQuotaOvercommit
annotations:
- description: Cluster has overcommitted memory resource requests for Namespaces.
+ description: Cluster {{ $labels.cluster }} has overcommitted memory resource requests for Namespaces.
runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubememoryquotaovercommit
summary: Cluster has overcommitted memory resource requests.
expr: |
- sum(min without(resource) (kube_resourcequota{job="kube-state-metrics", type="hard", resource=~"(memory|requests.memory)"}))
+ sum(min without(resource) (kube_resourcequota{job="kube-state-metrics", type="hard", resource=~"(memory|requests.memory)"})) by (cluster)
/
- sum(kube_node_status_allocatable{resource="memory", job="kube-state-metrics"})
+ sum(kube_node_status_allocatable{resource="memory", job="kube-state-metrics"}) by (cluster)
> 1.5
for: 5m
labels:
diff --git a/manifests/prometheusOperator-prometheusRule.yaml b/manifests/prometheusOperator-prometheusRule.yaml
index 45e4757aac5ddde1d54b09c6bf80a1c44d44858d..e109efc4b317ea88807de91947d1a7bf1c44eda1 100644
--- a/manifests/prometheusOperator-prometheusRule.yaml
+++ b/manifests/prometheusOperator-prometheusRule.yaml
@@ -20,7 +20,7 @@ spec:
runbook_url: https://runbooks.prometheus-operator.dev/runbooks/prometheus-operator/prometheusoperatorlisterrors
summary: Errors while performing list operations in controller.
expr: |
- (sum by (controller,namespace) (rate(prometheus_operator_list_operations_failed_total{job="prometheus-operator",namespace="monitoring"}[10m])) / sum by (controller,namespace) (rate(prometheus_operator_list_operations_total{job="prometheus-operator",namespace="monitoring"}[10m]))) > 0.4
+ (sum by (cluster,controller,namespace) (rate(prometheus_operator_list_operations_failed_total{job="prometheus-operator",namespace="monitoring"}[10m])) / sum by (cluster,controller,namespace) (rate(prometheus_operator_list_operations_total{job="prometheus-operator",namespace="monitoring"}[10m]))) > 0.4
for: 15m
labels:
severity: warning
@@ -30,7 +30,7 @@ spec:
runbook_url: https://runbooks.prometheus-operator.dev/runbooks/prometheus-operator/prometheusoperatorwatcherrors
summary: Errors while performing watch operations in controller.
expr: |
- (sum by (controller,namespace) (rate(prometheus_operator_watch_operations_failed_total{job="prometheus-operator",namespace="monitoring"}[5m])) / sum by (controller,namespace) (rate(prometheus_operator_watch_operations_total{job="prometheus-operator",namespace="monitoring"}[5m]))) > 0.4
+ (sum by (cluster,controller,namespace) (rate(prometheus_operator_watch_operations_failed_total{job="prometheus-operator",namespace="monitoring"}[5m])) / sum by (cluster,controller,namespace) (rate(prometheus_operator_watch_operations_total{job="prometheus-operator",namespace="monitoring"}[5m]))) > 0.4
for: 15m
labels:
severity: warning
@@ -50,7 +50,7 @@ spec:
runbook_url: https://runbooks.prometheus-operator.dev/runbooks/prometheus-operator/prometheusoperatorreconcileerrors
summary: Errors while reconciling controller.
expr: |
- (sum by (controller,namespace) (rate(prometheus_operator_reconcile_errors_total{job="prometheus-operator",namespace="monitoring"}[5m]))) / (sum by (controller,namespace) (rate(prometheus_operator_reconcile_operations_total{job="prometheus-operator",namespace="monitoring"}[5m]))) > 0.1
+ (sum by (cluster,controller,namespace) (rate(prometheus_operator_reconcile_errors_total{job="prometheus-operator",namespace="monitoring"}[5m]))) / (sum by (cluster,controller,namespace) (rate(prometheus_operator_reconcile_operations_total{job="prometheus-operator",namespace="monitoring"}[5m]))) > 0.1
for: 10m
labels:
severity: warning
@@ -70,7 +70,7 @@ spec:
runbook_url: https://runbooks.prometheus-operator.dev/runbooks/prometheus-operator/prometheusoperatornotready
summary: Prometheus operator not ready
expr: |
- min by (controller,namespace) (max_over_time(prometheus_operator_ready{job="prometheus-operator",namespace="monitoring"}[5m]) == 0)
+ min by (cluster,controller,namespace) (max_over_time(prometheus_operator_ready{job="prometheus-operator",namespace="monitoring"}[5m]) == 0)
for: 5m
labels:
severity: warning