From 2dad999332180d74e0aed50fad438b7d7458fd4d Mon Sep 17 00:00:00 2001
From: Matthias Loibl <mail@matthiasloibl.com>
Date: Fri, 4 Mar 2022 18:51:48 +0100
Subject: [PATCH] Add Pyrra as addon and commented out example.jsonnet parts
---
README.md | 3 +
...prometheus-rules-and-grafana-dashboards.md | 3 +
example.jsonnet | 3 +
.../kube-prometheus/addons/pyrra.libsonnet | 623 ++++++++++++++++++
jsonnet/kube-prometheus/jsonnetfile.json | 9 +
.../platforms/platforms.libsonnet | 6 +-
jsonnet/kube-prometheus/versions.json | 1 +
jsonnetfile.lock.json | 10 +
8 files changed, 656 insertions(+), 2 deletions(-)
create mode 100644 jsonnet/kube-prometheus/addons/pyrra.libsonnet
diff --git a/README.md b/README.md
index 90000c42..c0a33ad4 100644
--- a/README.md
+++ b/README.md
@@ -209,6 +209,7 @@ local kp =
// (import 'kube-prometheus/addons/static-etcd.libsonnet') +
// (import 'kube-prometheus/addons/custom-metrics.libsonnet') +
// (import 'kube-prometheus/addons/external-metrics.libsonnet') +
+ // (import 'kube-prometheus/addons/pyrra.libsonnet') +
{
values+:: {
common+: {
@@ -222,6 +223,7 @@ local kp =
['setup/prometheus-operator-' + name]: kp.prometheusOperator[name]
for name in std.filter((function(name) name != 'serviceMonitor' && name != 'prometheusRule'), std.objectFields(kp.prometheusOperator))
} +
+// { 'setup/pyrra-slo-CustomResourceDefinition': kp.pyrra.crd } +
// serviceMonitor and prometheusRule are separated so that they can be created after the CRDs are ready
{ 'prometheus-operator-serviceMonitor': kp.prometheusOperator.serviceMonitor } +
{ 'prometheus-operator-prometheusRule': kp.prometheusOperator.prometheusRule } +
@@ -229,6 +231,7 @@ local kp =
{ ['alertmanager-' + name]: kp.alertmanager[name] for name in std.objectFields(kp.alertmanager) } +
{ ['blackbox-exporter-' + name]: kp.blackboxExporter[name] for name in std.objectFields(kp.blackboxExporter) } +
{ ['grafana-' + name]: kp.grafana[name] for name in std.objectFields(kp.grafana) } +
+// { ['pyrra-' + name]: kp.pyrra[name] for name in std.objectFields(kp.pyrra) if name != 'crd' } +
{ ['kube-state-metrics-' + name]: kp.kubeStateMetrics[name] for name in std.objectFields(kp.kubeStateMetrics) } +
{ ['kubernetes-' + name]: kp.kubernetesControlPlane[name] for name in std.objectFields(kp.kubernetesControlPlane) }
{ ['node-exporter-' + name]: kp.nodeExporter[name] for name in std.objectFields(kp.nodeExporter) } +
diff --git a/docs/customizations/developing-prometheus-rules-and-grafana-dashboards.md b/docs/customizations/developing-prometheus-rules-and-grafana-dashboards.md
index c367ddbe..5d472508 100644
--- a/docs/customizations/developing-prometheus-rules-and-grafana-dashboards.md
+++ b/docs/customizations/developing-prometheus-rules-and-grafana-dashboards.md
@@ -32,6 +32,7 @@ local kp =
// (import 'kube-prometheus/addons/static-etcd.libsonnet') +
// (import 'kube-prometheus/addons/custom-metrics.libsonnet') +
// (import 'kube-prometheus/addons/external-metrics.libsonnet') +
+ // (import 'kube-prometheus/addons/pyrra.libsonnet') +
{
values+:: {
common+: {
@@ -45,6 +46,7 @@ local kp =
['setup/prometheus-operator-' + name]: kp.prometheusOperator[name]
for name in std.filter((function(name) name != 'serviceMonitor' && name != 'prometheusRule'), std.objectFields(kp.prometheusOperator))
} +
+// { 'setup/pyrra-slo-CustomResourceDefinition': kp.pyrra.crd } +
// serviceMonitor and prometheusRule are separated so that they can be created after the CRDs are ready
{ 'prometheus-operator-serviceMonitor': kp.prometheusOperator.serviceMonitor } +
{ 'prometheus-operator-prometheusRule': kp.prometheusOperator.prometheusRule } +
@@ -52,6 +54,7 @@ local kp =
{ ['alertmanager-' + name]: kp.alertmanager[name] for name in std.objectFields(kp.alertmanager) } +
{ ['blackbox-exporter-' + name]: kp.blackboxExporter[name] for name in std.objectFields(kp.blackboxExporter) } +
{ ['grafana-' + name]: kp.grafana[name] for name in std.objectFields(kp.grafana) } +
+// { ['pyrra-' + name]: kp.pyrra[name] for name in std.objectFields(kp.pyrra) if name != 'crd' } +
{ ['kube-state-metrics-' + name]: kp.kubeStateMetrics[name] for name in std.objectFields(kp.kubeStateMetrics) } +
{ ['kubernetes-' + name]: kp.kubernetesControlPlane[name] for name in std.objectFields(kp.kubernetesControlPlane) }
{ ['node-exporter-' + name]: kp.nodeExporter[name] for name in std.objectFields(kp.nodeExporter) } +
diff --git a/example.jsonnet b/example.jsonnet
index b181d647..8974158b 100644
--- a/example.jsonnet
+++ b/example.jsonnet
@@ -7,6 +7,7 @@ local kp =
// (import 'kube-prometheus/addons/static-etcd.libsonnet') +
// (import 'kube-prometheus/addons/custom-metrics.libsonnet') +
// (import 'kube-prometheus/addons/external-metrics.libsonnet') +
+ // (import 'kube-prometheus/addons/pyrra.libsonnet') +
{
values+:: {
common+: {
@@ -20,6 +21,7 @@ local kp =
['setup/prometheus-operator-' + name]: kp.prometheusOperator[name]
for name in std.filter((function(name) name != 'serviceMonitor' && name != 'prometheusRule'), std.objectFields(kp.prometheusOperator))
} +
+// { 'setup/pyrra-slo-CustomResourceDefinition': kp.pyrra.crd } +
// serviceMonitor and prometheusRule are separated so that they can be created after the CRDs are ready
{ 'prometheus-operator-serviceMonitor': kp.prometheusOperator.serviceMonitor } +
{ 'prometheus-operator-prometheusRule': kp.prometheusOperator.prometheusRule } +
@@ -27,6 +29,7 @@ local kp =
{ ['alertmanager-' + name]: kp.alertmanager[name] for name in std.objectFields(kp.alertmanager) } +
{ ['blackbox-exporter-' + name]: kp.blackboxExporter[name] for name in std.objectFields(kp.blackboxExporter) } +
{ ['grafana-' + name]: kp.grafana[name] for name in std.objectFields(kp.grafana) } +
+// { ['pyrra-' + name]: kp.pyrra[name] for name in std.objectFields(kp.pyrra) if name != 'crd' } +
{ ['kube-state-metrics-' + name]: kp.kubeStateMetrics[name] for name in std.objectFields(kp.kubeStateMetrics) } +
{ ['kubernetes-' + name]: kp.kubernetesControlPlane[name] for name in std.objectFields(kp.kubernetesControlPlane) }
{ ['node-exporter-' + name]: kp.nodeExporter[name] for name in std.objectFields(kp.nodeExporter) } +
diff --git a/jsonnet/kube-prometheus/addons/pyrra.libsonnet b/jsonnet/kube-prometheus/addons/pyrra.libsonnet
new file mode 100644
index 00000000..894141a8
--- /dev/null
+++ b/jsonnet/kube-prometheus/addons/pyrra.libsonnet
@@ -0,0 +1,623 @@
+{
+ values+:: {
+ common+: {
+ versions+: {
+ pyrra: error 'must provide version',
+ } + (import '../versions.json'),
+ images+: {
+ pyrra+: 'ghcr.io/pyrra-dev/pyrra:v' + $.values.common.versions.pyrra,
+ },
+ },
+ pyrra+: {
+ namespace: $.values.common.namespace,
+ version: $.values.common.versions.pyrra,
+ image: $.values.common.images.pyrra,
+ },
+ },
+
+ local defaults = {
+ local defaults = self,
+
+ name:: 'pyrra',
+ namespace:: error 'must provide namespace',
+ version:: error 'must provide version',
+ image: error 'must provide image',
+ replicas:: 1,
+ port:: 9099,
+
+ commonLabels:: {
+ 'app.kubernetes.io/name': 'pyrra',
+ 'app.kubernetes.io/version': defaults.version,
+ 'app.kubernetes.io/part-of': 'kube-prometheus',
+ },
+ },
+
+ local pyrra = function(params) {
+ local pyrra = self,
+ _config:: defaults + params,
+
+ crd: (
+ import 'github.com/pyrra-dev/pyrra/config/crd/bases/pyrra.dev_servicelevelobjectives.json'
+ ),
+
+
+ _apiMetadata:: {
+ name: pyrra._config.name + '-api',
+ namespace: pyrra._config.namespace,
+ labels: pyrra._config.commonLabels {
+ 'app.kubernetes.io/component': 'api',
+ },
+ },
+ apiSelectorLabels:: {
+ [labelName]: pyrra._apiMetadata.labels[labelName]
+ for labelName in std.objectFields(pyrra._apiMetadata.labels)
+ if !std.setMember(labelName, ['app.kubernetes.io/version'])
+ },
+
+ apiService: {
+ apiVersion: 'v1',
+ kind: 'Service',
+ metadata: pyrra._apiMetadata,
+ spec: {
+ ports: [
+ { name: 'http', targetPort: pyrra._config.port, port: pyrra._config.port },
+ ],
+ selector: pyrra.apiSelectorLabels,
+ },
+ },
+
+ apiDeployment:
+ local c = {
+ name: pyrra._config.name,
+ image: pyrra._config.image,
+ args: [
+ 'api',
+ '--api-url=http://%s.%s.svc.cluster.local:9444' % [pyrra.kubernetesService.metadata.name, pyrra.kubernetesService.metadata.namespace],
+ '--prometheus-url=http://prometheus-k8s.monitoring.svc.cluster.local:9090',
+ ],
+ // resources: pyrra._config.resources,
+ ports: [{ containerPort: pyrra._config.port }],
+ securityContext: {
+ allowPrivilegeEscalation: false,
+ readOnlyRootFilesystem: true,
+ },
+ };
+
+ {
+ apiVersion: 'apps/v1',
+ kind: 'Deployment',
+ metadata: pyrra._apiMetadata,
+ spec: {
+ replicas: pyrra._config.replicas,
+ selector: {
+ matchLabels: pyrra.apiSelectorLabels,
+ },
+ strategy: {
+ rollingUpdate: {
+ maxSurge: 1,
+ maxUnavailable: 1,
+ },
+ },
+ template: {
+ metadata: { labels: pyrra._apiMetadata.labels },
+ spec: {
+ containers: [c],
+ // serviceAccountName: $.serviceAccount.metadata.name,
+ nodeSelector: { 'kubernetes.io/os': 'linux' },
+ },
+ },
+ },
+ },
+
+ _kubernetesMetadata:: {
+ name: pyrra._config.name + '-kubernetes',
+ namespace: pyrra._config.namespace,
+ labels: pyrra._config.commonLabels {
+ 'app.kubernetes.io/component': 'kubernetes',
+ },
+ },
+ kubernetesSelectorLabels:: {
+ [labelName]: pyrra._kubernetesMetadata.labels[labelName]
+ for labelName in std.objectFields(pyrra._kubernetesMetadata.labels)
+ if !std.setMember(labelName, ['app.kubernetes.io/version'])
+ },
+
+ kubernetesServiceAccount: {
+ apiVersion: 'v1',
+ kind: 'ServiceAccount',
+ metadata: pyrra._kubernetesMetadata,
+ },
+
+ kubernetesClusterRole: {
+ apiVersion: 'rbac.authorization.k8s.io/v1',
+ kind: 'ClusterRole',
+ metadata: pyrra._kubernetesMetadata,
+ rules: [{
+ apiGroups: ['monitoring.coreos.com'],
+ resources: ['prometheusrules'],
+ verbs: ['create', 'delete', 'get', 'list', 'patch', 'update', 'watch'],
+ }, {
+ apiGroups: ['monitoring.coreos.com'],
+ resources: ['prometheusrules/status'],
+ verbs: ['get'],
+ }, {
+ apiGroups: ['pyrra.dev'],
+ resources: ['servicelevelobjectives'],
+ verbs: ['create', 'delete', 'get', 'list', 'patch', 'update', 'watch'],
+ }, {
+ apiGroups: ['pyrra.dev'],
+ resources: ['servicelevelobjectives/status'],
+ verbs: ['get', 'patch', 'update'],
+ }],
+ },
+
+ kubernetesClusterRoleBinding: {
+ apiVersion: 'rbac.authorization.k8s.io/v1',
+ kind: 'ClusterRoleBinding',
+ metadata: pyrra._kubernetesMetadata,
+ roleRef: {
+ apiGroup: 'rbac.authorization.k8s.io',
+ kind: 'ClusterRole',
+ name: pyrra.kubernetesClusterRole.metadata.name,
+ },
+ subjects: [{
+ kind: 'ServiceAccount',
+ name: pyrra.kubernetesServiceAccount.metadata.name,
+ namespace: pyrra._config.namespace,
+ }],
+ },
+
+ kubernetesService: {
+ apiVersion: 'v1',
+ kind: 'Service',
+ metadata: pyrra._kubernetesMetadata,
+ spec: {
+ ports: [
+ { name: 'http', targetPort: 9444, port: 9444 },
+ ],
+ selector: pyrra.kubernetesSelectorLabels,
+ },
+ },
+
+ kubernetesDeployment:
+ local c = {
+ name: pyrra._config.name,
+ image: pyrra._config.image,
+ args: [
+ 'kubernetes',
+ ],
+ // resources: pyrra._config.resources,
+ ports: [{ containerPort: pyrra._config.port }],
+ securityContext: {
+ allowPrivilegeEscalation: false,
+ readOnlyRootFilesystem: true,
+ },
+ };
+
+ {
+ apiVersion: 'apps/v1',
+ kind: 'Deployment',
+ metadata: pyrra._kubernetesMetadata {
+ name: pyrra._config.name + '-kubernetes',
+ },
+ spec: {
+ replicas: pyrra._config.replicas,
+ selector: {
+ matchLabels: pyrra.kubernetesSelectorLabels,
+ },
+ strategy: {
+ rollingUpdate: {
+ maxSurge: 1,
+ maxUnavailable: 1,
+ },
+ },
+ template: {
+ metadata: { labels: pyrra._kubernetesMetadata.labels },
+ spec: {
+ containers: [c],
+ serviceAccountName: pyrra.kubernetesServiceAccount.metadata.name,
+ nodeSelector: { 'kubernetes.io/os': 'linux' },
+ },
+ },
+ },
+ },
+
+ // Most of these should eventually be moved to the components themselves.
+ // For now, this is a good start to have everything in one place.
+ 'slo-apiserver-read-response-errors': {
+ apiVersion: 'pyrra.dev/v1alpha1',
+ kind: 'ServiceLevelObjective',
+ metadata: {
+ name: 'apiserver-read-response-errors',
+ namespace: pyrra._config.namespace,
+ labels: {
+ prometheus: 'k8s',
+ role: 'alert-rules',
+ },
+ },
+ spec: {
+ target: '99',
+ window: '2w',
+ description: '',
+ indicator: {
+ ratio: {
+ errors: {
+ metric: 'apiserver_request_total{job="apiserver",verb=~"LIST|GET",code=~"5.."}',
+ },
+ total: {
+ metric: 'apiserver_request_total{job="apiserver",verb=~"LIST|GET"}',
+ },
+ },
+ },
+ },
+ },
+
+ 'slo-apiserver-write-response-errors': {
+ apiVersion: 'pyrra.dev/v1alpha1',
+ kind: 'ServiceLevelObjective',
+ metadata: {
+ name: 'apiserver-write-response-errors',
+ namespace: pyrra._config.namespace,
+ labels: {
+ prometheus: 'k8s',
+ role: 'alert-rules',
+ },
+ },
+ spec: {
+ target: '99',
+ window: '2w',
+ description: '',
+ indicator: {
+ ratio: {
+ errors: {
+ metric: 'apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",code=~"5.."}',
+ },
+ total: {
+ metric: 'apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}',
+ },
+ },
+ },
+ },
+ },
+
+ 'slo-apiserver-read-resource-latency': {
+ apiVersion: 'pyrra.dev/v1alpha1',
+ kind: 'ServiceLevelObjective',
+ metadata: {
+ name: 'apiserver-read-resource-latency',
+ namespace: pyrra._config.namespace,
+ labels: {
+ prometheus: 'k8s',
+ role: 'alert-rules',
+ },
+ },
+ spec: {
+ target: '99',
+ window: '2w',
+ description: '',
+ indicator: {
+ latency: {
+ success: {
+ metric: 'apiserver_request_duration_seconds_bucket{job="apiserver",scope=~"resource|",verb=~"LIST|GET",le="0.1"}',
+ },
+ total: {
+ metric: 'apiserver_request_duration_seconds_count{job="apiserver",scope=~"resource|",verb=~"LIST|GET"}',
+ },
+ },
+ },
+ },
+ },
+
+ 'slo-apiserver-read-namespace-latency': {
+ apiVersion: 'pyrra.dev/v1alpha1',
+ kind: 'ServiceLevelObjective',
+ metadata: {
+ name: 'apiserver-read-namespace-latency',
+ namespace: pyrra._config.namespace,
+ labels: {
+ prometheus: 'k8s',
+ role: 'alert-rules',
+ },
+ },
+ spec: {
+ target: '99',
+ window: '2w',
+ description: '',
+ indicator: {
+ latency: {
+ success: {
+ metric: 'apiserver_request_duration_seconds_bucket{job="apiserver",scope=~"namespace|",verb=~"LIST|GET",le="5"}',
+ },
+ total: {
+ metric: 'apiserver_request_duration_seconds_count{job="apiserver",scope=~"namespace|",verb=~"LIST|GET"}',
+ },
+ },
+ },
+ },
+ },
+
+ 'slo-apiserver-read-cluster-latency': {
+ apiVersion: 'pyrra.dev/v1alpha1',
+ kind: 'ServiceLevelObjective',
+ metadata: {
+ name: 'apiserver-read-cluster-latency',
+ namespace: pyrra._config.namespace,
+ labels: {
+ prometheus: 'k8s',
+ role: 'alert-rules',
+ },
+ },
+ spec: {
+ target: '99',
+ window: '2w',
+ description: '',
+ indicator: {
+ latency: {
+ success: {
+ metric: 'apiserver_request_duration_seconds_bucket{job="apiserver",scope=~"cluster|",verb=~"LIST|GET",le="5"}',
+ },
+ total: {
+ metric: 'apiserver_request_duration_seconds_count{job="apiserver",scope=~"cluster|",verb=~"LIST|GET"}',
+ },
+ },
+ },
+ },
+ },
+
+ 'slo-kubelet-request-errors': {
+ apiVersion: 'pyrra.dev/v1alpha1',
+ kind: 'ServiceLevelObjective',
+ metadata: {
+ name: 'kubelet-request-errors',
+ namespace: pyrra._config.namespace,
+ labels: {
+ prometheus: 'k8s',
+ role: 'alert-rules',
+ },
+ },
+ spec: {
+ target: '99',
+ window: '2w',
+ description: '',
+ indicator: {
+ ratio: {
+ errors: {
+ metric: 'rest_client_requests_total{job="kubelet",code=~"5.."}',
+ },
+ total: {
+ metric: 'rest_client_requests_total{job="kubelet"}',
+ },
+ },
+ },
+ },
+ },
+
+ 'slo-kubelet-runtime-errors': {
+ apiVersion: 'pyrra.dev/v1alpha1',
+ kind: 'ServiceLevelObjective',
+ metadata: {
+ name: 'kubelet-runtime-errors',
+ namespace: pyrra._config.namespace,
+ labels: {
+ prometheus: 'k8s',
+ role: 'alert-rules',
+ },
+ },
+ spec: {
+ target: '99',
+ window: '2w',
+ description: '',
+ indicator: {
+ ratio: {
+ errors: {
+ metric: 'kubelet_runtime_operations_errors_total{job="kubelet"}',
+ },
+ total: {
+ metric: 'kubelet_runtime_operations_total{job="kubelet"}',
+ },
+ },
+ },
+ },
+ },
+
+ 'slo-coredns-response-errors': {
+ apiVersion: 'pyrra.dev/v1alpha1',
+ kind: 'ServiceLevelObjective',
+ metadata: {
+ name: 'coredns-response-errors',
+ namespace: pyrra._config.namespace,
+ labels: {
+ prometheus: 'k8s',
+ role: 'alert-rules',
+ },
+ },
+ spec: {
+ target: '99.99',
+ window: '2w',
+ description: '',
+ indicator: {
+ ratio: {
+ errors: {
+ metric: 'coredns_dns_responses_total{job="kube-dns",rcode="SERVFAIL"}',
+ },
+ total: {
+ metric: 'coredns_dns_responses_total{job="kube-dns"}',
+ },
+ },
+ },
+ },
+ },
+
+ 'slo-prometheus-operator-reconcile-errors': {
+ apiVersion: 'pyrra.dev/v1alpha1',
+ kind: 'ServiceLevelObjective',
+ metadata: {
+ name: 'prometheus-operator-reconcile-errors',
+ namespace: pyrra._config.namespace,
+ labels: {
+ prometheus: 'k8s',
+ role: 'alert-rules',
+ },
+ },
+ spec: {
+ target: '95',
+ window: '2w',
+ description: '',
+ indicator: {
+ ratio: {
+ errors: {
+ metric: 'prometheus_operator_reconcile_errors_total{job="prometheus-operator"}',
+ },
+ total: {
+ metric: 'prometheus_operator_reconcile_operations_total{job="prometheus-operator"}',
+ },
+ grouping: ['controller'],
+ },
+ },
+ },
+ },
+
+ 'slo-prometheus-operator-http-errors': {
+ apiVersion: 'pyrra.dev/v1alpha1',
+ kind: 'ServiceLevelObjective',
+ metadata: {
+ name: 'prometheus-operator-http-errors',
+ namespace: pyrra._config.namespace,
+ labels: {
+ prometheus: 'k8s',
+ role: 'alert-rules',
+ },
+ },
+ spec: {
+ target: '99.5',
+ window: '2w',
+ description: '',
+ indicator: {
+ ratio: {
+ errors: {
+ metric: 'prometheus_operator_kubernetes_client_http_requests_total{job="prometheus-operator",status_code=~"5.."}',
+ },
+ total: {
+ metric: 'prometheus_operator_kubernetes_client_http_requests_total{job="prometheus-operator"}',
+ },
+ },
+ },
+ },
+ },
+
+ 'slo-prometheus-rule-evaluation-failures': {
+ apiVersion: 'pyrra.dev/v1alpha1',
+ kind: 'ServiceLevelObjective',
+ metadata: {
+ name: 'prometheus-rule-evaluation-failures',
+ namespace: pyrra._config.namespace,
+ labels: {
+ prometheus: 'k8s',
+ role: 'alert-rules',
+ },
+ },
+ spec: {
+ target: '99.99',
+ window: '2w',
+ description: 'Rule and alerting rules are being evaluated every few seconds. This needs to work for recording rules to be created and most importantly for alerts to be evaluated.',
+ indicator: {
+ ratio: {
+ errors: {
+ metric: 'prometheus_rule_evaluation_failures_total{job="prometheus-k8s"}',
+ },
+ total: {
+ metric: 'prometheus_rule_evaluations_total{job="prometheus-k8s"}',
+ },
+ },
+ },
+ },
+ },
+
+ 'slo-prometheus-sd-kubernetes-errors': {
+ apiVersion: 'pyrra.dev/v1alpha1',
+ kind: 'ServiceLevelObjective',
+ metadata: {
+ name: 'prometheus-sd-kubernetes-errors',
+ namespace: pyrra._config.namespace,
+ labels: {
+ prometheus: 'k8s',
+ role: 'alert-rules',
+ },
+ },
+ spec: {
+ target: '99',
+ window: '2w',
+ description: 'If there are too many errors Prometheus is having a bad time discovering new Kubernetes services.',
+ indicator: {
+ ratio: {
+ errors: {
+ metric: 'prometheus_sd_kubernetes_http_request_total{job="prometheus-k8s",status_code=~"5..|<error>"}',
+ },
+ total: {
+ metric: 'prometheus_sd_kubernetes_http_request_total{job="prometheus-k8s"}',
+ },
+ },
+ },
+ },
+ },
+
+ 'slo-prometheus-query-errors': {
+ apiVersion: 'pyrra.dev/v1alpha1',
+ kind: 'ServiceLevelObjective',
+ metadata: {
+ name: 'prometheus-query-errors',
+ namespace: pyrra._config.namespace,
+ labels: {
+ prometheus: 'k8s',
+ role: 'alert-rules',
+ },
+ },
+ spec: {
+ target: '99',
+ window: '2w',
+ description: '',
+ indicator: {
+ ratio: {
+ grouping: ['handler'],
+ errors: {
+ metric: 'prometheus_http_requests_total{job="prometheus-k8s",handler=~"/api/v1/query.*",code=~"5.."}',
+ },
+ total: {
+ metric: 'prometheus_http_requests_total{job="prometheus-k8s",handler=~"/api/v1/query.*"}',
+ },
+ },
+ },
+ },
+ },
+
+ 'slo-prometheus-notification-errors': {
+ apiVersion: 'pyrra.dev/v1alpha1',
+ kind: 'ServiceLevelObjective',
+ metadata: {
+ name: 'prometheus-notification-errors',
+ namespace: pyrra._config.namespace,
+ labels: {
+ prometheus: 'k8s',
+ role: 'alert-rules',
+ },
+ },
+ spec: {
+ target: '99',
+ window: '2w',
+ description: '',
+ indicator: {
+ ratio: {
+ errors: {
+ metric: 'prometheus_notifications_errors_total{job="prometheus-k8s"}',
+ },
+ total: {
+ metric: 'prometheus_notifications_sent_total{job="prometheus-k8s"}',
+ },
+ },
+ },
+ },
+ },
+ },
+
+ pyrra: pyrra($.values.pyrra),
+}
diff --git a/jsonnet/kube-prometheus/jsonnetfile.json b/jsonnet/kube-prometheus/jsonnetfile.json
index b2d99cc7..c0d9adf2 100644
--- a/jsonnet/kube-prometheus/jsonnetfile.json
+++ b/jsonnet/kube-prometheus/jsonnetfile.json
@@ -104,6 +104,15 @@
"version": "main",
"name": "alertmanager"
},
+ {
+ "source": {
+ "git": {
+ "remote": "https://github.com/pyrra-dev/pyrra.git",
+ "subdir": "config/crd/bases"
+ }
+ },
+ "version": "main"
+ },
{
"source": {
"git": {
diff --git a/jsonnet/kube-prometheus/platforms/platforms.libsonnet b/jsonnet/kube-prometheus/platforms/platforms.libsonnet
index a3978a6c..1445761e 100644
--- a/jsonnet/kube-prometheus/platforms/platforms.libsonnet
+++ b/jsonnet/kube-prometheus/platforms/platforms.libsonnet
@@ -19,13 +19,14 @@ local platformPatch(p) = if p != null && std.objectHas(platforms, p) then platfo
alertmanager: {},
blackboxExporter: {},
grafana: {},
+ kubePrometheus: {},
+ kubernetesControlPlane: {},
kubeStateMetrics: {},
nodeExporter: {},
prometheus: {},
prometheusAdapter: {},
prometheusOperator: {},
- kubernetesControlPlane: {},
- kubePrometheus: {},
+ pyrra: {},
} + platformPatch($.values.common.platform),
alertmanager+: p.alertmanager,
@@ -38,4 +39,5 @@ local platformPatch(p) = if p != null && std.objectHas(platforms, p) then platfo
prometheusOperator+: p.prometheusOperator,
kubernetesControlPlane+: p.kubernetesControlPlane,
kubePrometheus+: p.kubePrometheus,
+ pyrra+: p.pyrra,
}
diff --git a/jsonnet/kube-prometheus/versions.json b/jsonnet/kube-prometheus/versions.json
index 5883dc1f..4b700d24 100644
--- a/jsonnet/kube-prometheus/versions.json
+++ b/jsonnet/kube-prometheus/versions.json
@@ -7,6 +7,7 @@
"prometheus": "2.34.0",
"prometheusAdapter": "0.9.1",
"prometheusOperator": "0.55.1",
+ "pyrra": "0.3.4",
"kubeRbacProxy": "0.11.0",
"configmapReload": "0.5.0"
}
diff --git a/jsonnetfile.lock.json b/jsonnetfile.lock.json
index 01938bd4..3e412a8f 100644
--- a/jsonnetfile.lock.json
+++ b/jsonnetfile.lock.json
@@ -144,6 +144,16 @@
"sum": "7mS7v4Tt0wypJhUrjbHNbSe5maAToU5qHFD8LsLs+jM=",
"name": "prometheus"
},
+ {
+ "source": {
+ "git": {
+ "remote": "https://github.com/pyrra-dev/pyrra.git",
+ "subdir": "config/crd/bases"
+ }
+ },
+ "version": "6b0b6c2f362bbbea151a6610e18870e7cb1c2b5b",
+ "sum": "GQ0GFKGdIWKx1b78VRs6jtC4SMqkBjT5jl65QUjPKK4="
+ },
{
"source": {
"git": {
--
GitLab