feat(matrix): Define SLO for matrix-synapse

This patch implements a first SLO using sloth for the matrix-synapse
deployment. First tests with sloth for SLO overview.

While this should be neccesary, but it could be useful to build some
nice dashboards and compare to other solutions.

apps/k8s01/matrix/kustomization.yaml

@@ -5,6 +5,7 @@ resources:
   - ../../base/matrix
   - certificate.yaml
   - signing-key.yaml
+  - slo.yaml
   - ../../../shared/resourcequotas/default.yaml
 patchesStrategicMerge:
   - database-override.yaml

apps/k8s01/matrix/slo.yaml

+apiVersion: sloth.slok.dev/v1
+kind: PrometheusServiceLevel
+metadata:
+  name: requests-matrix-synapse
+  namespace: matrix
+spec:
+  service: "matrix-synapse"
+  slos:
+    - name: "requests-availability"
+      objective: 95
+      description: "Matrix SLO based on availability for HTTP request responses."
+      sli:
+        events:
+          errorQuery: sum(rate(nginx_ingress_controller_requests{exported_namespace="matrix",ingress="matrix-synapse",status=~"(5..|429)"}[{{.window}}]))
+          totalQuery: sum(rate(nginx_ingress_controller_requests{exported_namespace="matrix",ingress="matrix-synapse"}[{{.window}}]))
+      alerting:
+        name: MatrixHighErrorRate
+        labels:
+          category: "availability"
+        annotations:
+          summary: "High error rate on 'matrix-synapse' requests responses"