diff --git a/infrastructure/rook/cluster-on-pvc.yaml b/infrastructure/rook/cluster-on-pvc.yaml
deleted file mode 100644
index dccfc44075b90d06698b026264fa0ba26afdc8ef..0000000000000000000000000000000000000000
--- a/infrastructure/rook/cluster-on-pvc.yaml
+++ /dev/null
@@ -1,136 +0,0 @@
-#################################################################################################################
-# Define the settings for the rook-ceph cluster with common settings for a production cluster.
-# All nodes with available raw devices will be used for the Ceph cluster. At least three nodes are required
-# in this example. See the documentation for more details on storage settings available.
-
-# For example, to create the cluster:
-# kubectl create -f crds.yaml -f common.yaml -f operator.yaml
-# kubectl create -f cluster-on-pvc.yaml
-#################################################################################################################
-apiVersion: ceph.rook.io/v1
-kind: CephCluster
-metadata:
- name: rook-ceph
- namespace: rook-ceph # namespace:cluster
-spec:
- dataDirHostPath: /var/lib/rook
- mon:
- # Set the number of mons to be started. Must be an odd number, and is generally recommended to be 3.
- count: 3
- # The mons should be on unique nodes. For production, at least 3 nodes are recommended for this reason.
- # Mons should only be allowed on the same node for test environments where data loss is acceptable.
- allowMultiplePerNode: false
- # A volume claim template can be specified in which case new monitors (and
- # monitors created during fail over) will construct a PVC based on the
- # template for the monitor's primary storage. Changes to the template do not
- # affect existing monitors. Log data is stored on the HostPath under
- # dataDirHostPath. If no storage requirement is specified, a default storage
- # size appropriate for monitor data will be used.
- volumeClaimTemplate:
- spec:
- storageClassName: hcloud-volumes
- resources:
- requests:
- storage: 10Gi
- cephVersion:
- image: quay.io/ceph/ceph:v16.2.5
- allowUnsupported: false
- skipUpgradeChecks: false
- continueUpgradeAfterChecksEvenIfNotHealthy: false
- mgr:
- count: 1
- modules:
- - name: pg_autoscaler
- enabled: true
- dashboard:
- enabled: true
- ssl: true
- crashCollector:
- disable: true
- storage:
- storageClassDeviceSets:
- - name: set1
- # The number of OSDs to create from this device set
- count: 3
- # IMPORTANT: If volumes specified by the storageClassName are not portable across nodes
- # this needs to be set to false. For example, if using the local storage provisioner
- # this should be false.
- portable: true
- # Certain storage class in the Cloud are slow
- # Rook can configure the OSD running on PVC to accommodate that by tuning some of the Ceph internal
- # Currently, "gp2" has been identified as such
- tuneDeviceClass: true
- # Certain storage class in the Cloud are fast
- # Rook can configure the OSD running on PVC to accommodate that by tuning some of the Ceph internal
- # Currently, "managed-premium" has been identified as such
- tuneFastDeviceClass: false
- # whether to encrypt the deviceSet or not
- encrypted: true
- # Since the OSDs could end up on any node, an effort needs to be made to spread the OSDs
- # across nodes as much as possible. Unfortunately the pod anti-affinity breaks down
- # as soon as you have more than one OSD per node. The topology spread constraints will
- # give us an even spread on K8s 1.18 or newer.
- placement:
- topologySpreadConstraints:
- - maxSkew: 1
- topologyKey: kubernetes.io/hostname
- whenUnsatisfiable: ScheduleAnyway
- labelSelector:
- matchExpressions:
- - key: app
- operator: In
- values:
- - rook-ceph-osd
- preparePlacement:
- podAntiAffinity:
- preferredDuringSchedulingIgnoredDuringExecution:
- - weight: 100
- podAffinityTerm:
- labelSelector:
- matchExpressions:
- - key: app
- operator: In
- values:
- - rook-ceph-osd
- - key: app
- operator: In
- values:
- - rook-ceph-osd-prepare
- topologyKey: kubernetes.io/hostname
- topologySpreadConstraints:
- - maxSkew: 1
- # IMPORTANT: If you don't have zone labels, change this to another key such as kubernetes.io/hostname
- topologyKey: kubernetes.io/hostname
- whenUnsatisfiable: DoNotSchedule
- labelSelector:
- matchExpressions:
- - key: app
- operator: In
- values:
- - rook-ceph-osd-prepare
- resources:
- # These are the OSD daemon limits. For OSD prepare limits, see the separate section below for "prepareosd" resources
- # limits:
- # cpu: "500m"
- # memory: "4Gi"
- # requests:
- # cpu: "500m"
- # memory: "4Gi"
- volumeClaimTemplates:
- - metadata:
- name: data
- spec:
- resources:
- requests:
- storage: 14Gi
- # IMPORTANT: Change the storage class depending on your environment (e.g. local-storage, gp2)
- storageClassName: hcloud-volumes
- volumeMode: Block
- accessModes:
- - ReadWriteOnce
- disruptionManagement:
- managePodBudgets: true
- osdMaintenanceTimeout: 30
- pgHealthCheckTimeout: 0
- manageMachineDisruptionBudgets: false
- machineDisruptionBudgetNamespace: openshift-machine-api
diff --git a/infrastructure/rook/kustomization.yaml b/infrastructure/rook/kustomization.yaml
index a9494c8d7c36abf7fe5a3d481e9e938091c4f7fe..5f6a24dee756b5ab9bc9d0797deb893e076ce5e1 100644
--- a/infrastructure/rook/kustomization.yaml
+++ b/infrastructure/rook/kustomization.yaml
@@ -2,13 +2,6 @@ apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
namespace: rook-ceph
resources:
- - https://git.shivering-isles.com/github-mirror/rook/rook/-/raw/v1.7.1/cluster/examples/kubernetes/ceph/crds.yaml
- - https://git.shivering-isles.com/github-mirror/rook/rook/-/raw/v1.7.1/cluster/examples/kubernetes/ceph/common.yaml
- - https://git.shivering-isles.com/github-mirror/rook/rook/-/raw/v1.7.1/cluster/examples/kubernetes/ceph/operator.yaml
- - https://git.shivering-isles.com/github-mirror/rook/rook/-/raw/v1.7.1/cluster/examples/kubernetes/ceph/csi/rbd/snapshotclass.yaml
- - cluster-on-pvc.yaml
- - storageclass.yaml
- - https://git.shivering-isles.com/github-mirror/rook/rook/-/raw/v1.7.1/cluster/examples/kubernetes/ceph/monitoring/service-monitor.yaml
- - https://git.shivering-isles.com/github-mirror/rook/rook/-/raw/v1.7.1/cluster/examples/kubernetes/ceph/monitoring/csi-metrics-service-monitor.yaml
- - prometheus-ceph-rules.yaml
- - prometheus-ceph-rules-external.yaml
+ - namespace.yaml
+ - repository.yaml
+ - release.yaml
diff --git a/infrastructure/rook/namespace.yaml b/infrastructure/rook/namespace.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..f046b87cda3f6830987e3fd4b3885f5944453d5d
--- /dev/null
+++ b/infrastructure/rook/namespace.yaml
@@ -0,0 +1,6 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+ name: rook-ceph
+ labels:
+ name: rook-ceph
diff --git a/infrastructure/rook/prometheus-ceph-rules-external.yaml b/infrastructure/rook/prometheus-ceph-rules-external.yaml
deleted file mode 100644
index e31ab3cb788487d9a243f40cb94cbd751ddd8d0f..0000000000000000000000000000000000000000
--- a/infrastructure/rook/prometheus-ceph-rules-external.yaml
+++ /dev/null
@@ -1,37 +0,0 @@
-apiVersion: monitoring.coreos.com/v1
-kind: PrometheusRule
-metadata:
- labels:
- role: alert-rules
- name: prometheus-ceph-rules-extermal
- namespace: rook-ceph
-spec:
- groups:
- - name: persistent-volume-alert.rules
- rules:
- - alert: PersistentVolumeUsageNearFull
- annotations:
- description: PVC {{ $labels.persistentvolumeclaim }} utilization has crossed
- 75%. Free up some space.
- message: PVC {{ $labels.persistentvolumeclaim }} is nearing full. Data deletion
- is required.
- severity_level: warning
- storage_type: ceph
- expr: |
- (kubelet_volume_stats_used_bytes * on (namespace,persistentvolumeclaim) group_left(storageclass, provisioner) (kube_persistentvolumeclaim_info * on (storageclass) group_left(provisioner) kube_storageclass_info {provisioner=~"(.*rbd.csi.ceph.com)|(.*cephfs.csi.ceph.com)"})) / (kubelet_volume_stats_capacity_bytes * on (namespace,persistentvolumeclaim) group_left(storageclass, provisioner) (kube_persistentvolumeclaim_info * on (storageclass) group_left(provisioner) kube_storageclass_info {provisioner=~"(.*rbd.csi.ceph.com)|(.*cephfs.csi.ceph.com)"})) > 0.75
- for: 5s
- labels:
- severity: warning
- - alert: PersistentVolumeUsageCritical
- annotations:
- description: PVC {{ $labels.persistentvolumeclaim }} utilization has crossed
- 85%. Free up some space immediately.
- message: PVC {{ $labels.persistentvolumeclaim }} is critically full. Data
- deletion is required.
- severity_level: error
- storage_type: ceph
- expr: |
- (kubelet_volume_stats_used_bytes * on (namespace,persistentvolumeclaim) group_left(storageclass, provisioner) (kube_persistentvolumeclaim_info * on (storageclass) group_left(provisioner) kube_storageclass_info {provisioner=~"(.*rbd.csi.ceph.com)|(.*cephfs.csi.ceph.com)"})) / (kubelet_volume_stats_capacity_bytes * on (namespace,persistentvolumeclaim) group_left(storageclass, provisioner) (kube_persistentvolumeclaim_info * on (storageclass) group_left(provisioner) kube_storageclass_info {provisioner=~"(.*rbd.csi.ceph.com)|(.*cephfs.csi.ceph.com)"})) > 0.85
- for: 5s
- labels:
- severity: critical
diff --git a/infrastructure/rook/prometheus-ceph-rules.yaml b/infrastructure/rook/prometheus-ceph-rules.yaml
deleted file mode 100644
index 77f0fa87c65ae1bd6a9a4910e0e7905d08d553ef..0000000000000000000000000000000000000000
--- a/infrastructure/rook/prometheus-ceph-rules.yaml
+++ /dev/null
@@ -1,351 +0,0 @@
-apiVersion: monitoring.coreos.com/v1
-kind: PrometheusRule
-metadata:
- labels:
- role: alert-rules
- name: prometheus-ceph-rules
- namespace: rook-ceph
-spec:
- groups:
- - name: ceph.rules
- rules:
- - expr: |
- kube_node_status_condition{condition="Ready",job="kube-state-metrics",status="true"} * on (node) group_right() max(label_replace(ceph_disk_occupation{job="rook-ceph-mgr"},"node","$1","exported_instance","(.*)")) by (node)
- record: cluster:ceph_node_down:join_kube
- - expr: |
- avg(topk by (ceph_daemon) (1, label_replace(label_replace(ceph_disk_occupation{job="rook-ceph-mgr"}, "instance", "$1", "exported_instance", "(.*)"), "device", "$1", "device", "/dev/(.*)")) * on(instance, device) group_right(ceph_daemon) topk by (instance,device) (1,(irate(node_disk_read_time_seconds_total[1m]) + irate(node_disk_write_time_seconds_total[1m]) / (clamp_min(irate(node_disk_reads_completed_total[1m]), 1) + irate(node_disk_writes_completed_total[1m])))))
- record: cluster:ceph_disk_latency:join_ceph_node_disk_irate1m
- - name: telemeter.rules
- rules:
- - expr: |
- count(ceph_osd_metadata{job="rook-ceph-mgr"})
- record: job:ceph_osd_metadata:count
- - expr: |
- count(kube_persistentvolume_info * on (storageclass) group_left(provisioner) kube_storageclass_info {provisioner=~"(.*rbd.csi.ceph.com)|(.*cephfs.csi.ceph.com)"})
- record: job:kube_pv:count
- - expr: |
- sum(ceph_pool_rd{job="rook-ceph-mgr"}+ ceph_pool_wr{job="rook-ceph-mgr"})
- record: job:ceph_pools_iops:total
- - expr: |
- sum(ceph_pool_rd_bytes{job="rook-ceph-mgr"}+ ceph_pool_wr_bytes{job="rook-ceph-mgr"})
- record: job:ceph_pools_iops_bytes:total
- - expr: |
- count(count(ceph_mon_metadata{job="rook-ceph-mgr"} or ceph_osd_metadata{job="rook-ceph-mgr"} or ceph_rgw_metadata{job="rook-ceph-mgr"} or ceph_mds_metadata{job="rook-ceph-mgr"} or ceph_mgr_metadata{job="rook-ceph-mgr"}) by(ceph_version))
- record: job:ceph_versions_running:count
- - name: ceph-mgr-status
- rules:
- - alert: CephMgrIsAbsent
- annotations:
- description: Ceph Manager has disappeared from Prometheus target discovery.
- message: Storage metrics collector service not available anymore.
- severity_level: critical
- storage_type: ceph
- expr: |
- absent(up{job="rook-ceph-mgr"} == 1)
- for: 5m
- labels:
- severity: critical
- - alert: CephMgrIsMissingReplicas
- annotations:
- description: Ceph Manager is missing replicas.
- message: Storage metrics collector service doesn't have required no of replicas.
- severity_level: warning
- storage_type: ceph
- expr: |
- sum(up{job="rook-ceph-mgr"}) < 1
- for: 5m
- labels:
- severity: warning
- - name: ceph-mds-status
- rules:
- - alert: CephMdsMissingReplicas
- annotations:
- description: Minimum required replicas for storage metadata service not available.
- Might affect the working of storage cluster.
- message: Insufficient replicas for storage metadata service.
- severity_level: warning
- storage_type: ceph
- expr: |
- sum(ceph_mds_metadata{job="rook-ceph-mgr"} == 1) < 2
- for: 5m
- labels:
- severity: warning
- - name: quorum-alert.rules
- rules:
- - alert: CephMonQuorumAtRisk
- annotations:
- description: Storage cluster quorum is low. Contact Support.
- message: Storage quorum at risk
- severity_level: error
- storage_type: ceph
- expr: |
- count(ceph_mon_quorum_status{job="rook-ceph-mgr"} == 1) <= ((count(ceph_mon_metadata{job="rook-ceph-mgr"}) % 2) + 1)
- for: 15m
- labels:
- severity: critical
- - alert: CephMonHighNumberOfLeaderChanges
- annotations:
- description: Ceph Monitor {{ $labels.ceph_daemon }} on host {{ $labels.hostname
- }} has seen {{ $value | printf "%.2f" }} leader changes per minute recently.
- message: Storage Cluster has seen many leader changes recently.
- severity_level: warning
- storage_type: ceph
- expr: |
- (ceph_mon_metadata{job="rook-ceph-mgr"} * on (ceph_daemon) group_left() (rate(ceph_mon_num_elections{job="rook-ceph-mgr"}[5m]) * 60)) > 0.95
- for: 5m
- labels:
- severity: warning
- - name: ceph-node-alert.rules
- rules:
- - alert: CephNodeDown
- annotations:
- description: Storage node {{ $labels.node }} went down. Please check the node
- immediately.
- message: Storage node {{ $labels.node }} went down
- severity_level: error
- storage_type: ceph
- expr: |
- cluster:ceph_node_down:join_kube == 0
- for: 30s
- labels:
- severity: critical
- - name: osd-alert.rules
- rules:
- - alert: CephOSDCriticallyFull
- annotations:
- description: Utilization of storage device {{ $labels.ceph_daemon }} of device_class
- type {{$labels.device_class}} has crossed 80% on host {{ $labels.hostname
- }}. Immediately free up some space or add capacity of type {{$labels.device_class}}.
- message: Back-end storage device is critically full.
- severity_level: error
- storage_type: ceph
- expr: |
- (ceph_osd_metadata * on (ceph_daemon) group_right(device_class) (ceph_osd_stat_bytes_used / ceph_osd_stat_bytes)) >= 0.80
- for: 40s
- labels:
- severity: critical
- - alert: CephOSDFlapping
- annotations:
- description: Storage daemon {{ $labels.ceph_daemon }} has restarted 5 times
- in last 5 minutes. Please check the pod events or ceph status to find out
- the cause.
- message: Ceph storage osd flapping.
- severity_level: error
- storage_type: ceph
- expr: |
- changes(ceph_osd_up[5m]) >= 10
- for: 0s
- labels:
- severity: critical
- - alert: CephOSDNearFull
- annotations:
- description: Utilization of storage device {{ $labels.ceph_daemon }} of device_class
- type {{$labels.device_class}} has crossed 75% on host {{ $labels.hostname
- }}. Immediately free up some space or add capacity of type {{$labels.device_class}}.
- message: Back-end storage device is nearing full.
- severity_level: warning
- storage_type: ceph
- expr: |
- (ceph_osd_metadata * on (ceph_daemon) group_right(device_class) (ceph_osd_stat_bytes_used / ceph_osd_stat_bytes)) >= 0.75
- for: 40s
- labels:
- severity: warning
- - alert: CephOSDDiskNotResponding
- annotations:
- description: Disk device {{ $labels.device }} not responding, on host {{ $labels.host
- }}.
- message: Disk not responding
- severity_level: error
- storage_type: ceph
- expr: |
- label_replace((ceph_osd_in == 1 and ceph_osd_up == 0),"disk","$1","ceph_daemon","osd.(.*)") + on(ceph_daemon) group_left(host, device) label_replace(ceph_disk_occupation,"host","$1","exported_instance","(.*)")
- for: 1m
- labels:
- severity: critical
- - alert: CephOSDDiskUnavailable
- annotations:
- description: Disk device {{ $labels.device }} not accessible on host {{ $labels.host
- }}.
- message: Disk not accessible
- severity_level: error
- storage_type: ceph
- expr: |
- label_replace((ceph_osd_in == 0 and ceph_osd_up == 0),"disk","$1","ceph_daemon","osd.(.*)") + on(ceph_daemon) group_left(host, device) label_replace(ceph_disk_occupation,"host","$1","exported_instance","(.*)")
- for: 1m
- labels:
- severity: critical
- - alert: CephOSDSlowOps
- annotations:
- description: '{{ $value }} Ceph OSD requests are taking too long to process.
- Please check ceph status to find out the cause.'
- message: OSD requests are taking too long to process.
- severity_level: warning
- storage_type: ceph
- expr: |
- ceph_healthcheck_slow_ops > 0
- for: 30s
- labels:
- severity: warning
- - alert: CephDataRecoveryTakingTooLong
- annotations:
- description: Data recovery has been active for too long. Contact Support.
- message: Data recovery is slow
- severity_level: warning
- storage_type: ceph
- expr: |
- ceph_pg_undersized > 0
- for: 2h
- labels:
- severity: warning
- - alert: CephPGRepairTakingTooLong
- annotations:
- description: Self heal operations taking too long. Contact Support.
- message: Self heal problems detected
- severity_level: warning
- storage_type: ceph
- expr: |
- ceph_pg_inconsistent > 0
- for: 1h
- labels:
- severity: warning
- - name: persistent-volume-alert.rules
- rules:
- - alert: PersistentVolumeUsageNearFull
- annotations:
- description: PVC {{ $labels.persistentvolumeclaim }} utilization has crossed
- 75%. Free up some space or expand the PVC.
- message: PVC {{ $labels.persistentvolumeclaim }} is nearing full. Data deletion
- or PVC expansion is required.
- severity_level: warning
- storage_type: ceph
- expr: |
- (kubelet_volume_stats_used_bytes * on (namespace,persistentvolumeclaim) group_left(storageclass, provisioner) (kube_persistentvolumeclaim_info * on (storageclass) group_left(provisioner) kube_storageclass_info {provisioner=~"(.*rbd.csi.ceph.com)|(.*cephfs.csi.ceph.com)"})) / (kubelet_volume_stats_capacity_bytes * on (namespace,persistentvolumeclaim) group_left(storageclass, provisioner) (kube_persistentvolumeclaim_info * on (storageclass) group_left(provisioner) kube_storageclass_info {provisioner=~"(.*rbd.csi.ceph.com)|(.*cephfs.csi.ceph.com)"})) > 0.75
- for: 5s
- labels:
- severity: warning
- - alert: PersistentVolumeUsageCritical
- annotations:
- description: PVC {{ $labels.persistentvolumeclaim }} utilization has crossed
- 85%. Free up some space or expand the PVC immediately.
- message: PVC {{ $labels.persistentvolumeclaim }} is critically full. Data
- deletion or PVC expansion is required.
- severity_level: error
- storage_type: ceph
- expr: |
- (kubelet_volume_stats_used_bytes * on (namespace,persistentvolumeclaim) group_left(storageclass, provisioner) (kube_persistentvolumeclaim_info * on (storageclass) group_left(provisioner) kube_storageclass_info {provisioner=~"(.*rbd.csi.ceph.com)|(.*cephfs.csi.ceph.com)"})) / (kubelet_volume_stats_capacity_bytes * on (namespace,persistentvolumeclaim) group_left(storageclass, provisioner) (kube_persistentvolumeclaim_info * on (storageclass) group_left(provisioner) kube_storageclass_info {provisioner=~"(.*rbd.csi.ceph.com)|(.*cephfs.csi.ceph.com)"})) > 0.85
- for: 5s
- labels:
- severity: critical
- - name: cluster-state-alert.rules
- rules:
- - alert: CephClusterErrorState
- annotations:
- description: Storage cluster is in error state for more than 10m.
- message: Storage cluster is in error state
- severity_level: error
- storage_type: ceph
- expr: |
- ceph_health_status{job="rook-ceph-mgr"} > 1
- for: 10m
- labels:
- severity: critical
- - alert: CephClusterWarningState
- annotations:
- description: Storage cluster is in warning state for more than 10m.
- message: Storage cluster is in degraded state
- severity_level: warning
- storage_type: ceph
- expr: |
- ceph_health_status{job="rook-ceph-mgr"} == 1
- for: 10m
- labels:
- severity: warning
- - alert: CephOSDVersionMismatch
- annotations:
- description: There are {{ $value }} different versions of Ceph OSD components
- running.
- message: There are multiple versions of storage services running.
- severity_level: warning
- storage_type: ceph
- expr: |
- count(count(ceph_osd_metadata{job="rook-ceph-mgr"}) by (ceph_version)) > 1
- for: 10m
- labels:
- severity: warning
- - alert: CephMonVersionMismatch
- annotations:
- description: There are {{ $value }} different versions of Ceph Mon components
- running.
- message: There are multiple versions of storage services running.
- severity_level: warning
- storage_type: ceph
- expr: |
- count(count(ceph_mon_metadata{job="rook-ceph-mgr"}) by (ceph_version)) > 1
- for: 10m
- labels:
- severity: warning
- - name: cluster-utilization-alert.rules
- rules:
- - alert: CephClusterNearFull
- annotations:
- description: Storage cluster utilization has crossed 75% and will become read-only
- at 85%. Free up some space or expand the storage cluster.
- message: Storage cluster is nearing full. Data deletion or cluster expansion
- is required.
- severity_level: warning
- storage_type: ceph
- expr: |
- ceph_cluster_total_used_raw_bytes / ceph_cluster_total_bytes > 0.75
- for: 5s
- labels:
- severity: warning
- - alert: CephClusterCriticallyFull
- annotations:
- description: Storage cluster utilization has crossed 80% and will become read-only
- at 85%. Free up some space or expand the storage cluster immediately.
- message: Storage cluster is critically full and needs immediate data deletion
- or cluster expansion.
- severity_level: error
- storage_type: ceph
- expr: |
- ceph_cluster_total_used_raw_bytes / ceph_cluster_total_bytes > 0.80
- for: 5s
- labels:
- severity: critical
- - alert: CephClusterReadOnly
- annotations:
- description: Storage cluster utilization has crossed 85% and will become read-only
- now. Free up some space or expand the storage cluster immediately.
- message: Storage cluster is read-only now and needs immediate data deletion
- or cluster expansion.
- severity_level: error
- storage_type: ceph
- expr: |
- ceph_cluster_total_used_raw_bytes / ceph_cluster_total_bytes >= 0.85
- for: 0s
- labels:
- severity: critical
- - name: pool-quota.rules
- rules:
- - alert: CephPoolQuotaBytesNearExhaustion
- annotations:
- description: Storage pool {{ $labels.name }} quota usage has crossed 70%.
- message: Storage pool quota(bytes) is near exhaustion.
- severity_level: warning
- storage_type: ceph
- expr: |
- (ceph_pool_stored_raw * on (pool_id) group_left(name)ceph_pool_metadata) / ((ceph_pool_quota_bytes * on (pool_id) group_left(name)ceph_pool_metadata) > 0) > 0.70
- for: 1m
- labels:
- severity: warning
- - alert: CephPoolQuotaBytesCriticallyExhausted
- annotations:
- description: Storage pool {{ $labels.name }} quota usage has crossed 90%.
- message: Storage pool quota(bytes) is critically exhausted.
- severity_level: critical
- storage_type: ceph
- expr: |
- (ceph_pool_stored_raw * on (pool_id) group_left(name)ceph_pool_metadata) / ((ceph_pool_quota_bytes * on (pool_id) group_left(name)ceph_pool_metadata) > 0) > 0.90
- for: 1m
- labels:
- severity: critical
diff --git a/infrastructure/rook/release.yaml b/infrastructure/rook/release.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..ba31df96557af73dde2882b06e80a69677c9a9b1
--- /dev/null
+++ b/infrastructure/rook/release.yaml
@@ -0,0 +1,160 @@
+apiVersion: helm.toolkit.fluxcd.io/v2beta1
+kind: HelmRelease
+metadata:
+ name: rook-operator
+ namespace: rook-ceph
+spec:
+ releaseName: rook-ceph
+ chart:
+ spec:
+ chart: rook-ceph
+ sourceRef:
+ kind: HelmRepository
+ name: rook-release
+ version: v1.7.1
+ interval: 5m
+---
+apiVersion: helm.toolkit.fluxcd.io/v2beta1
+kind: HelmRelease
+metadata:
+ name: rook-ceph-cluster
+ namespace: rook-ceph
+spec:
+ releaseName: rook-ceph-cluster
+ chart:
+ spec:
+ chart: rook-ceph-cluster
+ sourceRef:
+ kind: HelmRepository
+ name: rook-release
+ version: v1.7.1
+ dependsOn:
+ - name: rook-operator
+ interval: 5m
+ values:
+ CephClusterSpec:
+ dataDirHostPath: /var/lib/rook
+ mon:
+ # Set the number of mons to be started. Must be an odd number, and is generally recommended to be 3.
+ count: 3
+ # The mons should be on unique nodes. For production, at least 3 nodes are recommended for this reason.
+ # Mons should only be allowed on the same node for test environments where data loss is acceptable.
+ allowMultiplePerNode: false
+ # A volume claim template can be specified in which case new monitors (and
+ # monitors created during fail over) will construct a PVC based on the
+ # template for the monitor's primary storage. Changes to the template do not
+ # affect existing monitors. Log data is stored on the HostPath under
+ # dataDirHostPath. If no storage requirement is specified, a default storage
+ # size appropriate for monitor data will be used.
+ volumeClaimTemplate:
+ spec:
+ storageClassName: hcloud-volumes
+ resources:
+ requests:
+ storage: 10Gi
+ skipUpgradeChecks: false
+ continueUpgradeAfterChecksEvenIfNotHealthy: false
+ mgr:
+ count: 1
+ modules:
+ - name: pg_autoscaler
+ enabled: true
+ dashboard:
+ enabled: false
+ crashCollector:
+ disable: true
+ storage:
+ storageClassDeviceSets:
+ - name: set1
+ # The number of OSDs to create from this device set
+ count: 3
+ # IMPORTANT: If volumes specified by the storageClassName are not portable across nodes
+ # this needs to be set to false. For example, if using the local storage provisioner
+ # this should be false.
+ portable: true
+ # Certain storage class in the Cloud are slow
+ # Rook can configure the OSD running on PVC to accommodate that by tuning some of the Ceph internal
+ # Currently, "gp2" has been identified as such
+ tuneDeviceClass: true
+ # Certain storage class in the Cloud are fast
+ # Rook can configure the OSD running on PVC to accommodate that by tuning some of the Ceph internal
+ # Currently, "managed-premium" has been identified as such
+ tuneFastDeviceClass: false
+ # whether to encrypt the deviceSet or not
+ encrypted: true
+ # Since the OSDs could end up on any node, an effort needs to be made to spread the OSDs
+ # across nodes as much as possible. Unfortunately the pod anti-affinity breaks down
+ # as soon as you have more than one OSD per node. The topology spread constraints will
+ # give us an even spread on K8s 1.18 or newer.
+ placement:
+ topologySpreadConstraints:
+ - maxSkew: 1
+ topologyKey: kubernetes.io/hostname
+ whenUnsatisfiable: ScheduleAnyway
+ labelSelector:
+ matchExpressions:
+ - key: app
+ operator: In
+ values:
+ - rook-ceph-osd
+ preparePlacement:
+ podAntiAffinity:
+ preferredDuringSchedulingIgnoredDuringExecution:
+ - weight: 100
+ podAffinityTerm:
+ labelSelector:
+ matchExpressions:
+ - key: app
+ operator: In
+ values:
+ - rook-ceph-osd
+ - key: app
+ operator: In
+ values:
+ - rook-ceph-osd-prepare
+ topologyKey: kubernetes.io/hostname
+ topologySpreadConstraints:
+ - maxSkew: 1
+ # IMPORTANT: If you don't have zone labels, change this to another key such as kubernetes.io/hostname
+ topologyKey: kubernetes.io/hostname
+ whenUnsatisfiable: DoNotSchedule
+ labelSelector:
+ matchExpressions:
+ - key: app
+ operator: In
+ values:
+ - rook-ceph-osd-prepare
+ resources:
+ # These are the OSD daemon limits. For OSD prepare limits, see the separate section below for "prepareosd" resources
+ # limits:
+ # cpu: "500m"
+ # memory: "4Gi"
+ # requests:
+ # cpu: "500m"
+ # memory: "4Gi"
+ volumeClaimTemplates:
+ - metadata:
+ name: data
+ spec:
+ resources:
+ requests:
+ storage: 14Gi
+ # IMPORTANT: Change the storage class depending on your environment (e.g. local-storage, gp2)
+ storageClassName: hcloud-volumes
+ volumeMode: Block
+ accessModes:
+ - ReadWriteOnce
+ disruptionManagement:
+ managePodBudgets: true
+ osdMaintenanceTimeout: 30
+ pgHealthCheckTimeout: 0
+ cephBlockPools:
+ - name: replicapool
+ spec:
+ failureDomain: host
+ replicated:
+ size: 2
+ requireSafeReplicaSize: true
+ storageClass:
+ enabled: true
+ isDefault: true
diff --git a/infrastructure/rook/repository.yaml b/infrastructure/rook/repository.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..81b7e3ee7cfe7fadbdc2f477b858626dce7c15d3
--- /dev/null
+++ b/infrastructure/rook/repository.yaml
@@ -0,0 +1,7 @@
+apiVersion: source.toolkit.fluxcd.io/v1beta1
+kind: HelmRepository
+metadata:
+ name: rook-release
+spec:
+ interval: 30m
+ url: https://charts.rook.io/release
diff --git a/infrastructure/rook/storageclass.yaml b/infrastructure/rook/storageclass.yaml
deleted file mode 100644
index b04a250f71504edff4e54a7a78f13fe06b57aa9f..0000000000000000000000000000000000000000
--- a/infrastructure/rook/storageclass.yaml
+++ /dev/null
@@ -1,76 +0,0 @@
-apiVersion: ceph.rook.io/v1
-kind: CephBlockPool
-metadata:
- name: replicapool
- namespace: rook-ceph
-spec:
- failureDomain: host
- replicated:
- size: 2
- # Disallow setting pool with replica 1, this could lead to data loss without recovery.
- # Make sure you're *ABSOLUTELY CERTAIN* that is what you want
- requireSafeReplicaSize: true
- # gives a hint (%) to Ceph in terms of expected consumption of the total cluster capacity of a given pool
- # for more info: https://docs.ceph.com/docs/master/rados/operations/placement-groups/#specifying-expected-pool-size
- #targetSizeRatio: .5
----
-apiVersion: storage.k8s.io/v1
-kind: StorageClass
-metadata:
- name: rook-ceph-block
- annotations:
- storageclass.kubernetes.io/is-default-class: 'true'
-# Change "rook-ceph" provisioner prefix to match the operator namespace if needed
-provisioner: rook-ceph.rbd.csi.ceph.com
-parameters:
- # clusterID is the namespace where the rook cluster is running
- # If you change this namespace, also change the namespace below where the secret namespaces are defined
- clusterID: rook-ceph # namespace:cluster
-
- # If you want to use erasure coded pool with RBD, you need to create
- # two pools. one erasure coded and one replicated.
- # You need to specify the replicated pool here in the `pool` parameter, it is
- # used for the metadata of the images.
- # The erasure coded pool must be set as the `dataPool` parameter below.
- #dataPool: ec-data-pool
- pool: replicapool
-
- # (optional) mapOptions is a comma-separated list of map options.
- # For krbd options refer
- # https://docs.ceph.com/docs/master/man/8/rbd/#kernel-rbd-krbd-options
- # For nbd options refer
- # https://docs.ceph.com/docs/master/man/8/rbd-nbd/#options
- # mapOptions: lock_on_read,queue_depth=1024
-
- # (optional) unmapOptions is a comma-separated list of unmap options.
- # For krbd options refer
- # https://docs.ceph.com/docs/master/man/8/rbd/#kernel-rbd-krbd-options
- # For nbd options refer
- # https://docs.ceph.com/docs/master/man/8/rbd-nbd/#options
- # unmapOptions: force
-
- # RBD image format. Defaults to "2".
- imageFormat: "2"
-
- # RBD image features. Available for imageFormat: "2". CSI RBD currently supports only `layering` feature.
- imageFeatures: layering
-
- # The secrets contain Ceph admin credentials. These are generated automatically by the operator
- # in the same namespace as the cluster.
- csi.storage.k8s.io/provisioner-secret-name: rook-csi-rbd-provisioner
- csi.storage.k8s.io/provisioner-secret-namespace: rook-ceph # namespace:cluster
- csi.storage.k8s.io/controller-expand-secret-name: rook-csi-rbd-provisioner
- csi.storage.k8s.io/controller-expand-secret-namespace: rook-ceph # namespace:cluster
- csi.storage.k8s.io/node-stage-secret-name: rook-csi-rbd-node
- csi.storage.k8s.io/node-stage-secret-namespace: rook-ceph # namespace:cluster
- # Specify the filesystem type of the volume. If not specified, csi-provisioner
- # will set default as `ext4`. Note that `xfs` is not recommended due to potential deadlock
- # in hyperconverged settings where the volume is mounted on the same node as the osds.
- csi.storage.k8s.io/fstype: ext4
-# uncomment the following to use rbd-nbd as mounter on supported nodes
-# **IMPORTANT**: If you are using rbd-nbd as the mounter, during upgrade you will be hit a ceph-csi
-# issue that causes the mount to be disconnected. You will need to follow special upgrade steps
-# to restart your application pods. Therefore, this option is not recommended.
-#mounter: rbd-nbd
-allowVolumeExpansion: true
-reclaimPolicy: Delete