From c8063eb78a842d98c089cd21a9a15589e36d0e6b Mon Sep 17 00:00:00 2001
From: Polina Bungina <27892524+hughcapet@users.noreply.github.com>
Date: Thu, 30 Jan 2025 12:41:58 +0300
Subject: [PATCH] Protect Pods from disruptions during upgrades (#2844)
Co-authored-by: Felix Kunde <felix-kunde@gmx.de>
---
pkg/cluster/majorversionupgrade.go | 27 +++++++++++++++++++++++++++
1 file changed, 27 insertions(+)
diff --git a/pkg/cluster/majorversionupgrade.go b/pkg/cluster/majorversionupgrade.go
index b75702bc..d8a1fb91 100644
--- a/pkg/cluster/majorversionupgrade.go
+++ b/pkg/cluster/majorversionupgrade.go
@@ -106,6 +106,22 @@ func (c *Cluster) removeFailuresAnnotation() error {
return nil
}
+func (c *Cluster) criticalOperationLabel(pods []v1.Pod, value *string) error {
+ metadataReq := map[string]map[string]map[string]*string{"metadata": {"labels": {"critical-operation": value}}}
+
+ patchReq, err := json.Marshal(metadataReq)
+ if err != nil {
+ return fmt.Errorf("could not marshal ObjectMeta: %v", err)
+ }
+ for _, pod := range pods {
+ _, err = c.KubeClient.Pods(c.Namespace).Patch(context.TODO(), pod.Name, types.StrategicMergePatchType, patchReq, metav1.PatchOptions{})
+ if err != nil {
+ return err
+ }
+ }
+ return nil
+}
+
/*
Execute upgrade when mode is set to manual or full or when the owning team is allowed for upgrade (and mode is "off").
@@ -224,6 +240,17 @@ func (c *Cluster) majorVersionUpgrade() error {
if allRunning && masterPod != nil {
c.logger.Infof("healthy cluster ready to upgrade, current: %d desired: %d", c.currentMajorVersion, desiredVersion)
if c.currentMajorVersion < desiredVersion {
+ defer func() error {
+ if err = c.criticalOperationLabel(pods, nil); err != nil {
+ return fmt.Errorf("failed to remove critical-operation label: %s", err)
+ }
+ return nil
+ }()
+ val := "true"
+ if err = c.criticalOperationLabel(pods, &val); err != nil {
+ return fmt.Errorf("failed to assign critical-operation label: %s", err)
+ }
+
podName := &spec.NamespacedName{Namespace: masterPod.Namespace, Name: masterPod.Name}
c.logger.Infof("triggering major version upgrade on pod %s of %d pods", masterPod.Name, numberOfPods)
c.eventRecorder.Eventf(c.GetReference(), v1.EventTypeNormal, "Major Version Upgrade", "starting major version upgrade on pod %s of %d pods", masterPod.Name, numberOfPods)
--
GitLab