From 7756faec1f30ae721e2569d13d42daea72955a0a Mon Sep 17 00:00:00 2001
From: Stefan Prodan <stefan.prodan@gmail.com>
Date: Tue, 22 Mar 2022 15:16:39 +0200
Subject: [PATCH] Retry bootstrap operations on Git conflict errors When
running bootstrap in-parallel for many clusters that target the same
repository, the 2nd commit with the sync files fails with ` non-fast-forward
update`. We now detect the conflict, and we retry the operations by creating
a fresh clone from upstream.
Signed-off-by: Stefan Prodan <stefan.prodan@gmail.com>
---
internal/bootstrap/bootstrap_plain_git.go | 21 ++++++++++++++++++++-
1 file changed, 20 insertions(+), 1 deletion(-)
diff --git a/internal/bootstrap/bootstrap_plain_git.go b/internal/bootstrap/bootstrap_plain_git.go
index f661e936..a5979476 100644
--- a/internal/bootstrap/bootstrap_plain_git.go
+++ b/internal/bootstrap/bootstrap_plain_git.go
@@ -24,6 +24,7 @@ import (
"strings"
"time"
+ gogit "github.com/go-git/go-git/v5"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apimachinery/pkg/util/wait"
@@ -287,10 +288,28 @@ func (b *PlainGitBootstrapper) ReconcileSyncConfig(ctx context.Context, options
if err != nil && err != git.ErrNoStagedFiles {
return fmt.Errorf("failed to commit sync manifests: %w", err)
}
+
if err == nil {
b.logger.Successf("committed sync manifests to %q (%q)", b.branch, commit)
b.logger.Actionf("pushing sync manifests to %q", b.url)
- if err = b.git.Push(ctx, b.caBundle); err != nil {
+ err = b.git.Push(ctx, b.caBundle)
+ if err != nil {
+ if strings.HasPrefix(err.Error(), gogit.ErrNonFastForwardUpdate.Error()) {
+ b.logger.Waitingf("git conflict detected, retrying with a fresh clone")
+ if err := os.RemoveAll(b.git.Path()); err != nil {
+ return fmt.Errorf("failed to remove tmp dir: %w", err)
+ }
+ if err := os.Mkdir(b.git.Path(), 0o700); err != nil {
+ return fmt.Errorf("failed to recreate tmp dir: %w", err)
+ }
+ if err = retry(1, 2*time.Second, func() (err error) {
+ _, err = b.git.Clone(ctx, b.url, b.branch, b.caBundle)
+ return
+ }); err != nil {
+ return fmt.Errorf("failed to clone repository: %w", err)
+ }
+ return b.ReconcileSyncConfig(ctx, options)
+ }
return fmt.Errorf("failed to push sync manifests: %w", err)
}
} else {
--
GitLab