add retries to RemoveBackupRemote

Altinity · Aug 29, 2024 · 55c7b4f · 55c7b4f
1 parent dfa2811
commit 55c7b4f
Show file tree

Hide file tree

Showing 3 changed files with 13 additions and 6 deletions.
diff --git a/pkg/backup/delete.go b/pkg/backup/delete.go
@@ -304,7 +304,7 @@ func (b *Backuper) RemoveBackupRemote(ctx context.Context, backupName string) er
 				return err
 			}
 
-			if err = bd.RemoveBackupRemote(ctx, backup); err != nil {
+			if err = bd.RemoveBackupRemote(ctx, backup, b.cfg); err != nil {
 				log.Warn().Msgf("bd.RemoveBackup return error: %v", err)
 				return err
 			}

diff --git a/pkg/backup/upload.go b/pkg/backup/upload.go
@@ -279,7 +279,7 @@ func (b *Backuper) RemoveOldBackupsRemote(ctx context.Context) error {
 			return err
 		}
 
-		if err := b.dst.RemoveBackupRemote(ctx, backupToDelete); err != nil {
+		if err := b.dst.RemoveBackupRemote(ctx, backupToDelete, b.cfg); err != nil {
 			log.Warn().Msgf("can't deleteKey %s return error : %v", backupToDelete.BackupName, err)
 		}
 		log.Info().Fields(map[string]interface{}{

diff --git a/pkg/storage/general.go b/pkg/storage/general.go
@@ -51,19 +51,26 @@ type BackupDestination struct {
 
 var metadataCacheLock sync.RWMutex
 
-func (bd *BackupDestination) RemoveBackupRemote(ctx context.Context, backup Backup) error {
+func (bd *BackupDestination) RemoveBackupRemote(ctx context.Context, backup Backup, cfg *config.Config) error {
+	retry := retrier.New(retrier.ConstantBackoff(cfg.General.RetriesOnFailure, cfg.General.RetriesDuration), nil)
 	if bd.Kind() == "SFTP" || bd.Kind() == "FTP" {
-		return bd.DeleteFile(ctx, backup.BackupName)
+		return retry.RunCtx(ctx, func(ctx context.Context) error {
+			return bd.DeleteFile(ctx, backup.BackupName)
+		})
 	}
 	return bd.Walk(ctx, backup.BackupName+"/", true, func(ctx context.Context, f RemoteFile) error {
 		if bd.Kind() == "azblob" {
 			if f.Size() > 0 || !f.LastModified().IsZero() {
-				return bd.DeleteFile(ctx, path.Join(backup.BackupName, f.Name()))
+				return retry.RunCtx(ctx, func(ctx context.Context) error {
+					return bd.DeleteFile(ctx, path.Join(backup.BackupName, f.Name()))
+				})
 			} else {
 				return nil
 			}
 		}
-		return bd.DeleteFile(ctx, path.Join(backup.BackupName, f.Name()))
+		return retry.RunCtx(ctx, func(ctx context.Context) error {
+			return bd.DeleteFile(ctx, path.Join(backup.BackupName, f.Name()))
+		})
 	})
 }