From 55c7b4fa18c8e2c6ef844d6319a2a3a49afbdb21 Mon Sep 17 00:00:00 2001 From: Slach Date: Thu, 29 Aug 2024 16:13:53 +0500 Subject: [PATCH] add retries to RemoveBackupRemote --- pkg/backup/delete.go | 2 +- pkg/backup/upload.go | 2 +- pkg/storage/general.go | 15 +++++++++++---- 3 files changed, 13 insertions(+), 6 deletions(-) diff --git a/pkg/backup/delete.go b/pkg/backup/delete.go index 14def780..3e171400 100644 --- a/pkg/backup/delete.go +++ b/pkg/backup/delete.go @@ -304,7 +304,7 @@ func (b *Backuper) RemoveBackupRemote(ctx context.Context, backupName string) er return err } - if err = bd.RemoveBackupRemote(ctx, backup); err != nil { + if err = bd.RemoveBackupRemote(ctx, backup, b.cfg); err != nil { log.Warn().Msgf("bd.RemoveBackup return error: %v", err) return err } diff --git a/pkg/backup/upload.go b/pkg/backup/upload.go index 86f25ba0..fc8446a7 100644 --- a/pkg/backup/upload.go +++ b/pkg/backup/upload.go @@ -279,7 +279,7 @@ func (b *Backuper) RemoveOldBackupsRemote(ctx context.Context) error { return err } - if err := b.dst.RemoveBackupRemote(ctx, backupToDelete); err != nil { + if err := b.dst.RemoveBackupRemote(ctx, backupToDelete, b.cfg); err != nil { log.Warn().Msgf("can't deleteKey %s return error : %v", backupToDelete.BackupName, err) } log.Info().Fields(map[string]interface{}{ diff --git a/pkg/storage/general.go b/pkg/storage/general.go index fc6f7a1e..532825fa 100644 --- a/pkg/storage/general.go +++ b/pkg/storage/general.go @@ -51,19 +51,26 @@ type BackupDestination struct { var metadataCacheLock sync.RWMutex -func (bd *BackupDestination) RemoveBackupRemote(ctx context.Context, backup Backup) error { +func (bd *BackupDestination) RemoveBackupRemote(ctx context.Context, backup Backup, cfg *config.Config) error { + retry := retrier.New(retrier.ConstantBackoff(cfg.General.RetriesOnFailure, cfg.General.RetriesDuration), nil) if bd.Kind() == "SFTP" || bd.Kind() == "FTP" { - return bd.DeleteFile(ctx, backup.BackupName) + return retry.RunCtx(ctx, func(ctx context.Context) error { + return bd.DeleteFile(ctx, backup.BackupName) + }) } return bd.Walk(ctx, backup.BackupName+"/", true, func(ctx context.Context, f RemoteFile) error { if bd.Kind() == "azblob" { if f.Size() > 0 || !f.LastModified().IsZero() { - return bd.DeleteFile(ctx, path.Join(backup.BackupName, f.Name())) + return retry.RunCtx(ctx, func(ctx context.Context) error { + return bd.DeleteFile(ctx, path.Join(backup.BackupName, f.Name())) + }) } else { return nil } } - return bd.DeleteFile(ctx, path.Join(backup.BackupName, f.Name())) + return retry.RunCtx(ctx, func(ctx context.Context) error { + return bd.DeleteFile(ctx, path.Join(backup.BackupName, f.Name())) + }) }) }