fix(backup): only delete backups' resources when synchronization

controller/backup_backing_image_controller.go

@@ -214,7 +214,11 @@ func (bc *BackupBackingImageController) reconcile(backupBackingImageName string)
 
 	// Examine DeletionTimestamp to determine if object is under deletion
 	if !bbi.DeletionTimestamp.IsZero() {
-		if backupTarget.Spec.BackupTargetURL != "" {
+		needsCleanupRemoteData, err := checkIfRemoteDataCleanupIsNeeded(bbi, backupTarget)
+		if err != nil {
+			return errors.Wrap(err, "failed to check if it needs to delete remote backup backing image data")
+		}
+		if needsCleanupRemoteData {
 			backupTargetClient, err := newBackupTargetClientFromDefaultEngineImage(bc.ds, backupTarget)
 			if err != nil {
 				log.WithError(err).Warn("Failed to init backup target clients")

controller/backup_controller.go

@@ -280,8 +280,12 @@ func (bc *BackupController) reconcile(backupName string) (err error) {
 			return err
 		}
 
-		if backupTarget.Spec.BackupTargetURL != "" &&
-			backupVolume != nil && backupVolume.DeletionTimestamp == nil {
+		needsCleanupRemoteData, err := checkIfRemoteDataCleanupIsNeeded(backup, backupTarget)
+		if err != nil {
+			return errors.Wrap(err, "failed to check if it needs to delete remote backup data")
+		}
+
+		if needsCleanupRemoteData && backupVolume != nil && backupVolume.DeletionTimestamp == nil {
 			backupTargetClient, err := newBackupTargetClientFromDefaultEngineImage(bc.ds, backupTarget)
 			if err != nil {
 				log.WithError(err).Warn("Failed to init backup target clients")

controller/backup_target_controller.go

@@ -295,128 +295,128 @@
 	return newBackupTargetClient(ds, backupTarget, defaultEngineImage)
 }
 
 func (btc *BackupTargetController) reconcile(name string) (err error) {
 	backupTarget, err := btc.ds.GetBackupTarget(name)
 	if err != nil {
 		if !apierrors.IsNotFound(err) {
 			return err
 		}
 		return nil
 	}
 
 	log := getLoggerForBackupTarget(btc.logger, backupTarget)
 
 	// Every controller should do the clean up even it is not responsible for the CR
 	if backupTarget.Spec.BackupTargetURL == "" {
 		if err := btc.cleanUpAllMounts(backupTarget); err != nil {
 			log.WithError(err).Warn("Failed to clean up all mount points")
 		}
 	}
 
 	// Check the responsible node
 	defaultEngineImage, err := btc.ds.GetSettingValueExisted(types.SettingNameDefaultEngineImage)
 	if err != nil {
 		return err
 	}
 	isResponsible, err := btc.isResponsibleFor(backupTarget, defaultEngineImage)
 	if err != nil {
 		return nil
 	}
 	if !isResponsible {
 		return nil
 	}
 	if backupTarget.Status.OwnerID != btc.controllerID {
 		backupTarget.Status.OwnerID = btc.controllerID
 		backupTarget, err = btc.ds.UpdateBackupTargetStatus(backupTarget)
 		if err != nil {
 			// we don't mind others coming first
 			if apierrors.IsConflict(errors.Cause(err)) {
 				return nil
 			}
 			return err
 		}
 	}
 
 	// Check the controller should run synchronization
 	if !backupTarget.Status.LastSyncedAt.IsZero() &&
 		!backupTarget.Spec.SyncRequestedAt.After(backupTarget.Status.LastSyncedAt.Time) {
 		return nil
 	}
 
 	existingBackupTarget := backupTarget.DeepCopy()
 
 	syncTime := metav1.Time{Time: time.Now().UTC()}
 	syncTimeRequired := false
 
 	defer func() {
 		if err != nil {
 			return
 		}
 		if syncTimeRequired {
 			// If there is something wrong with the backup target config and Longhorn cannot launch the client,
 			// lacking the credential; for example, Longhorn won't even try to connect with the remote backupstore.
 			// In this case, the controller should not update `Status.LastSyncedAt`.
 			backupTarget.Status.LastSyncedAt = syncTime
 		}
 		if reflect.DeepEqual(existingBackupTarget.Status, backupTarget.Status) {
 			return
 		}
 		if _, err := btc.ds.UpdateBackupTargetStatus(backupTarget); err != nil && apierrors.IsConflict(errors.Cause(err)) {
 			log.WithError(err).Debugf("Requeue %v due to conflict", name)
 			btc.enqueueBackupTarget(backupTarget)
 		}
 	}()
 
 	if backupTarget.Spec.BackupTargetURL == "" {
 		backupTarget.Status.Available = false
 		backupTarget.Status.Conditions = types.SetCondition(backupTarget.Status.Conditions,
 			longhorn.BackupTargetConditionTypeUnavailable, longhorn.ConditionStatusTrue,
 			longhorn.BackupTargetConditionReasonUnavailable, "backup target URL is empty")
 		if err := btc.cleanupBackupVolumes(); err != nil {
 			return errors.Wrap(err, "failed to clean up BackupVolumes")
 		}
 
 		if err := btc.cleanupSystemBackups(); err != nil {
 			return errors.Wrap(err, "failed to clean up SystemBackups")
 		}
 
 		if err := btc.cleanupBackupBackingImages(); err != nil {
 			return errors.Wrap(err, "failed to clean up BackupBackingImages")
 		}
 
 		return nil
 	}
 
 	info, err := btc.getInfoFromBackupStore(backupTarget)
 	if err != nil {
 		backupTarget.Status.Available = false
 		backupTarget.Status.Conditions = types.SetCondition(backupTarget.Status.Conditions,
 			longhorn.BackupTargetConditionTypeUnavailable, longhorn.ConditionStatusTrue,
 			longhorn.BackupTargetConditionReasonUnavailable, err.Error())
 		log.WithError(err).Error("Failed to get info from backup store")
 		return nil // Ignore error to allow status update as well as preventing enqueue
 	}
 	syncTimeRequired = true // Errors beyond this point are NOT backup target related.
 
 	backupTarget.Status.Available = true
 	backupTarget.Status.Conditions = types.SetCondition(backupTarget.Status.Conditions,
 		longhorn.BackupTargetConditionTypeUnavailable, longhorn.ConditionStatusFalse,
 		"", "")
 
 	if err = btc.syncBackupVolume(info.backupStoreBackupVolumeNames, syncTime, log); err != nil {
 		return err
 	}
 
 	if err = btc.syncBackupBackingImage(info.backupStoreBackingImageNames, syncTime, log); err != nil {
 		return err
 	}
 
 	if err = btc.syncSystemBackup(info.backupStoreSystemBackups, log); err != nil {
 		return err
 	}
 	return nil
 }
 
 func (btc *BackupTargetController) cleanUpAllMounts(backupTarget *longhorn.BackupTarget) (err error) {
 	log := getLoggerForBackupTarget(btc.logger, backupTarget)
 	engineClientProxy, backupTargetClient, err := getBackupTarget(btc.controllerID, backupTarget, btc.ds, log, btc.proxyConnCounter)
@@ -506,9 +506,12 @@
 		log.Infof("Found %d backup volumes in the backup target that do not exist in the cluster and need to be deleted from the cluster", count)
 	}
 	for backupVolumeName := range backupVolumesToDelete {
-		log.WithField("backupVolume", backupVolumeName).Info("Deleting backup volume from cluster")
+		log.WithField("backupVolume", backupVolumeName).Info("Deleting BackupVolume not exist in backupstore")
+		if err = datastore.AddBackupVolumeDeleteCustomResourceOnlyLabel(btc.ds, backupVolumeName); err != nil {
+			return errors.Wrapf(err, "failed to add label delete-custom-resource-only to Backupvolume %s", backupVolumeName)
+		}
 		if err = btc.ds.DeleteBackupVolume(backupVolumeName); err != nil {
-			return errors.Wrapf(err, "failed to delete backup volume %s from cluster", backupVolumeName)
+			return errors.Wrapf(err, "failed to delete BackupVolume %s not exist in backupstore", backupVolumeName)
 		}
 	}
 
@@ -561,9 +564,12 @@
 		log.Infof("Found %d backup backing images in the cluster that do not exist in the backup target and need to be deleted", count)
 	}
 	for backupBackingImageName := range backupBackingImagesToDelete {
-		log.WithField("backupBackingImage", backupBackingImageName).Info("Deleting backup backing image from cluster")
+		log.WithField("backupBackingImage", backupBackingImageName).Info("Deleting BackupBackingImage not exist in backupstore")
+		if err = datastore.AddBackupBackingImageDeleteCustomResourceOnlyLabel(btc.ds, backupBackingImageName); err != nil {
+			return errors.Wrapf(err, "failed to add label delete-custom-resource-only to BackupBackingImage %s", backupBackingImageName)
+		}
 		if err = btc.ds.DeleteBackupBackingImage(backupBackingImageName); err != nil {
-			return errors.Wrapf(err, "failed to delete backup backing image %s from cluster", backupBackingImageName)
+			return errors.Wrapf(err, "failed to delete BackupBackingImage %s not exist in backupstore", backupBackingImageName)
 		}
 	}
 
@@ -617,6 +623,9 @@
 	delSystemBackupsInCluster := clusterReadySystemBackupNames.Difference(backupstoreSystemBackupNames)
 	for name := range delSystemBackupsInCluster {
 		log.WithField("systemBackup", name).Info("Deleting SystemBackup not exist in backupstore")
+		if err = datastore.AddSystemBackupDeleteCustomResourceOnlyLabel(btc.ds, name); err != nil {
+			return errors.Wrapf(err, "failed to add label delete-custom-resource-only to SystemBackup %v", name)
+		}
 		if err = btc.ds.DeleteSystemBackup(name); err != nil {
 			return errors.Wrapf(err, "failed to delete SystemBackup %v not exist in backupstore", name)
 		}
@@ -666,6 +675,10 @@
 
 	var errs []string
 	for backupVolumeName := range clusterBackupVolumes {
+		if err = datastore.AddBackupVolumeDeleteCustomResourceOnlyLabel(btc.ds, backupVolumeName); err != nil {
+			errs = append(errs, err.Error())
+			continue
+		}
 		if err = btc.ds.DeleteBackupVolume(backupVolumeName); err != nil && !apierrors.IsNotFound(err) {
 			errs = append(errs, err.Error())
 			continue
@@ -686,6 +699,10 @@
 
 	var errs []string
 	for backupBackingImageName := range clusterBackupBackingImages {
+		if err = datastore.AddBackupBackingImageDeleteCustomResourceOnlyLabel(btc.ds, backupBackingImageName); err != nil {
+			errs = append(errs, err.Error())
+			continue
+		}
 		if err = btc.ds.DeleteBackupBackingImage(backupBackingImageName); err != nil && !apierrors.IsNotFound(err) {
 			errs = append(errs, err.Error())
 			continue
@@ -706,6 +723,10 @@
 
 	var errs []string
 	for systemBackup := range systemBackups {
+		if err = datastore.AddSystemBackupDeleteCustomResourceOnlyLabel(btc.ds, systemBackup); err != nil {
+			errs = append(errs, err.Error())
+			continue
+		}
 		if err = btc.ds.DeleteSystemBackup(systemBackup); err != nil && !apierrors.IsNotFound(err) {
 			errs = append(errs, err.Error())
 			continue

controller/backup_volume_controller.go

@@ -223,13 +223,16 @@ func (bvc *BackupVolumeController) reconcile(backupVolumeName string) (err error
 
 	// Examine DeletionTimestamp to determine if object is under deletion
 	if !backupVolume.DeletionTimestamp.IsZero() {
-
 		if err := bvc.ds.DeleteAllBackupsForBackupVolume(backupVolumeName); err != nil {
 			return errors.Wrap(err, "failed to delete backups")
 		}
 
+		needsCleanupRemoteData, err := checkIfRemoteDataCleanupIsNeeded(backupVolume, backupTarget)
+		if err != nil {
+			return errors.Wrap(err, "failed to check if it needs to delete remote backup volume data")
+		}
 		// Delete the backup volume from the remote backup target
-		if backupTarget.Spec.BackupTargetURL != "" {
+		if needsCleanupRemoteData {
 			engineClientProxy, backupTargetClient, err := getBackupTarget(bvc.controllerID, backupTarget, bvc.ds, log, bvc.proxyConnCounter)
 			if err != nil || engineClientProxy == nil {
 				log.WithError(err).Error("Failed to init backup target clients")
@@ -354,6 +357,9 @@ func (bvc *BackupVolumeController) reconcile(backupVolumeName string) (err error
 		log.Infof("Found %d backups in the backup target that do not exist in the cluster and need to be deleted from the cluster", count)
 	}
 	for backupName := range backupsToDelete {
+		if err = datastore.AddBackupDeleteCustomResourceOnlyLabel(bvc.ds, backupName); err != nil {
+			return errors.Wrapf(err, "failed to add label delete-custom-resource-only to backup %s", backupName)
+		}
 		if err = bvc.ds.DeleteBackup(backupName); err != nil {
 			return errors.Wrapf(err, "failed to delete backup %s from cluster", backupName)
 		}

controller/system_backup_controller.go

@@ -275,7 +275,7 @@
 		return err
 	}
 
-	return c.reconcile(name, backupTargetClient)
+	return c.reconcile(name, backupTargetClient, backupTarget)
 }
 
 func getLoggerForSystemBackup(logger logrus.FieldLogger, systemBackup *longhorn.SystemBackup) logrus.FieldLogger {
@@ -318,158 +318,158 @@
 	record.message = message
 }
 
-func (c *SystemBackupController) reconcile(name string, backupTargetClient engineapi.SystemBackupOperationInterface) (err error) {
+func (c *SystemBackupController) reconcile(name string, backupTargetClient engineapi.SystemBackupOperationInterface, backupTarget *longhorn.BackupTarget) (err error) {
 	systemBackup, err := c.ds.GetSystemBackup(name)
 	if err != nil {
 		if !apierrors.IsNotFound(err) {
 			return err
 		}
 		return nil
 	}
 
 	if !c.isResponsibleFor(systemBackup) {
 		return nil
 	}
 
 	log := getLoggerForSystemBackup(c.logger, systemBackup)
 
 	if systemBackup.Status.OwnerID != c.controllerID {
 		systemBackup.Status.OwnerID = c.controllerID
 		systemBackup, err = c.ds.UpdateSystemBackupStatus(systemBackup)
 		if err != nil {
 			// we don't mind others coming first
 			if apierrors.IsConflict(errors.Cause(err)) {
 				return nil
 			}
 			return err
 		}
 
 		log.Infof("System backup got new owner %v", c.controllerID)
 	}
 
 	record := &systemBackupRecord{}
 	existingSystemBackup := systemBackup.DeepCopy()
 	defer c.handleStatusUpdate(record, systemBackup, existingSystemBackup, err, log)
 
 	if !systemBackup.DeletionTimestamp.IsZero() &&
 		systemBackup.Status.State != longhorn.SystemBackupStateDeleting {
 		c.updateSystemBackupRecord(record,
 			systemBackupRecordTypeNormal, longhorn.SystemBackupStateDeleting,
 			constant.EventReasonDeleting, SystemBackupMsgDeletingRemote,
 		)
 		return
 	}
 
 	tempBackupArchivePath := filepath.Join(SystemBackupTempDir, systemBackup.Name+types.SystemBackupExtension)
 	tempBackupDir := filepath.Join(SystemBackupTempDir, systemBackup.Name)
 
 	switch systemBackup.Status.State {
 	case longhorn.SystemBackupStateSyncing:
 		err = syncSystemBackupFromBackupTarget(systemBackup, backupTargetClient)
 		if err != nil {
 			c.updateSystemBackupRecord(record,
 				systemBackupRecordTypeError, longhorn.SystemBackupStateError,
 				longhorn.SystemBackupConditionReasonSync, SystemBackupErrSync,
 			)
 			return err
 		}
 
 		c.updateSystemBackupRecord(record,
 			systemBackupRecordTypeNormal, longhorn.SystemBackupStateReady,
 			constant.EventReasonSynced, SystemBackupMsgSyncedBackupTarget,
 		)
 
 	case longhorn.SystemBackupStateNone:
 		// Sync the ready SystemBackups that are not in the current cluster.
 		if isSystemBackupFromRemoteBackupTarget(systemBackup) {
 			c.updateSystemBackupRecord(record,
 				systemBackupRecordTypeNormal, longhorn.SystemBackupStateSyncing,
 				constant.EventReasonSyncing, SystemBackupMsgSyncingBackupTarget,
 			)
 
 			return
 		}
 
 		var longhornVersion string
 		longhornVersion, err = getSystemBackupVersionExistInRemoteBackupTarget(systemBackup, backupTargetClient)
 		if err != nil {
 			c.updateSystemBackupRecord(record,
 				systemBackupRecordTypeError, longhorn.SystemBackupStateError,
 				constant.EventReasonStart, err.Error(),
 			)
 			return nil
 		}
 
 		if longhornVersion != "" {
 			if err := datastore.LabelSystemBackupVersion(longhornVersion, systemBackup); err != nil {
 				return err
 			}
 
 			_, err = c.ds.UpdateSystemBackup(systemBackup)
 			if err != nil {
 				return err
 			}
 			return
 		}
 
 		err = c.InitSystemBackup(systemBackup, log)
 		if err != nil {
 			return err
 		}
 
 		c.updateSystemBackupRecord(record,
 			systemBackupRecordTypeNormal, longhorn.SystemBackupStateVolumeBackup,
 			constant.EventReasonStart, SystemBackupMsgStarting,
 		)
 
 	case longhorn.SystemBackupStateVolumeBackup:
 		backups, err := c.BackupVolumes(systemBackup)
 		if err != nil {
 			c.updateSystemBackupRecord(record,
 				systemBackupRecordTypeError, longhorn.SystemBackupStateError,
 				constant.EventReasonStart, err.Error(),
 			)
 			return nil
 		}
 
 		// TODO: handle error check
 		go c.WaitForVolumeBackupToComplete(backups, systemBackup) // nolint: errcheck
 
 	case longhorn.SystemBackupStateBackingImageBackup:
 		backupBackingImages, err := c.BackupBackingImage()
 		if err != nil {
 			c.updateSystemBackupRecord(record,
 				systemBackupRecordTypeError, longhorn.SystemBackupStateError,
 				constant.EventReasonStart, err.Error(),
 			)
 			return nil
 		}
 
 		go c.WaitForBackingImageBackupToComplete(backupBackingImages, systemBackup)
 
 	case longhorn.SystemBackupStateGenerating:
 		go c.GenerateSystemBackup(systemBackup, tempBackupArchivePath, tempBackupDir)
 
 	case longhorn.SystemBackupStateUploading:
 		go c.UploadSystemBackup(systemBackup, tempBackupArchivePath, tempBackupDir, backupTargetClient)
 
 	case longhorn.SystemBackupStateReady, longhorn.SystemBackupStateError:
 		cleanupLocalSystemBackupFiles(tempBackupArchivePath, tempBackupDir, log)
 
 	case longhorn.SystemBackupStateDeleting:
-		cleanupRemoteSystemBackupFiles(systemBackup, backupTargetClient, log)
+		cleanupRemoteSystemBackupFiles(systemBackup, backupTargetClient, backupTarget, log)
 
 		cleanupLocalSystemBackupFiles(tempBackupArchivePath, tempBackupDir, log)
 
 		err = c.ds.RemoveFinalizerForSystemBackup(systemBackup)
 		if err != nil {
 			return err
 		}
 	}
 
 	return nil
 }
 
 func getSystemBackupVersionExistInRemoteBackupTarget(systemBackup *longhorn.SystemBackup, backupTargetClient engineapi.SystemBackupOperationInterface) (string, error) {
 	systemBackupsInBackupstore, err := backupTargetClient.ListSystemBackup()
 	if err != nil {
@@ -592,11 +592,17 @@
 	}
 }
 
-func cleanupRemoteSystemBackupFiles(systemBackup *longhorn.SystemBackup, backupTargetClient engineapi.SystemBackupOperationInterface, log logrus.FieldLogger) {
+func cleanupRemoteSystemBackupFiles(systemBackup *longhorn.SystemBackup, backupTargetClient engineapi.SystemBackupOperationInterface, backupTarget *longhorn.BackupTarget, log logrus.FieldLogger) {
 	if systemBackup.Status.Version == "" {
 		// The backup store sync might not have finished
 		return
 	}
+	if needsCleanupRemoteData, err := checkIfRemoteDataCleanupIsNeeded(systemBackup, backupTarget); err != nil {
+		log.WithError(err).Warn("failed to check if it needs to delete remote system backup data")
+		return
+	} else if !needsCleanupRemoteData {
+		return
+	}
 
 	systemBackupsFromBackupTarget, err := backupTargetClient.ListSystemBackup()
 	if err != nil {

controller/system_backup_controller_test.go

@@ -278,7 +278,7 @@ func (s *TestSuite) TestReconcileSystemBackup(c *C) {
 			systemBackupController.UploadSystemBackup(systemBackup, archievePath, tempDir, backupTargetClient)
 
 		default:
-			err = systemBackupController.reconcile(tc.systemBackupName, backupTargetClient)
+			err = systemBackupController.reconcile(tc.systemBackupName, backupTargetClient, nil)
 			if tc.expectError {
 				c.Assert(err, NotNil)
 			} else {

controller/utils.go

@@ -1,10 +1,15 @@
 package controller
 
 import (
-	"github.com/longhorn/longhorn-manager/types"
 	"github.com/sirupsen/logrus"
+
+	"k8s.io/apimachinery/pkg/runtime"
+
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 
+	"github.com/longhorn/longhorn-manager/datastore"
+	"github.com/longhorn/longhorn-manager/types"
+
 	longhorn "github.com/longhorn/longhorn-manager/k8s/pkg/apis/longhorn/v1beta2"
 )
 
@@ -90,3 +95,15 @@ func isRegularRWXVolume(v *longhorn.Volume) bool {
 	}
 	return v.Spec.AccessMode == longhorn.AccessModeReadWriteMany && !v.Spec.Migratable
 }
+
+func checkIfRemoteDataCleanupIsNeeded(obj runtime.Object, bt *longhorn.BackupTarget) (bool, error) {
+	if obj == nil || bt == nil {
+		return false, nil
+	}
+	exists, err := datastore.IsLabelLonghornDeleteCustomResourceOnlyExisting(obj)
+	if err != nil {
+		return false, err
+	}
+
+	return !exists && bt.Spec.BackupTargetURL != "", nil
+}

datastore/longhorn.go

@@ -3255,6 +3255,121 @@ func labelBackupVolume(backupVolumeName string, obj k8sruntime.Object) error {
 	return nil
 }
 
+// labelLonghornDeleteCustomResourceOnly labels the object with the label `longhorn.io/delete-custom-resource-only: true`
+func labelLonghornDeleteCustomResourceOnly(obj k8sruntime.Object) error {
+	metadata, err := meta.Accessor(obj)
+	if err != nil {
+		return err
+	}
+
+	labels := metadata.GetLabels()
+	if labels == nil {
+		labels = map[string]string{}
+	}
+	labels[types.GetLonghornLabelKey(types.DeleteCustomResourceOnly)] = "true"
+	metadata.SetLabels(labels)
+	return nil
+}
+
+// IsLabelLonghornDeleteCustomResourceOnlyExisting check if the object label `longhorn.io/delete-custom-resource-only` exists
+func IsLabelLonghornDeleteCustomResourceOnlyExisting(obj k8sruntime.Object) (bool, error) {
+	metadata, err := meta.Accessor(obj)
+	if err != nil {
+		return false, err
+	}
+
+	labels := metadata.GetLabels()
+	if labels == nil {
+		return false, nil
+	}
+	_, ok := labels[types.GetLonghornLabelKey(types.DeleteCustomResourceOnly)]
+	return ok, nil
+}
+
+// AddBackupVolumeDeleteCustomResourceOnlyLabel adds the label `longhorn.io/delete-custom-resource-only: true` to the BackupVolume
+func AddBackupVolumeDeleteCustomResourceOnlyLabel(ds *DataStore, backupVolumeName string) error {
+	bv, err := ds.GetBackupVolume(backupVolumeName)
+	if err != nil {
+		return err
+	}
+	if exists, err := IsLabelLonghornDeleteCustomResourceOnlyExisting(bv); err != nil {
+		return err
+	} else if exists {
+		return nil
+	}
+	if err := labelLonghornDeleteCustomResourceOnly(bv); err != nil {
+		return err
+	}
+	if _, err = ds.UpdateBackupVolume(bv); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// AddBackupDeleteCustomResourceOnlyLabel adds the label `longhorn.io/delete-custom-resource-only: true` to the Backup
+func AddBackupDeleteCustomResourceOnlyLabel(ds *DataStore, backupName string) error {
+	backup, err := ds.GetBackup(backupName)
+	if err != nil {
+		return err
+	}
+	if exists, err := IsLabelLonghornDeleteCustomResourceOnlyExisting(backup); err != nil {
+		return err
+	} else if exists {
+		return nil
+	}
+	if err := labelLonghornDeleteCustomResourceOnly(backup); err != nil {
+		return err
+	}
+	if _, err = ds.UpdateBackup(backup); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// AddBackupBackingImageDeleteCustomResourceOnlyLabel adds the label `longhorn.io/delete-custom-resource-only: true` to the BackupBackingImage
+func AddBackupBackingImageDeleteCustomResourceOnlyLabel(ds *DataStore, backupBackingImageName string) error {
+	bbi, err := ds.GetBackupBackingImage(backupBackingImageName)
+	if err != nil {
+		return err
+	}
+	if exists, err := IsLabelLonghornDeleteCustomResourceOnlyExisting(bbi); err != nil {
+		return err
+	} else if exists {
+		return nil
+	}
+	if err := labelLonghornDeleteCustomResourceOnly(bbi); err != nil {
+		return err
+	}
+	if _, err = ds.UpdateBackupBackingImage(bbi); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// AddSystemBackupDeleteCustomResourceOnlyLabel adds the label `longhorn.io/delete-custom-resource-only: true` to the SystemBackup
+func AddSystemBackupDeleteCustomResourceOnlyLabel(ds *DataStore, systemBackupName string) error {
+	sb, err := ds.GetSystemBackup(systemBackupName)
+	if err != nil {
+		return err
+	}
+	if exists, err := IsLabelLonghornDeleteCustomResourceOnlyExisting(sb); err != nil {
+		return err
+	} else if exists {
+		return nil
+	}
+	if err := labelLonghornDeleteCustomResourceOnly(sb); err != nil {
+		return err
+	}
+	if _, err = ds.UpdateSystemBackup(sb); err != nil {
+		return err
+	}
+
+	return nil
+}
+
 func FixupRecurringJob(v *longhorn.Volume) error {
 	if err := labelRecurringJobDefault(v); err != nil {
 		return err

types/types.go

@@ -128,6 +128,8 @@ const (
 	ConfigMapResourceVersionKey = "configmap-resource-version"
 	UpdateSettingFromLonghorn   = "update-setting-from-longhorn"
 
+	DeleteCustomResourceOnly = "delete-custom-resource-only"
+
 	KubernetesStatusLabel = "KubernetesStatus"
 	KubernetesReplicaSet  = "ReplicaSet"
 	KubernetesStatefulSet = "StatefulSet"

webhook/resources/backupvolume/mutator.go

@@ -45,9 +45,40 @@ func (b *backupVolumeMutator) Create(request *admission.Request, newObj runtime.
 }
 
 func (b *backupVolumeMutator) Update(request *admission.Request, oldObj runtime.Object, newObj runtime.Object) (admission.PatchOps, error) {
+	backupVolume, ok := newObj.(*longhorn.BackupVolume)
+	if !ok {
+		return nil, werror.NewInvalidError(fmt.Sprintf("%v is not a *longhorn.BackupVolume", newObj), "")
+	}
+
+	deleteCustomResourceOnlyLabelExists, err := datastore.IsLabelLonghornDeleteCustomResourceOnlyExisting(backupVolume)
+	if err != nil {
+		err := errors.Wrap(err, "failed to check if the label longhorn.io/delete-custom-resource-only exists")
+		return nil, werror.NewInvalidError(err.Error(), "")
+	}
+	if deleteCustomResourceOnlyLabelExists {
+		if err := b.addBackupsDeleteCustomResourceLabel(backupVolume); err != nil {
+			err := errors.Wrapf(err, "failed to add the label longhorn.io/delete-custom-resource-only to backups of the backup volume %v", backupVolume.Name)
+			return nil, werror.NewInvalidError(err.Error(), "")
+		}
+	}
+
 	return mutate(newObj)
 }
 
+func (b *backupVolumeMutator) addBackupsDeleteCustomResourceLabel(bv *longhorn.BackupVolume) error {
+	backups, err := b.ds.ListBackupsWithBackupVolumeName(bv.Name)
+	if err != nil {
+		return errors.Wrap(err, "failed to list backups of the backup volume")
+	}
+	for _, backup := range backups {
+		if err = datastore.AddBackupDeleteCustomResourceOnlyLabel(b.ds, backup.Name); err != nil {
+			return errors.Wrapf(err, "failed to add the label longhorn.io/delete-custom-resource-only to backup %s", backup.Name)
+		}
+	}
+
+	return nil
+}
+
 // mutate contains functionality shared by Create and Update.
 func mutate(newObj runtime.Object) (admission.PatchOps, error) {
 	backupVolume, ok := newObj.(*longhorn.BackupVolume)