[#323] Unity: add advanced dedup support

Add Advanced Deduplication support for Unity LUNs.

storops/exception.py

@@ -1425,3 +1425,23 @@ class UnityQoSMaxKBPSOutOfRangeError(UnityReplicationError):
 
 class UnitySnapScheduleNameInUseError(UnityException):
     pass
+
+
+@rest_exception
+class UnityAdvancedDedupNotSupportedError(UnityException):
+    error_code = 108009040
+
+
+@rest_exception
+class UnityAdvancedDedupRequireCompressionEnabledError(UnityException):
+    error_code = 108007752
+
+
+@rest_exception
+class UnityCompressionRequireLunIsThinError(UnityException):
+    error_code = 108007750
+
+
+@rest_exception
+class UnityCompressionRequireAllFlashPoolError(UnityException):
+    error_code = 108009014

storops/unity/enums.py

@@ -1205,3 +1205,9 @@ class DayOfWeekEnumList(UnityEnumList):
     @classmethod
     def get_enum_class(cls):
         return DayOfWeekEnum
+
+
+class DataReductionStatusEnum(UnityEnum):
+    DISABLED = (0, 'Disabled')
+    ENABLED = (1, 'Enabled')
+    MIXED = (65535, 'Mixed')

storops/unity/parser_configs.yaml

@@ -42,6 +42,7 @@ UnityLun:
     - label: isThinEnabled
     - label: isCompressionEnabled
     - label: isDataReductionEnabled
+    - label: isAdvancedDedupEnabled
     - label: storageResource
       converter: UnityStorageResource
     - label: pool
@@ -73,6 +74,9 @@ UnityLun:
       converter: UnityLun
     - label: familyCloneCount
     - label: isThinClone
+    - label: dataReductionSizeSaved
+    - label: dataReductionPercent
+    - label: dataReductionRatio
 
 
 UnityHealth:
@@ -150,6 +154,13 @@ UnityStorageResource: &UnityStorageResource
       converter: UnityHostVvolDatastoreList
     - label: virtualVolumes
       converter: UnityVirtualVolumeList
+    - label: dataReductionStatus
+      converter: DataReductionStatusEnum
+    - label: advancedDedupStatus
+      converter: DedupStatusEnum
+    - label: dataReductionSizeSaved
+    - label: dataReductionPercent
+    - label: dataReductionRatio
 
 
 UnityConsistencyGroup:

storops/unity/resource/lun.py

@@ -76,7 +76,8 @@ def make_compression_body(cli=None,
         fastVPParameters=UnityClient.make_body(
             tieringPolicy=kwargs.get('tiering_policy')),
         ioLimitParameters=UnityClient.make_body(
-            ioLimitPolicy=kwargs.get('io_limit_policy')))
+            ioLimitPolicy=kwargs.get('io_limit_policy')),
+        isAdvancedDedupEnabled=kwargs.get('is_advanced_dedup_enabled'))
 
     compression_body = make_compression_body(
         cli,
@@ -114,7 +115,7 @@ def create(cls, cli, name, pool, size, sp=None, host_access=None,
                is_repl_dst=None, tiering_policy=None, snap_schedule=None,
                is_snap_schedule_paused=None, skip_sync_to_remote_system=None,
                is_compression=None, create_vmfs=False, major_version=None,
-               block_size=None):
+               block_size=None, is_advanced_dedup_enabled=None):
         pool_clz = storops.unity.resource.pool.UnityPool
         pool = pool_clz.get(cli, pool)
 
@@ -126,7 +127,8 @@ def create(cls, cli, name, pool, size, sp=None, host_access=None,
             is_snap_schedule_paused=is_snap_schedule_paused,
             skip_sync_to_remote_system=skip_sync_to_remote_system,
             is_compression=is_compression, major_version=major_version,
-            block_size=block_size)
+            block_size=block_size,
+            is_advanced_dedup_enabled=is_advanced_dedup_enabled)
 
         create_method = 'createVmwareLun' if create_vmfs else 'createLun'
 
@@ -250,7 +252,8 @@ def modify(self, name=None, size=None, host_access=None,
                description=None, sp=None, io_limit_policy=None,
                is_repl_dst=None, tiering_policy=None, snap_schedule=None,
                is_snap_schedule_paused=None, skip_sync_to_remote_system=None,
-               is_compression=None, major_version=None, block_size=None):
+               is_compression=None, major_version=None, block_size=None,
+               is_advanced_dedup_enabled=None):
         if self.is_cg_member:
             if any(each is not None for each in [is_repl_dst, snap_schedule,
                                                  is_snap_schedule_paused,
@@ -274,7 +277,8 @@ def modify(self, name=None, size=None, host_access=None,
                 is_snap_schedule_paused=is_snap_schedule_paused,
                 skip_sync_to_remote_system=skip_sync_to_remote_system,
                 is_compression=is_compression, major_version=major_version,
-                block_size=block_size)
+                block_size=block_size,
+                is_advanced_dedup_enabled=is_advanced_dedup_enabled)
 
             if self.is_vmware_vmfs:
                 resp = self._cli.action(UnityStorageResource().resource_class,
@@ -407,24 +411,43 @@ def thin_clone(self, name, io_limit_policy=None, description=None):
         return TCHelper.thin_clone(self._cli, self, name, io_limit_policy,
                                    description)
 
-    def _is_move_session_supported(self, dest):
+    def _is_move_session_supported(self, dest, is_compressed=None,
+                                   is_advanced_dedup_enabled=None):
         if self.is_thin_clone:
             log.error('Not support move session, source lun is thin clone.')
             return False
-        if self.is_data_reduction_enabled and not dest.is_all_flash:
-            log.error('Not support move session, source lun is compressed, '
-                      'but destination pool is not all flash pool.')
+        if is_compressed and not dest.is_compression_supported():
+            log.error('Not support move session, target lun is compressed, '
+                      'but destination pool is not supported compression.')
+            return False
+        if (is_advanced_dedup_enabled and
+                not dest.is_advanced_dedup_supported()):
+            log.error('Not support move session, target lun is advanced '
+                      'deduplication enabled, but destination pool is not '
+                      'supported advanced deduplication.')
             return False
         return True
 
     def migrate(self, dest, **kwargs):
-        if not self._is_move_session_supported(dest):
-            return False
-
         interval = kwargs.pop('interval', 5)
         timeout = kwargs.pop('timeout', 1800)
-        is_thin = kwargs.get('is_thin')
-        is_compressed = kwargs.get('is_compressed')
+        is_thin = kwargs.get('is_thin', self.is_thin_enabled)
+
+        if is_thin:
+            is_compressed = kwargs.get('is_compressed',
+                                       self.is_data_reduction_enabled)
+        else:
+            is_compressed = False
+
+        if is_thin and is_compressed:
+            is_advanced_dedup_enabled = kwargs.get(
+                'is_advanced_dedup_enabled', self.is_advanced_dedup_enabled)
+        else:
+            is_advanced_dedup_enabled = False
+
+        if not self._is_move_session_supported(dest, is_compressed,
+                                               is_advanced_dedup_enabled):
+            return False
 
         @retryz.retry(timeout=timeout, wait=interval,
                       on_return=lambda x: not isinstance(x, bool))
@@ -437,12 +460,12 @@ def _do_check_move_session(move_session_id):
                 return False
 
         clz = storops.unity.resource.move_session.UnityMoveSession
-        if is_compressed not in (True, False):
-            is_compressed = self.is_data_reduction_enabled
         try:
-            move_session = clz.create(self._cli, self, dest,
-                                      is_data_reduction_applied=is_compressed,
-                                      is_dest_thin=is_thin)
+            move_session = clz.create(
+                self._cli, self, dest,
+                is_data_reduction_applied=is_compressed,
+                is_dest_thin=is_thin,
+                is_advanced_dedup_applied=is_advanced_dedup_enabled)
             return _do_check_move_session(move_session.id)
         except UnityMigrationSourceHasThinCloneError:
             log.error('Not support move session, source lun has thin clone.')

storops/unity/resource/move_session.py

@@ -28,13 +28,15 @@ class UnityMoveSession(UnityResource):
     @classmethod
     def create(cls, cli, source_storage_resource, destination_pool,
                source_member_lun=None, is_dest_thin=None,
-               is_data_reduction_applied=None, priority=None):
+               is_data_reduction_applied=None, is_advanced_dedup_applied=None,
+               priority=None):
         req_body = cls._compose_move_session_parameter(
             cli, source_storage_resource=source_storage_resource,
             destination_pool=destination_pool,
             source_member_lun=source_member_lun,
             is_dest_thin=is_dest_thin,
             is_data_reduction_applied=is_data_reduction_applied,
+            is_advanced_dedup_applied=is_advanced_dedup_applied,
             priority=priority)
         resp = cli.post(cls().resource_class, **req_body)
         resp.raise_if_err()
@@ -59,13 +61,15 @@ def _compose_move_session_parameter(cli, source_storage_resource=None,
                                         source_member_lun=None,
                                         is_dest_thin=None,
                                         is_data_reduction_applied=None,
+                                        is_advanced_dedup_applied=None,
                                         priority=None):
         req_body = cli.make_body(
             sourceStorageResource=source_storage_resource,
             destinationPool=destination_pool,
             sourceMemberLun=source_member_lun,
             isDestThin=is_dest_thin,
             isDataReductionApplied=is_data_reduction_applied,
+            isAdvancedDedupApplied=is_advanced_dedup_applied,
             priority=priority)
         return req_body
 

storops/unity/resource/pool.py

@@ -21,6 +21,7 @@
 import bitmath
 
 import storops.unity.resource.filesystem
+from storops.lib.version import version
 from storops.unity.resource import UnityResource, \
     UnityAttributeResource, UnityResourceList
 from storops.unity.resource.disk import UnityDiskGroup, UnityDiskList
@@ -104,7 +105,8 @@ def create_lun(self, lun_name=None, size_gb=1, sp=None, host_access=None,
                    is_repl_dst=None, snap_schedule=None,
                    is_snap_schedule_paused=None,
                    skip_sync_to_remote_system=None,
-                   io_limit_policy=None, is_compression=None):
+                   io_limit_policy=None, is_compression=None,
+                   is_advanced_dedup_enabled=None):
         size = int(bitmath.GiB(size_gb).to_Byte().value)
         return UnityLun.create(
             self._cli, lun_name, self, size, sp=sp,
@@ -116,7 +118,8 @@ def create_lun(self, lun_name=None, size_gb=1, sp=None, host_access=None,
             is_snap_schedule_paused=is_snap_schedule_paused,
             skip_sync_to_remote_system=skip_sync_to_remote_system,
             io_limit_policy=io_limit_policy,
-            is_compression=is_compression)
+            is_compression=is_compression,
+            is_advanced_dedup_enabled=is_advanced_dedup_enabled)
 
     def create_vmfs(self, vmfs_name=None, size_gb=1, sp=None, host_access=None,
                     is_thin=None, description=None, tiering_policy=None,
@@ -203,6 +206,47 @@ def disk_groups(self):
                 dgs[pd.disk_group.get_id()] = [pd]
         return dgs
 
+    @version('<5.1')
+    def is_compression_supported(self):
+        return self.is_all_flash
+
+    @version('>=5.1')  # noqa
+    def is_compression_supported(self):
+        resp = self._cli.action(self.resource_class, self.get_id(),
+                                'verifyDataReduction')
+        resp.raise_if_err()
+        return resp.body.get('content') and resp.body.get('content').get(
+            'verifyPass')
+
+    def verify_advanced_dedup_by_unity_model(self, unity_support_matrix):
+        from storops.unity.resource.system import UnitySystem
+        unity_system = UnitySystem(cli=self._cli)
+        unity_model = unity_system.model.split()[-1]
+        return unity_model in unity_support_matrix and self.is_all_flash
+
+    @version('<4.5')
+    def is_advanced_dedup_supported(self):
+        return False
+
+    @version('=4.5')  # noqa
+    def is_advanced_dedup_supported(self):
+        unity_support_matrix = ['450F', '550F', '650F']
+        return self.verify_advanced_dedup_by_unity_model(unity_support_matrix)
+
+    @version('=5.0')  # noqa
+    def is_advanced_dedup_supported(self):
+        unity_support_matrix = ['450F', '550F', '650F', '380', '480', '680',
+                                '880', '380F', '480F', '680F', '880F']
+        return self.verify_advanced_dedup_by_unity_model(unity_support_matrix)
+
+    @version('>=5.1')  # noqa
+    def is_advanced_dedup_supported(self):
+        resp = self._cli.action(self.resource_class, self.get_id(),
+                                'verifyDeduplication')
+        resp.raise_if_err()
+        return resp.body.get('content') and resp.body.get('content').get(
+            'verifyPass')
+
 
 class UnityPoolList(UnityResourceList):
     @classmethod