[#323] Unity: add advanced dedup support

Add Advanced Deduplication support for Unity LUNs.

storops/exception.py

@@ -1425,3 +1425,18 @@ class UnityQoSMaxKBPSOutOfRangeError(UnityReplicationError):
 
 class UnitySnapScheduleNameInUseError(UnityException):
     pass
+
+
+@rest_exception
+class UnityAdvancedDedupNotSupportedError(UnityException):
+    error_code = 108009040
+
+
+@rest_exception
+class UnityAdvancedDedupRequireCompressionEnabledError(UnityException):
+    error_code = 108007752
+
+
+@rest_exception
+class UnityCompressionRequireAllFlashPoolError(UnityException):
+    error_code = 108009014

storops/unity/enums.py

@@ -1205,3 +1205,9 @@ class DayOfWeekEnumList(UnityEnumList):
     @classmethod
     def get_enum_class(cls):
         return DayOfWeekEnum
+
+
+class DataReductionStatusEnum(UnityEnum):
+    DISABLED = (0, 'Disabled')
+    ENABLED = (1, 'Enabled')
+    MIXED = (65535, 'Mixed')

storops/unity/parser_configs.yaml

@@ -42,6 +42,7 @@ UnityLun:
     - label: isThinEnabled
     - label: isCompressionEnabled
     - label: isDataReductionEnabled
+    - label: isAdvancedDedupEnabled
     - label: storageResource
       converter: UnityStorageResource
     - label: pool
@@ -73,6 +74,9 @@ UnityLun:
       converter: UnityLun
     - label: familyCloneCount
     - label: isThinClone
+    - label: dataReductionSizeSaved
+    - label: dataReductionPercent
+    - label: dataReductionRatio
 
 
 UnityHealth:
@@ -150,6 +154,13 @@ UnityStorageResource: &UnityStorageResource
       converter: UnityHostVvolDatastoreList
     - label: virtualVolumes
       converter: UnityVirtualVolumeList
+    - label: dataReductionStatus
+      converter: DataReductionStatusEnum
+    - label: advancedDedupStatus
+      converter: DedupStatusEnum
+    - label: dataReductionSizeSaved
+    - label: dataReductionPercent
+    - label: dataReductionRatio
 
 
 UnityConsistencyGroup:

storops/unity/resource/lun.py

@@ -76,7 +76,8 @@ def make_compression_body(cli=None,
         fastVPParameters=UnityClient.make_body(
             tieringPolicy=kwargs.get('tiering_policy')),
         ioLimitParameters=UnityClient.make_body(
-            ioLimitPolicy=kwargs.get('io_limit_policy')))
+            ioLimitPolicy=kwargs.get('io_limit_policy')),
+        isAdvancedDedupEnabled=kwargs.get('is_advanced_dedup_enabled'))
 
     compression_body = make_compression_body(
         cli,
@@ -114,7 +115,7 @@ def create(cls, cli, name, pool, size, sp=None, host_access=None,
                is_repl_dst=None, tiering_policy=None, snap_schedule=None,
                is_snap_schedule_paused=None, skip_sync_to_remote_system=None,
                is_compression=None, create_vmfs=False, major_version=None,
-               block_size=None):
+               block_size=None, is_advanced_dedup_enabled=None):
         pool_clz = storops.unity.resource.pool.UnityPool
         pool = pool_clz.get(cli, pool)
 
@@ -126,7 +127,8 @@ def create(cls, cli, name, pool, size, sp=None, host_access=None,
             is_snap_schedule_paused=is_snap_schedule_paused,
             skip_sync_to_remote_system=skip_sync_to_remote_system,
             is_compression=is_compression, major_version=major_version,
-            block_size=block_size)
+            block_size=block_size,
+            is_advanced_dedup_enabled=is_advanced_dedup_enabled)
 
         create_method = 'createVmwareLun' if create_vmfs else 'createLun'
 
@@ -250,7 +252,8 @@ def modify(self, name=None, size=None, host_access=None,
                description=None, sp=None, io_limit_policy=None,
                is_repl_dst=None, tiering_policy=None, snap_schedule=None,
                is_snap_schedule_paused=None, skip_sync_to_remote_system=None,
-               is_compression=None, major_version=None, block_size=None):
+               is_compression=None, major_version=None, block_size=None,
+               is_advanced_dedup_enabled=None):
         if self.is_cg_member:
             if any(each is not None for each in [is_repl_dst, snap_schedule,
                                                  is_snap_schedule_paused,
@@ -274,7 +277,8 @@ def modify(self, name=None, size=None, host_access=None,
                 is_snap_schedule_paused=is_snap_schedule_paused,
                 skip_sync_to_remote_system=skip_sync_to_remote_system,
                 is_compression=is_compression, major_version=major_version,
-                block_size=block_size)
+                block_size=block_size,
+                is_advanced_dedup_enabled=is_advanced_dedup_enabled)
 
             if self.is_vmware_vmfs:
                 resp = self._cli.action(UnityStorageResource().resource_class,
@@ -411,9 +415,16 @@ def _is_move_session_supported(self, dest):
         if self.is_thin_clone:
             log.error('Not support move session, source lun is thin clone.')
             return False
-        if self.is_data_reduction_enabled and not dest.is_all_flash:
+        if (self.is_data_reduction_enabled and
+                not dest.is_compression_supported):
             log.error('Not support move session, source lun is compressed, '
-                      'but destination pool is not all flash pool.')
+                      'but destination pool is not supported compression.')
+            return False
+        if (self.is_advanced_dedup_enabled and
+                not dest.is_advanced_dedup_supported):
+            log.error('Not support move session, source lun is advanced '
+                      'deduplication enabled, but destination pool is not '
+                      'supported advanced deduplication.')
             return False
         return True
 
@@ -423,8 +434,11 @@ def migrate(self, dest, **kwargs):
 
         interval = kwargs.pop('interval', 5)
         timeout = kwargs.pop('timeout', 1800)
-        is_thin = kwargs.get('is_thin')
-        is_compressed = kwargs.get('is_compressed')
+        is_thin = kwargs.get('is_thin', self.is_thin_enabled)
+        is_compressed = kwargs.get('is_compressed',
+                                   self.is_data_reduction_enabled)
+        is_advanced_dedup_enabled = kwargs.get('is_advanced_dedup_enabled',
+                                               self.is_advanced_dedup_enabled)
 
         @retryz.retry(timeout=timeout, wait=interval,
                       on_return=lambda x: not isinstance(x, bool))
@@ -437,12 +451,12 @@ def _do_check_move_session(move_session_id):
                 return False
 
         clz = storops.unity.resource.move_session.UnityMoveSession
-        if is_compressed not in (True, False):
-            is_compressed = self.is_data_reduction_enabled
         try:
-            move_session = clz.create(self._cli, self, dest,
-                                      is_data_reduction_applied=is_compressed,
-                                      is_dest_thin=is_thin)
+            move_session = clz.create(
+                self._cli, self, dest,
+                is_data_reduction_applied=is_compressed,
+                is_dest_thin=is_thin,
+                is_advanced_dedup_enabled=is_advanced_dedup_enabled)
             return _do_check_move_session(move_session.id)
         except UnityMigrationSourceHasThinCloneError:
             log.error('Not support move session, source lun has thin clone.')

storops/unity/resource/move_session.py

@@ -28,13 +28,15 @@ class UnityMoveSession(UnityResource):
     @classmethod
     def create(cls, cli, source_storage_resource, destination_pool,
                source_member_lun=None, is_dest_thin=None,
-               is_data_reduction_applied=None, priority=None):
+               is_data_reduction_applied=None, is_advanced_dedup_applied=None,
+               priority=None):
         req_body = cls._compose_move_session_parameter(
             cli, source_storage_resource=source_storage_resource,
             destination_pool=destination_pool,
             source_member_lun=source_member_lun,
             is_dest_thin=is_dest_thin,
             is_data_reduction_applied=is_data_reduction_applied,
+            is_advanced_dedup_applied=is_advanced_dedup_applied,
             priority=priority)
         resp = cli.post(cls().resource_class, **req_body)
         resp.raise_if_err()
@@ -59,13 +61,15 @@ def _compose_move_session_parameter(cli, source_storage_resource=None,
                                         source_member_lun=None,
                                         is_dest_thin=None,
                                         is_data_reduction_applied=None,
+                                        is_advanced_dedup_enabled=None,
                                         priority=None):
         req_body = cli.make_body(
             sourceStorageResource=source_storage_resource,
             destinationPool=destination_pool,
             sourceMemberLun=source_member_lun,
             isDestThin=is_dest_thin,
             isDataReductionApplied=is_data_reduction_applied,
+            isAdvancedDedupApplied=is_advanced_dedup_enabled,
             priority=priority)
         return req_body
 

storops/unity/resource/pool.py

@@ -21,6 +21,7 @@
 import bitmath
 
 import storops.unity.resource.filesystem
+from storops.lib.version import version
 from storops.unity.resource import UnityResource, \
     UnityAttributeResource, UnityResourceList
 from storops.unity.resource.disk import UnityDiskGroup, UnityDiskList
@@ -104,7 +105,8 @@ def create_lun(self, lun_name=None, size_gb=1, sp=None, host_access=None,
                    is_repl_dst=None, snap_schedule=None,
                    is_snap_schedule_paused=None,
                    skip_sync_to_remote_system=None,
-                   io_limit_policy=None, is_compression=None):
+                   io_limit_policy=None, is_compression=None,
+                   is_advanced_dedup_enabled=None):
         size = int(bitmath.GiB(size_gb).to_Byte().value)
         return UnityLun.create(
             self._cli, lun_name, self, size, sp=sp,
@@ -116,7 +118,8 @@ def create_lun(self, lun_name=None, size_gb=1, sp=None, host_access=None,
             is_snap_schedule_paused=is_snap_schedule_paused,
             skip_sync_to_remote_system=skip_sync_to_remote_system,
             io_limit_policy=io_limit_policy,
-            is_compression=is_compression)
+            is_compression=is_compression,
+            is_advanced_dedup_enabled=is_advanced_dedup_enabled)
 
     def create_vmfs(self, vmfs_name=None, size_gb=1, sp=None, host_access=None,
                     is_thin=None, description=None, tiering_policy=None,
@@ -203,6 +206,51 @@ def disk_groups(self):
                 dgs[pd.disk_group.get_id()] = [pd]
         return dgs
 
+    @version('<5.1')
+    def is_compression_supported(self):
+        return self.is_all_flash
+
+    @version('>=5.1')  # noqa
+    def is_compression_supported(self):
+        resp = self._cli.action(self.resource_class, self.get_id(),
+                                'verifyDataReduction')
+        resp.raise_if_err()
+        if resp.body.get('content') and resp.body.get('content').get(
+                'verifyPass') is True:
+            return True
+        return False
+
+    def verify_advanced_dedup_by_unity_model(self, unity_support_matrix):
+        from storops.unity.resource.system import UnitySystem
+        unity_system = UnitySystem(cli=self._cli)
+        unity_model = unity_system.model.split()[-1]
+        return unity_model in unity_support_matrix and self.is_all_flash
+
+    @version('<4.5')
+    def is_advanced_dedup_supported(self):
+        return False
+
+    @version('=4.5')  # noqa
+    def is_advanced_dedup_supported(self):
+        unity_support_matrix = ['450F', '550F', '650F']
+        return self.verify_advanced_dedup_by_unity_model(unity_support_matrix)
+
+    @version('=5.0')  # noqa
+    def is_advanced_dedup_supported(self):
+        unity_support_matrix = ['450F', '550F', '650F', '380', '480', '680',
+                                '880', '380F', '480F', '680F', '880F']
+        return self.verify_advanced_dedup_by_unity_model(unity_support_matrix)
+
+    @version('>=5.1')  # noqa
+    def is_advanced_dedup_supported(self):
+        resp = self._cli.action(self.resource_class, self.get_id(),
+                                'verifyDeduplication')
+        resp.raise_if_err()
+        if resp.body.get('content') and resp.body.get('content').get(
+                'verifyPass') is True:
+            return True
+        return False
+
 
 class UnityPoolList(UnityResourceList):
     @classmethod

storops_test/unity/resource/test_lun.py

@@ -23,7 +23,6 @@
 from hamcrest import assert_that, calling, only_contains, instance_of, \
     contains_string, raises, none, has_item, is_not
 from hamcrest import equal_to
-from storops.unity.resource.snap_schedule import UnitySnapSchedule
 
 from storops import UnitySystem, TieringPolicyEnum
 from storops.exception import UnitySnapNameInUseError, \
@@ -32,7 +31,8 @@
     UnityThinCloneLimitExceededError, UnityCGMemberActionNotSupportError, \
     UnityThinCloneNotAllowedError, UnityMigrationTimeoutException, \
     UnityMigrationSourceDestNotExistsError, JobStateError, \
-    JobTimeoutException
+    JobTimeoutException, UnityAdvancedDedupRequireCompressionEnabledError, \
+    UnityCompressionRequireAllFlashPoolError
 from storops.unity.enums import HostLUNAccessEnum, NodeEnum, RaidTypeEnum, \
     ESXFilesystemBlockSizeEnum, ESXFilesystemMajorVersionEnum
 from storops.unity.resource.disk import UnityDisk
@@ -43,6 +43,7 @@
     UnityIoLimitRuleSetting
 from storops.unity.resource.remote_system import UnityRemoteSystem
 from storops.unity.resource.snap import UnitySnap
+from storops.unity.resource.snap_schedule import UnitySnapSchedule
 from storops.unity.resource.sp import UnityStorageProcessor
 from storops.unity.resource.storage_resource import UnityStorageResource
 from storops.unity.resp import RestResponse
@@ -83,6 +84,10 @@ def test_get_lun_sv2_simple_property(self):
         assert_that(lun.io_limit_rule, none())
         assert_that(lun.is_compression_enabled, equal_to(False))
         assert_that(lun.is_data_reduction_enabled, equal_to(False))
+        assert_that(lun.is_advanced_dedup_enabled, equal_to(False))
+        assert_that(lun.data_reduction_size_saved, equal_to(0))
+        assert_that(lun.data_reduction_percent, equal_to(0))
+        assert_that(lun.data_reduction_ratio, equal_to(1.0))
 
     @patch_rest
     def test_lun_modify_host_access(self):
@@ -152,6 +157,13 @@ def test_lun_modify_compression_enabled(self):
         lun.update()
         assert_that(lun.is_data_reduction_enabled, equal_to(True))
 
+    @patch_rest
+    def test_lun_modify_dedup_enabled(self):
+        lun = UnityLun(_id='sv_19', cli=t_rest(version='5.0.0'))
+        lun.modify(is_advanced_dedup_enabled=True)
+        lun.update()
+        assert_that(lun.is_advanced_dedup_enabled, equal_to(True))
+
     @mock.patch(target='storops.lib.job_helper.JobHelper',
                 new=MockJobHelper)
     @patch_rest
@@ -341,6 +353,42 @@ def test_create_with_io_limit(self):
         assert_that(rule.max_kbps_density, equal_to(1100))
         assert_that(rule.name, equal_to('Density_1100_KBPS_rule'))
 
+    @patch_rest
+    def test_create_with_dedup_enabled_compression_enabled(self):
+        cli = t_rest(version='5.0.0')
+        pool = UnityPool('pool_1', cli=cli)
+        lun = pool.create_lun('lun_1',
+                              is_compression=True,
+                              is_advanced_dedup_enabled=True)
+        assert_that(lun.name, equal_to('lun_1'))
+        assert_that(lun.is_data_reduction_enabled, equal_to(True))
+        assert_that(lun.is_advanced_dedup_enabled, equal_to(True))
+
+    @patch_rest
+    def test_create_with_dedup_enabled_compression_disabled(self):
+        def f():
+            cli = t_rest(version='5.0.0')
+            pool = UnityPool('pool_1', cli=cli)
+            pool.create_lun('lun_2',
+                            is_compression=False,
+                            is_advanced_dedup_enabled=True)
+
+        assert_that(f,
+                    raises(UnityAdvancedDedupRequireCompressionEnabledError))
+
+    @patch_rest
+    def test_create_with_dedup_enabled_in_non_all_flash_pool(self):
+        def f():
+            cli = t_rest(version='5.0.0')
+            pool = UnityPool('pool_1', cli=cli)
+            pool.is_all_flash = False
+            pool.create_lun('lun_3',
+                            is_compression=True,
+                            is_advanced_dedup_enabled=True)
+
+        assert_that(f,
+                    raises(UnityCompressionRequireAllFlashPoolError))
+
     @patch_rest
     def test_expand_lun_success(self):
         lun = UnityLun('sv_2', cli=t_rest())