[#323] Unity: add advanced dedup support

Add Advanced Deduplication support for Unity LUNs, and then Unity OpenStack Cinder driver would leverage it to create volumes with deduplication enabled.
emc-openstack · Sep 16, 2020 · 254879d · 254879d
1 parent 8f8ef8f
commit 254879d
Show file tree

Hide file tree

Showing 18 changed files with 450 additions and 14 deletions.
diff --git a/storops/exception.py b/storops/exception.py
@@ -1425,3 +1425,15 @@ class UnityQoSMaxKBPSOutOfRangeError(UnityReplicationError):
 
 class UnitySnapScheduleNameInUseError(UnityException):
     pass
+
+@rest_exception
+class UnityAdvancedDedupNotSupportedError(UnityException):
+    error_code = 108009040
+
+@rest_exception
+class UnityAdvancedDedupRequireCompressionEnabledError(UnityException):
+    error_code = 108007752
+
+@rest_exception
+class UnityCompressionRequireAllFlashPoolError(UnityException):
+    error_code = 108009014
diff --git a/storops/unity/enums.py b/storops/unity/enums.py
@@ -1205,3 +1205,15 @@ class DayOfWeekEnumList(UnityEnumList):
     @classmethod
     def get_enum_class(cls):
         return DayOfWeekEnum
+
+
+class DataReductionStatusEnum(UnityEnum):
+    DISABLED = (0, 'Disabled')
+    ENABLED = (1, 'Enabled')
+    MIXED = (65535, 'Mixed')
+
+
+class DedupStatusEnum(UnityEnum):
+    DISABLED = (0, 'Disabled')
+    ENABLED = (1, 'Enabled')
+    MIXED = (65535, 'Mixed')
diff --git a/storops/unity/parser_configs.yaml b/storops/unity/parser_configs.yaml
@@ -42,6 +42,7 @@ UnityLun:
     - label: isThinEnabled
     - label: isCompressionEnabled
     - label: isDataReductionEnabled
+    - label: isAdvancedDedupEnabled
     - label: storageResource
       converter: UnityStorageResource
     - label: pool
@@ -73,6 +74,9 @@ UnityLun:
       converter: UnityLun
     - label: familyCloneCount
     - label: isThinClone
+    - label: dataReductionSizeSaved
+    - label: dataReductionPercent
+    - label: dataReductionRatio
 
 
 UnityHealth:
@@ -150,6 +154,13 @@ UnityStorageResource: &UnityStorageResource
       converter: UnityHostVvolDatastoreList
     - label: virtualVolumes
       converter: UnityVirtualVolumeList
+    - label: dataReductionStatus
+      converter: DataReductionStatusEnum
+    - label: advancedDedupStatus
+      converter: DedupStatusEnum
+    - label: dataReductionSizeSaved
+    - label: dataReductionPercent
+    - label: dataReductionRatio
 
 
 UnityConsistencyGroup:

diff --git a/storops/unity/resource/lun.py b/storops/unity/resource/lun.py
@@ -76,7 +76,8 @@ def make_compression_body(cli=None,
         fastVPParameters=UnityClient.make_body(
             tieringPolicy=kwargs.get('tiering_policy')),
         ioLimitParameters=UnityClient.make_body(
-            ioLimitPolicy=kwargs.get('io_limit_policy')))
+            ioLimitPolicy=kwargs.get('io_limit_policy')),
+        isAdvancedDedupEnabled=kwargs.get('is_advanced_dedup_enabled'))
 
     compression_body = make_compression_body(
         cli,
@@ -114,7 +115,7 @@ def create(cls, cli, name, pool, size, sp=None, host_access=None,
                is_repl_dst=None, tiering_policy=None, snap_schedule=None,
                is_snap_schedule_paused=None, skip_sync_to_remote_system=None,
                is_compression=None, create_vmfs=False, major_version=None,
-               block_size=None):
+               block_size=None, is_advanced_dedup_enabled=None):
         pool_clz = storops.unity.resource.pool.UnityPool
         pool = pool_clz.get(cli, pool)
 
@@ -126,7 +127,8 @@ def create(cls, cli, name, pool, size, sp=None, host_access=None,
             is_snap_schedule_paused=is_snap_schedule_paused,
             skip_sync_to_remote_system=skip_sync_to_remote_system,
             is_compression=is_compression, major_version=major_version,
-            block_size=block_size)
+            block_size=block_size,
+            is_advanced_dedup_enabled=is_advanced_dedup_enabled)
 
         create_method = 'createVmwareLun' if create_vmfs else 'createLun'
 
@@ -250,7 +252,8 @@ def modify(self, name=None, size=None, host_access=None,
                description=None, sp=None, io_limit_policy=None,
                is_repl_dst=None, tiering_policy=None, snap_schedule=None,
                is_snap_schedule_paused=None, skip_sync_to_remote_system=None,
-               is_compression=None, major_version=None, block_size=None):
+               is_compression=None, major_version=None, block_size=None,
+               is_advanced_dedup_enabled=None):
         if self.is_cg_member:
             if any(each is not None for each in [is_repl_dst, snap_schedule,
                                                  is_snap_schedule_paused,
@@ -274,7 +277,8 @@ def modify(self, name=None, size=None, host_access=None,
                 is_snap_schedule_paused=is_snap_schedule_paused,
                 skip_sync_to_remote_system=skip_sync_to_remote_system,
                 is_compression=is_compression, major_version=major_version,
-                block_size=block_size)
+                block_size=block_size,
+                is_advanced_dedup_enabled=is_advanced_dedup_enabled)
 
             if self.is_vmware_vmfs:
                 resp = self._cli.action(UnityStorageResource().resource_class,

diff --git a/storops/unity/resource/pool.py b/storops/unity/resource/pool.py
@@ -104,7 +104,8 @@ def create_lun(self, lun_name=None, size_gb=1, sp=None, host_access=None,
                    is_repl_dst=None, snap_schedule=None,
                    is_snap_schedule_paused=None,
                    skip_sync_to_remote_system=None,
-                   io_limit_policy=None, is_compression=None):
+                   io_limit_policy=None, is_compression=None,
+                   is_advanced_dedup_enabled=None):
         size = int(bitmath.GiB(size_gb).to_Byte().value)
         return UnityLun.create(
             self._cli, lun_name, self, size, sp=sp,
@@ -116,7 +117,8 @@ def create_lun(self, lun_name=None, size_gb=1, sp=None, host_access=None,
             is_snap_schedule_paused=is_snap_schedule_paused,
             skip_sync_to_remote_system=skip_sync_to_remote_system,
             io_limit_policy=io_limit_policy,
-            is_compression=is_compression)
+            is_compression=is_compression,
+            is_advanced_dedup_enabled=is_advanced_dedup_enabled)
 
     def create_vmfs(self, vmfs_name=None, size_gb=1, sp=None, host_access=None,
                     is_thin=None, description=None, tiering_policy=None,

diff --git a/storops_test/unity/resource/test_lun.py b/storops_test/unity/resource/test_lun.py
@@ -23,7 +23,6 @@
 from hamcrest import assert_that, calling, only_contains, instance_of, \
     contains_string, raises, none, has_item, is_not
 from hamcrest import equal_to
-from storops.unity.resource.snap_schedule import UnitySnapSchedule
 
 from storops import UnitySystem, TieringPolicyEnum
 from storops.exception import UnitySnapNameInUseError, \
@@ -32,7 +31,8 @@
     UnityThinCloneLimitExceededError, UnityCGMemberActionNotSupportError, \
     UnityThinCloneNotAllowedError, UnityMigrationTimeoutException, \
     UnityMigrationSourceDestNotExistsError, JobStateError, \
-    JobTimeoutException
+    JobTimeoutException, UnityAdvancedDedupRequireCompressionEnabledError, \
+    UnityCompressionRequireAllFlashPoolError
 from storops.unity.enums import HostLUNAccessEnum, NodeEnum, RaidTypeEnum, \
     ESXFilesystemBlockSizeEnum, ESXFilesystemMajorVersionEnum
 from storops.unity.resource.disk import UnityDisk
@@ -43,6 +43,7 @@
     UnityIoLimitRuleSetting
 from storops.unity.resource.remote_system import UnityRemoteSystem
 from storops.unity.resource.snap import UnitySnap
+from storops.unity.resource.snap_schedule import UnitySnapSchedule
 from storops.unity.resource.sp import UnityStorageProcessor
 from storops.unity.resource.storage_resource import UnityStorageResource
 from storops.unity.resp import RestResponse
@@ -83,6 +84,10 @@ def test_get_lun_sv2_simple_property(self):
         assert_that(lun.io_limit_rule, none())
         assert_that(lun.is_compression_enabled, equal_to(False))
         assert_that(lun.is_data_reduction_enabled, equal_to(False))
+        assert_that(lun.is_advanced_dedup_enabled, equal_to(False))
+        assert_that(lun.data_reduction_size_saved, equal_to(0))
+        assert_that(lun.data_reduction_percent, equal_to(0))
+        assert_that(lun.data_reduction_ratio, equal_to(1.0))
 
     @patch_rest
     def test_lun_modify_host_access(self):
@@ -152,6 +157,13 @@ def test_lun_modify_compression_enabled(self):
         lun.update()
         assert_that(lun.is_data_reduction_enabled, equal_to(True))
 
+    @patch_rest
+    def test_lun_modify_dedup_enabled(self):
+        lun = UnityLun(_id='sv_19', cli=t_rest(version='5.0.0'))
+        lun.modify(is_advanced_dedup_enabled=True)
+        lun.update()
+        assert_that(lun.is_advanced_dedup_enabled, equal_to(True))
+
     @mock.patch(target='storops.lib.job_helper.JobHelper',
                 new=MockJobHelper)
     @patch_rest
@@ -341,6 +353,42 @@ def test_create_with_io_limit(self):
         assert_that(rule.max_kbps_density, equal_to(1100))
         assert_that(rule.name, equal_to('Density_1100_KBPS_rule'))
 
+    @patch_rest
+    def test_create_with_dedup_enabled_compression_enabled(self):
+        cli = t_rest(version='5.0.0')
+        pool = UnityPool('pool_1', cli=cli)
+        lun = pool.create_lun('lun_1',
+                              is_compression=True,
+                              is_advanced_dedup_enabled=True)
+        assert_that(lun.name, equal_to('lun_1'))
+        assert_that(lun.is_data_reduction_enabled, equal_to(True))
+        assert_that(lun.is_advanced_dedup_enabled, equal_to(True))
+
+    @patch_rest
+    def test_create_with_dedup_enabled_compression_disabled(self):
+        def f():
+            cli = t_rest(version='5.0.0')
+            pool = UnityPool('pool_1', cli=cli)
+            pool.create_lun('lun_2',
+                            is_compression=False,
+                            is_advanced_dedup_enabled=True)
+
+        assert_that(f,
+                    raises(UnityAdvancedDedupRequireCompressionEnabledError))
+
+    @patch_rest
+    def test_create_with_dedup_enabled_in_non_all_flash_pool(self):
+        def f():
+            cli = t_rest(version='5.0.0')
+            pool = UnityPool('pool_1', cli=cli)
+            pool.is_all_flash = False
+            pool.create_lun('lun_3',
+                            is_compression=True,
+                            is_advanced_dedup_enabled=True)
+
+        assert_that(f,
+                    raises(UnityCompressionRequireAllFlashPoolError))
+
     @patch_rest
     def test_expand_lun_success(self):
         lun = UnityLun('sv_2', cli=t_rest())

diff --git a/storops_test/unity/resource/test_pool.py b/storops_test/unity/resource/test_pool.py
@@ -18,7 +18,6 @@
 from unittest import TestCase
 
 from hamcrest import assert_that, equal_to, instance_of, raises
-from storops.unity.resource.snap_schedule import UnitySnapSchedule
 
 from storops.exception import UnityLunNameInUseError, JobStateError, \
     UnityPoolNameInUseError
@@ -29,12 +28,12 @@
     StoragePoolTypeEnum, ESXFilesystemMajorVersionEnum, \
     ESXFilesystemBlockSizeEnum
 from storops.unity.resource.disk import UnityDiskGroup, UnityDisk
+from storops.unity.resource.lun import UnityLun
+from storops.unity.resource.nas_server import UnityNasServer
 from storops.unity.resource.pool import UnityPool, UnityPoolList, \
     RaidGroupParameter
-from storops.unity.resource.lun import UnityLun
+from storops.unity.resource.snap_schedule import UnitySnapSchedule
 from storops.unity.resource.sp import UnityStorageProcessor
-from storops.unity.resource.nas_server import UnityNasServer
-
 from storops_test.unity.rest_mock import t_rest, patch_rest
 
 __author__ = 'Cedric Zhuang'
@@ -282,6 +281,14 @@ def test_create_lun_with_compression_enabled(self):
                               is_compression=True)
         assert_that(lun, instance_of(UnityLun))
 
+    @patch_rest
+    def test_create_lun_with_dedup_enabled(self):
+        pool = UnityPool(_id='pool_1', cli=t_rest(version='5.0'))
+        lun = pool.create_lun("LunName", 100,
+                              is_compression=True,
+                              is_advanced_dedup_enabled=True)
+        assert_that(lun, instance_of(UnityLun))
+
     @patch_rest
     def test_create_vmfs(self):
         pool = UnityPool(_id='pool_1', cli=t_rest())

diff --git a/storops_test/unity/resource/test_storage_resource.py b/storops_test/unity/resource/test_storage_resource.py
@@ -20,7 +20,8 @@
 from hamcrest import assert_that, equal_to, instance_of
 
 from storops.unity.enums import StorageResourceTypeEnum, ReplicationTypeEnum, \
-    ThinStatusEnum, TieringPolicyEnum
+    ThinStatusEnum, TieringPolicyEnum, DataReductionStatusEnum, \
+    DedupStatusEnum
 from storops.unity.resource.filesystem import UnityFileSystem
 from storops.unity.resource.health import UnityHealth
 from storops.unity.resource.pool import UnityPoolList
@@ -56,6 +57,13 @@ def test_get_properties(self):
         assert_that(sr.snap_count, equal_to(0))
         assert_that(sr.pools, instance_of(UnityPoolList))
         assert_that(sr.filesystem, instance_of(UnityFileSystem))
+        assert_that(sr.advanced_dedup_status,
+                    equal_to(DedupStatusEnum.DISABLED))
+        assert_that(sr.data_reduction_status,
+                    equal_to(DataReductionStatusEnum.DISABLED))
+        assert_that(sr.data_reduction_size_saved, equal_to(0))
+        assert_that(sr.data_reduction_percent, equal_to(0))
+        assert_that(sr.data_reduction_ratio, equal_to(1.0))
 
     @patch_rest
     def test_get_all(self):

diff --git a/storops_test/unity/rest_data/lun/index.json b/storops_test/unity/rest_data/lun/index.json
@@ -96,6 +96,10 @@
       "url": "/api/instances/lun/sv_18?compact=True&fields=auSize,creationTime,currentNode,defaultNode,description,health,hostAccess,hostAccess.host.name,id,instanceId,ioLimitPolicy,isCompressionEnabled,isDataReductionEnabled,isReplicationDestination,isSnapSchedulePaused,isThinEnabled,lunGroupObjectId,metadataSize,metadataSizeAllocated,modificationTime,name,objectId,operationalStatus,perTierSizeUsed,pool,pool.isFASTCacheEnabled,pool.name,pool.raidType,sizeAllocated,sizeTotal,sizeUsed,smpBackendId,snapCount,snapSchedule,snapWwn,snapsSize,snapsSizeAllocated,storageResource,storageResource.type,tieringPolicy,type,wwn",
       "response": "sv_18.json"
     },
+    {
+      "url": "/api/instances/lun/sv_19?compact=True&fields=auSize,creationTime,currentNode,defaultNode,description,health,hostAccess,hostAccess.host.name,id,instanceId,ioLimitPolicy,isCompressionEnabled,isDataReductionEnabled,isReplicationDestination,isSnapSchedulePaused,isThinEnabled,lunGroupObjectId,metadataSize,metadataSizeAllocated,modificationTime,name,objectId,operationalStatus,perTierSizeUsed,pool,pool.isFASTCacheEnabled,pool.name,pool.raidType,sizeAllocated,sizeTotal,sizeUsed,smpBackendId,snapCount,snapSchedule,snapWwn,snapsSize,snapsSizeAllocated,storageResource,storageResource.type,tieringPolicy,type,wwn",
+      "response": "sv_19.json"
+    },
     {
       "url": "/api/instances/lun/sv_58?compact=True&fields=auSize,creationTime,currentNode,defaultNode,description,health,hostAccess,hostAccess.host.name,id,instanceId,ioLimitPolicy,isCompressionEnabled,isDataReductionEnabled,isReplicationDestination,isSnapSchedulePaused,isThinEnabled,lunGroupObjectId,metadataSize,metadataSizeAllocated,modificationTime,name,objectId,operationalStatus,perTierSizeUsed,pool,pool.isFASTCacheEnabled,pool.name,pool.raidType,sizeAllocated,sizeTotal,sizeUsed,smpBackendId,snapCount,snapSchedule,snapWwn,snapsSize,snapsSizeAllocated,storageResource,storageResource.type,tieringPolicy,type,wwn",
       "response": "sv_58.json"
@@ -180,6 +184,10 @@
       "url": "/api/instances/lun/sv_5612?compact=True&fields=auSize,creationTime,currentNode,defaultNode,description,health,hostAccess,hostAccess.host.name,id,instanceId,ioLimitPolicy,isCompressionEnabled,isDataReductionEnabled,isReplicationDestination,isSnapSchedulePaused,isThinEnabled,lunGroupObjectId,metadataSize,metadataSizeAllocated,modificationTime,name,objectId,operationalStatus,perTierSizeUsed,pool,pool.isFASTCacheEnabled,pool.name,pool.raidType,sizeAllocated,sizeTotal,sizeUsed,smpBackendId,snapCount,snapSchedule,snapWwn,snapsSize,snapsSizeAllocated,storageResource,storageResource.type,tieringPolicy,type,wwn",
       "response": "sv_5612.json"
     },
+    {
+      "url": "/api/instances/lun/sv_5620?compact=True&fields=auSize,creationTime,currentNode,defaultNode,description,health,hostAccess,hostAccess.host.name,id,instanceId,ioLimitPolicy,isCompressionEnabled,isDataReductionEnabled,isReplicationDestination,isSnapSchedulePaused,isThinEnabled,lunGroupObjectId,metadataSize,metadataSizeAllocated,modificationTime,name,objectId,operationalStatus,perTierSizeUsed,pool,pool.isFASTCacheEnabled,pool.name,pool.raidType,sizeAllocated,sizeTotal,sizeUsed,smpBackendId,snapCount,snapSchedule,snapWwn,snapsSize,snapsSizeAllocated,storageResource,storageResource.type,tieringPolicy,type,wwn",
+      "response": "sv_5620.json"
+    },
     {
       "url": "/api/instances/lun/sv_59?compact=True&fields=auSize,creationTime,currentNode,defaultNode,description,health,hostAccess,hostAccess.host.name,id,instanceId,ioLimitPolicy,isCompressionEnabled,isDataReductionEnabled,isReplicationDestination,isSnapSchedulePaused,isThinEnabled,lunGroupObjectId,metadataSize,metadataSizeAllocated,modificationTime,name,objectId,operationalStatus,perTierSizeUsed,pool,pool.isFASTCacheEnabled,pool.name,pool.raidType,sizeAllocated,sizeTotal,sizeUsed,smpBackendId,snapCount,snapSchedule,snapWwn,snapsSize,snapsSizeAllocated,storageResource,storageResource.type,tieringPolicy,type,wwn",
       "response": "sv_59.json"

diff --git a/storops_test/unity/rest_data/lun/sv_19.json b/storops_test/unity/rest_data/lun/sv_19.json
@@ -0,0 +1,73 @@
+{
+  "content": {
+    "id": "sv_19",
+    "operationalStatus": [
+      2
+    ],
+    "type": 2,
+    "tieringPolicy": 0,
+    "defaultNode": 1,
+    "currentNode": 1,
+    "auSize": 8192,
+    "migrationState": 1,
+    "instanceId": "root/emc:EMC_UEM_StorageVolumeLeaf%InstanceID=sv_19",
+    "creationTime": "2020-09-16T02:44:30.000Z",
+    "health": {
+      "value": 5,
+      "descriptionIds": [
+        "ALRT_VOL_OK"
+      ],
+      "descriptions": [
+        "The LUN is operating normally. No action is required."
+      ]
+    },
+    "name": "lun_1",
+    "modificationTime": "2020-09-16T02:44:31.996Z",
+    "sizeTotal": 1073741824,
+    "sizeAllocated": 0,
+    "sizePreallocated": 268435456,
+    "sizeAllocatedTotal": 3221225472,
+    "dataReductionSizeSaved": 0,
+    "dataReductionPercent": 0,
+    "dataReductionRatio": 1.0,
+    "perTierSizeUsed": [
+      0,
+      3489660928,
+      0,
+      0
+    ],
+    "isThinEnabled": true,
+    "isDataReductionEnabled": true,
+    "isAdvancedDedupEnabled": true,
+    "wwn": "60:06:01:60:6A:B0:4D:00:0D:7C:61:5F:C4:0C:EB:99",
+    "isReplicationDestination": false,
+    "isSnapSchedulePaused": false,
+    "objectId": 42949672967,
+    "metadataSize": 3758096384,
+    "metadataSizeAllocated": 3221225472,
+    "snapWwn": "60:06:01:60:6A:B0:4D:00:EA:E2:47:CF:73:4E:4A:4D",
+    "snapsSize": 0,
+    "snapsSizeAllocated": 0,
+    "snapCount": 0,
+    "lunNumber": 21,
+    "isThinClone": false,
+    "familySnapCount": 0,
+    "familyCloneCount": 0,
+    "nonBaseSizeAllocated": 0,
+    "familySizeAllocatedTotal": 3221225472,
+    "nonBaseSize": 0,
+    "storageResource": {
+      "id": "sv_19",
+      "type": 8
+    },
+    "pool": {
+      "id": "pool_1",
+      "raidType": 1,
+      "name": "flash_pool",
+      "isFASTCacheEnabled": false
+    },
+    "familyBaseLun": {
+      "id": "sv_19"
+    }
+  }
+}
diff --git a/storops_test/unity/rest_data/lun/sv_2.json b/storops_test/unity/rest_data/lun/sv_2.json
@@ -33,6 +33,10 @@
     "isThinEnabled": true,
     "isCompressionEnabled": false,
     "isDataReductionEnabled": false,
+    "isAdvancedDedupEnabled": false,
+    "dataReductionSizeSaved": 0,
+    "dataReductionPercent": 0,
+    "dataReductionRatio": 1.0,
     "wwn": "60:06:01:60:17:50:3C:00:C2:0A:D5:56:92:D1:BA:12",
     "isReplicationDestination": false,
     "isSnapSchedulePaused": false,