[Efs encryption][Draft] Support HeadNode Efs SharedStorage Encryption

cli/src/pcluster/config/cluster_config.py

@@ -33,6 +33,7 @@
 from pcluster.config.common import Imds as TopLevelImds
 from pcluster.config.common import (
     Resource,
+    SharedStorageType,
 )
 from pcluster.constants import (
     CIDR_ALL_IPS,
@@ -114,6 +115,7 @@
     SchedulerValidator,
     SharedEbsPerformanceBottleNeckValidator,
     SharedFileCacheNotHomeValidator,
+    SharedStorageEfsSettingsValidator,
     SharedStorageMountDirValidator,
     SharedStorageNameValidator,
     UnmanagedFsxMultiAzValidator,
@@ -291,15 +293,6 @@ def __init__(self, root_volume: RootVolume = None, ephemeral_volume: EphemeralVo
         self.ephemeral_volume = ephemeral_volume
 
 
-class SharedStorageType(Enum):
-    """Define storage types to be used as shared storage."""
-
-    EBS = "ebs"
-    RAID = "raid"
-    EFS = "efs"
-    FSX = "fsx"
-
-
 class SharedEbs(Ebs):
     """Represent a shared EBS, inherits from both _SharedStorage and Ebs classes."""
 
@@ -851,6 +844,14 @@ def __init__(self, allowed_ips: str = None, **kwargs):
         self.allowed_ips = Resource.init_param(allowed_ips)
 
 
+class SharedStorageEfsSettings(Resource):
+    """Represent the settings of Efs shared storage used by HeadNode."""
+
+    def __init__(self, encrypted: bool = False):
+        super().__init__()
+        self.encrypted = encrypted
+
+
 class Dcv(Resource):
     """Represent the DCV configuration."""
 
@@ -1434,6 +1435,7 @@ def __init__(
         disable_simultaneous_multithreading: bool = None,
         local_storage: LocalStorage = None,
         shared_storage_type: str = None,
+        shared_storage_efs_settings: SharedStorageEfsSettings = None,
         dcv: Dcv = None,
         custom_actions: CustomActions = None,
         iam: Iam = None,
@@ -1452,6 +1454,7 @@ def __init__(
             shared_storage_type,
             default="Ebs",
         )
+        self.shared_storage_efs_settings = shared_storage_efs_settings
         self.dcv = dcv
         self.custom_actions = custom_actions
         self.iam = iam or Iam(implied=True)
@@ -1461,6 +1464,11 @@ def __init__(
 
     def _register_validators(self, context: ValidatorContext = None):  # noqa: D102 #pylint: disable=unused-argument
         self._register_validator(InstanceTypeValidator, instance_type=self.instance_type)
+        self._register_validator(
+            SharedStorageEfsSettingsValidator,
+            shared_storage_type=self.shared_storage_type,
+            shared_storage_efs_settings=self.shared_storage_efs_settings,
+        )
 
     @property
     def architecture(self) -> str:

cli/src/pcluster/config/common.py

@@ -434,3 +434,12 @@ def dump_json(self):
         attribute_json = {"cluster": self._cluster_attributes}
         attribute_json.update(self._extra_attributes)
         return json.dumps(attribute_json, sort_keys=True)
+
+
+class SharedStorageType(Enum):
+    """Define storage types to be used as shared storage."""
+
+    EBS = "ebs"
+    RAID = "raid"
+    EFS = "efs"
+    FSX = "fsx"

cli/src/pcluster/schemas/cluster_schema.py

@@ -85,6 +85,7 @@
     SharedEbs,
     SharedEfs,
     SharedFsxLustre,
+    SharedStorageEfsSettings,
     SlurmClusterConfig,
     SlurmComputeResource,
     SlurmComputeResourceNetworking,
@@ -792,6 +793,17 @@ def make_resource(self, data, **kwargs):
         return HeadNodeSsh(**data)
 
 
+class SharedStorageEfsSettingsSchema(BaseSchema):
+    """Represent the schema of SharedStorageEfsSettings for the HeadNode."""
+
+    encrypted = fields.Bool(metadata={"update_policy": UpdatePolicy.UNSUPPORTED})
+
+    @post_load
+    def make_resource(self, data, **kwargs):
+        """Generate resource."""
+        return SharedStorageEfsSettings(**data)
+
+
 class DcvSchema(BaseSchema):
     """Represent the schema of DCV."""
 
@@ -1363,6 +1375,9 @@ class HeadNodeSchema(BaseSchema):
         metadata={"update_policy": UpdatePolicy.UNSUPPORTED},
         validate=validate.OneOf(["Ebs", "Efs"]),
     )
+    shared_storage_efs_settings = fields.Nested(
+        SharedStorageEfsSettingsSchema, metadata={"update_policy": UpdatePolicy.UNSUPPORTED}
+    )
     dcv = fields.Nested(DcvSchema, metadata={"update_policy": UpdatePolicy.UNSUPPORTED})
     custom_actions = fields.Nested(HeadNodeCustomActionsSchema, metadata={"update_policy": UpdatePolicy.IGNORED})
     iam = fields.Nested(HeadNodeIamSchema, metadata={"update_policy": UpdatePolicy.SUPPORTED})

cli/src/pcluster/templates/awsbatch_builder.py

@@ -22,7 +22,8 @@
 from aws_cdk.aws_ec2 import CfnSecurityGroup
 from aws_cdk.core import CfnOutput, CfnResource, Construct, Fn, Stack
 
-from pcluster.config.cluster_config import AwsBatchClusterConfig, CapacityType, SharedStorageType
+from pcluster.config.cluster_config import AwsBatchClusterConfig, CapacityType
+from pcluster.config.common import SharedStorageType
 from pcluster.constants import AWSBATCH_CLI_REQUIREMENTS, CW_LOG_GROUP_NAME_PREFIX, IAM_ROLE_PATH
 from pcluster.models.s3_bucket import S3Bucket
 from pcluster.templates.cdk_builder_utils import (

cli/src/pcluster/templates/cdk_builder_utils.py

@@ -29,11 +29,11 @@
     BaseQueue,
     HeadNode,
     LoginNodesPool,
-    SharedStorageType,
     SlurmClusterConfig,
     SlurmComputeResource,
     SlurmQueue,
 )
+from pcluster.config.common import SharedStorageType
 from pcluster.constants import (
     COOKBOOK_PACKAGES_VERSIONS,
     CW_LOGS_RETENTION_DAYS_DEFAULT,

cli/src/pcluster/templates/cluster_stack.py

@@ -53,10 +53,9 @@
     SharedEbs,
     SharedEfs,
     SharedFsxLustre,
-    SharedStorageType,
     SlurmClusterConfig,
 )
-from pcluster.config.common import DefaultUserHomeType
+from pcluster.config.common import DefaultUserHomeType, SharedStorageType
 from pcluster.constants import (
     ALL_PORTS_RANGE,
     CW_ALARM_DATAPOINTS_TO_ALARM_DEFAULT,
@@ -264,12 +263,9 @@ def _add_resources(self):
         # Add the internal use shared storage to the stack
         # This FS will be mounted, the shared dirs will be added,
         # then it will be unmounted and the shared dirs will be
-        # mounted.  We need to create the additional mount points first.
+        # mounted. We need to create the additional mount points first.
         if self.config.head_node.shared_storage_type.lower() == SharedStorageType.EFS.value:
-            internal_efs_storage_shared = SharedEfs(
-                mount_dir="/opt/parallelcluster/init_shared", name="internal_pcluster_shared", throughput_mode="elastic"
-            )
-            self._add_shared_storage(internal_efs_storage_shared)
+            self._add_internal_efs_shared_storage()
 
         # Add user configured shared storage
         if self.config.shared_storage:
@@ -335,6 +331,19 @@ def _add_resources(self):
                 head_node_alarms=self.head_node_alarms,
             )
 
+    def _add_internal_efs_shared_storage(self):
+        if self.config.head_node.shared_storage_efs_settings:
+            encrypted = self.config.head_node.shared_storage_efs_settings.encrypted
+        else:
+            encrypted = False
+        internal_efs_storage_shared = SharedEfs(
+            mount_dir="/opt/parallelcluster/init_shared",
+            name="internal_pcluster_shared",
+            throughput_mode="elastic",
+            encrypted=encrypted,
+        )
+        self._add_shared_storage(internal_efs_storage_shared)
+
     def _cw_metric_head_node(
         self, namespace, metric_name, statistic="Maximum", period_seconds=CW_ALARM_PERIOD_DEFAULT, extra_dimensions=None
     ):

cli/src/pcluster/templates/cw_dashboard_builder.py

@@ -17,7 +17,8 @@
 from aws_cdk.aws_cloudwatch import IAlarm
 from aws_cdk.core import Construct, Duration, Stack
 
-from pcluster.config.cluster_config import BaseClusterConfig, ExistingFileCache, SharedFsxLustre, SharedStorageType
+from pcluster.config.cluster_config import BaseClusterConfig, ExistingFileCache, SharedFsxLustre
+from pcluster.config.common import SharedStorageType
 from pcluster.constants import Feature
 from pcluster.utils import is_feature_supported
 

cli/src/pcluster/templates/login_nodes_stack.py

@@ -8,8 +8,8 @@
 from aws_cdk.core import CfnTag, Construct, Fn, NestedStack, Stack, Tags
 
 from pcluster.aws.aws_api import AWSApi
-from pcluster.config.cluster_config import LoginNodesPool, SharedStorageType, SlurmClusterConfig
-from pcluster.config.common import DefaultUserHomeType
+from pcluster.config.cluster_config import LoginNodesPool, SlurmClusterConfig
+from pcluster.config.common import DefaultUserHomeType, SharedStorageType
 from pcluster.constants import (
     DEFAULT_EPHEMERAL_DIR,
     NODE_BOOTSTRAP_TIMEOUT,

cli/src/pcluster/templates/queues_stack.py

@@ -7,8 +7,8 @@
 from constructs import Construct
 
 from pcluster.aws.aws_api import AWSApi
-from pcluster.config.cluster_config import SharedStorageType, SlurmClusterConfig, SlurmComputeResource, SlurmQueue
-from pcluster.config.common import DefaultUserHomeType
+from pcluster.config.cluster_config import SlurmClusterConfig, SlurmComputeResource, SlurmQueue
+from pcluster.config.common import DefaultUserHomeType, SharedStorageType
 from pcluster.constants import (
     DEFAULT_EPHEMERAL_DIR,
     NODE_BOOTSTRAP_TIMEOUT,

cli/src/pcluster/validators/cluster_validators.py

@@ -20,7 +20,7 @@
 from pcluster.aws.aws_resources import InstanceTypeInfo
 from pcluster.aws.common import AWSClientError
 from pcluster.cli.commands.dcv_util import get_supported_dcv_os
-from pcluster.config.common import CapacityType
+from pcluster.config.common import CapacityType, SharedStorageType
 from pcluster.constants import (
     CIDR_ALL_IPS,
     DELETE_POLICY,
@@ -676,7 +676,7 @@ def _check_file_storage(self, security_groups_by_nodes, file_storages, subnet_id
                         self._add_failure(
                             f"The current security group settings on file storage '{file_storage_id}' does not"
                             " satisfy mounting requirement. The file storage must be associated to a security group"
-                            f" that allows {direction } {protocol.upper()} traffic through ports {ports}. "
+                            f" that allows {direction} {protocol.upper()} traffic through ports {ports}. "
                             f"Missing ports: {missing_ports}",
                             FailureLevel.ERROR,
                         )
@@ -1336,6 +1336,24 @@ def _validate(self, head_node_instance_type: str, total_max_compute_nodes: int):
             )
 
 
+class SharedStorageEfsSettingsValidator(Validator):
+    """
+    HeadNode SharedStorageEfsSettings Validator.
+
+    Verify HeadNode SharedStorageEfsSettings can only be used with Efs SharedStorageType.
+    """
+
+    def _validate(self, shared_storage_type: str, shared_storage_efs_settings):
+        if shared_storage_efs_settings and shared_storage_type.lower() != SharedStorageType.EFS.value:
+            self._add_failure(
+                "SharedStorageEfsSettings is specified "
+                f"but the SharedStorageType is set to {shared_storage_type}. "
+                "SharedStorageEfsSettings can only be used when SharedStorageType "
+                f"is specified as {SharedStorageType.EFS.value}.",
+                FailureLevel.ERROR,
+            )
+
+
 class SharedEbsPerformanceBottleNeckValidator(Validator):
     """Warn potential performance bottleneck of using Shared EBS."""
 

cli/tests/pcluster/templates/test_cluster_stack/test_cluster_config_limits/slurm.full.all_resources.yaml

@@ -72,6 +72,8 @@ HeadNode:
       DeleteOnTermination: true
     EphemeralVolume:
       MountDir: /test
+  SharedStorageEfsSettings:
+    Encrypted: false
   SharedStorageType: Efs  # Ebs
   Dcv:
     Enabled: true

cli/tests/pcluster/templates/test_cluster_stack/test_cluster_config_limits/slurm.full_config.snapshot.yaml

@@ -70,6 +70,8 @@ HeadNode:
       HttpProxyAddress: https://proxy-address:port
     SecurityGroups: null
     SubnetId: subnet-12345678
+  SharedStorageEfsSettings:
+    Encrypted: false
   SharedStorageType: Efs
   Ssh:
     AllowedIps: 1.2.3.4/32

cli/tests/pcluster/templates/test_cw_dashboard_builder.py

@@ -15,7 +15,7 @@
 import yaml
 from assertpy import assert_that
 
-from pcluster.config.cluster_config import SharedStorageType
+from pcluster.config.common import SharedStorageType
 from pcluster.constants import Feature
 from pcluster.schemas.cluster_schema import ClusterSchema
 from pcluster.templates.cdk_builder import CDKTemplateBuilder

cli/tests/pcluster/validators/test_all_validators.py

@@ -198,6 +198,9 @@ def test_slurm_validators_are_called_with_correct_argument(test_datadir, mocker)
     number_of_storage_validator = mocker.patch(
         cluster_validators + ".NumberOfStorageValidator._validate", return_value=[]
     )
+    shared_storage_efs_settings_validator = mocker.patch(
+        cluster_validators + ".SharedStorageEfsSettingsValidator._validate", return_value=[]
+    )
     deletion_policy_validator = mocker.patch(cluster_validators + ".DeletionPolicyValidator._validate", return_value=[])
     root_volume_encryption_consistency_validator = mocker.patch(
         cluster_validators + ".RootVolumeEncryptionConsistencyValidator._validate", return_value=[]
@@ -392,6 +395,10 @@ def test_slurm_validators_are_called_with_correct_argument(test_datadir, mocker)
         ],
         any_order=True,
     )
+    shared_storage_efs_settings_validator.assert_has_calls(
+        [call(shared_storage_type="Ebs", shared_storage_efs_settings=None)],
+        any_order=True,
+    )
     # capacity reservation validators
     capacity_reservation_validator.assert_has_calls(
         [

cli/tests/pcluster/validators/test_cluster_validators.py

@@ -31,7 +31,7 @@
     SlurmSettings,
     Tag,
 )
-from pcluster.config.common import CapacityType
+from pcluster.config.common import CapacityType, SharedStorageType
 from pcluster.constants import PCLUSTER_NAME_MAX_LENGTH, PCLUSTER_NAME_MAX_LENGTH_SLURM_ACCOUNTING
 from pcluster.validators.cluster_validators import (
     FSX_MESSAGES,
@@ -71,6 +71,7 @@
     SchedulerDisableSudoAccessForDefaultUserValidator,
     SchedulerOsValidator,
     SharedFileCacheNotHomeValidator,
+    SharedStorageEfsSettingsValidator,
     SharedStorageMountDirValidator,
     SharedStorageNameValidator,
     UnmanagedFsxMultiAzValidator,
@@ -2075,6 +2076,50 @@ def test_mixed_security_group_overwrite_validator(head_node_security_groups, que
     assert_failure_messages(actual_failures, expected_message)
 
 
+@pytest.mark.parametrize(
+    "shared_storage_type, shared_storage_efs_settings, expected_message",
+    [
+        (
+            "Efs",
+            {"encrypted": True},
+            None,
+        ),
+        (
+            "Efs",
+            {"encrypted": False},
+            None,
+        ),
+        (
+            "Efs",
+            None,
+            None,
+        ),
+        (
+            "Ebs",
+            {"encrypted": True},
+            f"SharedStorageEfsSettings is specified but the SharedStorageType is set to Ebs. "
+            "SharedStorageEfsSettings can only be used when SharedStorageType "
+            f"is specified as {SharedStorageType.EFS.value}.",
+        ),
+        (
+            "Ebs",
+            {"encrypted": False},
+            f"SharedStorageEfsSettings is specified but the SharedStorageType is set to Ebs. "
+            "SharedStorageEfsSettings can only be used when SharedStorageType "
+            f"is specified as {SharedStorageType.EFS.value}.",
+        ),
+        (
+            "Ebs",
+            None,
+            None,
+        ),
+    ],
+)
+def test_shared_storage_efs_settings_validator(shared_storage_type, shared_storage_efs_settings, expected_message):
+    actual_failures = SharedStorageEfsSettingsValidator().execute(shared_storage_type, shared_storage_efs_settings)
+    assert_failure_messages(actual_failures, expected_message)
+
+
 @pytest.mark.parametrize(
     "root_volume_size, ami_size, expected_message",
     [