From 52db2affbc813450dbb60cd518c0e07f25154cb7 Mon Sep 17 00:00:00 2001 From: Giacomo Marciani Date: Mon, 14 Jul 2025 17:01:48 -0400 Subject: [PATCH 1/3] [Test] In test_build_image, when Os is Rocky9, use the vanilla AMI, update packages. This is because we want to guarantee that we work on the latest version of Rocky9. As a consequence, we need to disable the installation of Lustre for this test case (as we do for other OSes) because we cannot guarantee that Lustre releases the client on time with the updated version, and we want the test to be decoupled from it. --- tests/integration-tests/tests/createami/test_createami.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/tests/integration-tests/tests/createami/test_createami.py b/tests/integration-tests/tests/createami/test_createami.py index 0b985514f5..1ec1e8db14 100644 --- a/tests/integration-tests/tests/createami/test_createami.py +++ b/tests/integration-tests/tests/createami/test_createami.py @@ -132,7 +132,7 @@ def test_build_image( # Test Deep Learning AMIs base_ami = retrieve_latest_ami(region, os, ami_type="remarkable", architecture=architecture) enable_nvidia = False # Deep learning AMIs have Nvidia pre-installed - elif "rhel" in os or "rocky" in os or "ubuntu" in os: + elif "rhel" in os or "ubuntu" in os or os == "rocky8": # Test AMIs from first stage build. Because RHEL/Rocky and Ubuntu have specific requirement of kernel versions. try: base_ami = retrieve_latest_ami(region, os, ami_type="first_stage", architecture=architecture) @@ -141,12 +141,14 @@ def test_build_image( logging.info("First stage AMI not available, using official AMI instead.") base_ami = retrieve_latest_ami(region, os, ami_type="official", architecture=architecture) update_os_packages = True - if os in ["ubuntu2204", "rhel9", "rocky9"]: + if os in ["ubuntu2204", "rhel9"]: enable_lustre_client = False else: # Test vanilla AMIs. base_ami = retrieve_latest_ami(region, os, ami_type="official", architecture=architecture) - if os in ["alinux2", "alinux2023"]: + if os in ["rocky9"]: + enable_lustre_client = False + if os in ["alinux2", "alinux2023", "rocky9"]: update_os_packages = True image_config = pcluster_config_reader( config_file="image.config.yaml", From 78656df5daeb96ff3b6550a615366c9a97d18b6d Mon Sep 17 00:00:00 2001 From: Giacomo Marciani Date: Mon, 14 Jul 2025 17:43:32 -0400 Subject: [PATCH 2/3] [Test] In test_pcluster_configure, assert that head node and compute instance types are taken from the current free tier. --- .../configure/test_pcluster_configure.py | 31 ++++++++++++++----- tests/integration-tests/utils.py | 26 ++++++++++++++++ 2 files changed, 50 insertions(+), 7 deletions(-) diff --git a/tests/integration-tests/tests/configure/test_pcluster_configure.py b/tests/integration-tests/tests/configure/test_pcluster_configure.py index 09148a1bce..abdcd7db27 100644 --- a/tests/integration-tests/tests/configure/test_pcluster_configure.py +++ b/tests/integration-tests/tests/configure/test_pcluster_configure.py @@ -21,14 +21,17 @@ from cfn_stacks_factory import CfnVpcStack from conftest import inject_additional_config_settings from conftest_networking import CIDR_FOR_CUSTOM_SUBNETS -from utils import get_instance_info +from utils import get_free_tier_instance_types, get_instance_info, or_regex PROMPTS = { "region": lambda region: {"prompt": r"AWS Region ID \[.*\]: ", "response": region}, "key_pair": lambda key_name: {"prompt": r"EC2 Key Pair Name \[.*\]: ", "response": key_name}, "scheduler": lambda scheduler: {"prompt": r"Scheduler \[slurm\]: ", "response": scheduler}, "os": lambda os: {"prompt": r"Operating System \[alinux2\]: ", "response": os, "skip_for_batch": True}, - "head_instance_type": lambda instance: {"prompt": r"Head node instance type \[t.\.micro\]: ", "response": instance}, + "head_instance_type": lambda free_tier_instance_types, instance: { + "prompt": rf"Head node instance type \[({or_regex(free_tier_instance_types)})\]: ", + "response": instance, + }, "no_of_queues": lambda n: {"prompt": rf"Number of queues \[{n}\]: ", "response": f"{n}", "skip_for_batch": True}, "queue_name": lambda queue, name: {"prompt": rf"Name of queue {queue} \[queue{queue}\]: ", "response": name}, "no_of_compute_resources": lambda queue_name, queue, n: { @@ -36,8 +39,9 @@ "response": f"{n}", "skip_for_batch": True, }, - "compute_instance_type": lambda resource, queue_name, instance: { - "prompt": rf"Compute instance type for compute resource {resource} in {queue_name} \[t.\.micro\]: ", + "compute_instance_type": lambda free_tier_instance_types, resource, queue_name, instance: { + "prompt": rf"Compute instance type for compute resource {resource} in {queue_name} " + + rf"\[({or_regex(free_tier_instance_types)})\]: ", "response": instance, "skip_for_batch": True, }, @@ -161,7 +165,12 @@ def test_efa_and_placement_group( PROMPTS["no_of_queues"](1), PROMPTS["queue_name"](queue=1, name="myqueue"), PROMPTS["no_of_compute_resources"](queue_name="myqueue", queue=1, n=1), - PROMPTS["compute_instance_type"](resource=1, queue_name="myqueue", instance=instance), + PROMPTS["compute_instance_type"]( + free_tier_instance_types=get_free_tier_instance_types(region), + resource=1, + queue_name="myqueue", + instance=instance, + ), PROMPTS["enable_efa"](efa_response), prompt_max_size(scheduler=scheduler), ] @@ -237,7 +246,10 @@ def standard_first_stage_prompts(region, key_name, scheduler, os, instance): PROMPTS["key_pair"](key_name), PROMPTS["scheduler"](scheduler), PROMPTS["os"](os), - PROMPTS["head_instance_type"](instance), + PROMPTS["head_instance_type"]( + free_tier_instance_types=get_free_tier_instance_types(region), + instance=instance, + ), ] @@ -246,7 +258,12 @@ def standard_queue_prompts(scheduler, instance, region, size=""): PROMPTS["no_of_queues"](1), PROMPTS["queue_name"](queue=1, name="myqueue"), PROMPTS["no_of_compute_resources"](queue_name="myqueue", queue=1, n=1), - PROMPTS["compute_instance_type"](resource=1, queue_name="myqueue", instance=instance), + PROMPTS["compute_instance_type"]( + free_tier_instance_types=get_free_tier_instance_types(region), + resource=1, + queue_name="myqueue", + instance=instance, + ), ] is_efa_supported = False diff --git a/tests/integration-tests/utils.py b/tests/integration-tests/utils.py index 97a8fad568..65653b7fb3 100644 --- a/tests/integration-tests/utils.py +++ b/tests/integration-tests/utils.py @@ -908,3 +908,29 @@ def find_stack_by_tag(tag, region, stack_prefix): logging.info(f"Found stack: {name} (created on {creation_date})") return name return None + + +def get_free_tier_instance_types(region: str = None): + instance_types = [] + ec2 = boto3.client("ec2", region_name=region) + + # Pagination here is not only a best practice; it is required to make DescribeInstanceTypes return + # the expected values when we want to filter all instance types. + # If you remove pagination, this is going to return an empty list of instance types. + paginator = ec2.get_paginator("describe_instance_types") + pages = paginator.paginate( + Filters=[ + {"Name": "free-tier-eligible", "Values": ["true"]}, + {"Name": "current-generation", "Values": ["true"]}, + ] + ) + + for page in pages: + instance_types.extend([instance["InstanceType"] for instance in page["InstanceTypes"]]) + + logging.info(f"Free tier instance types in region {region}: {instance_types}") + return instance_types + + +def or_regex(items: list): + return "|".join(map(re.escape, items)) From 2a14cbab563a4244339c31fd029359d91b8726fb Mon Sep 17 00:00:00 2001 From: Giacomo Marciani Date: Tue, 15 Jul 2025 09:59:00 -0400 Subject: [PATCH 3/3] [CodeStyle] Address linter checks. --- cli/tests/pcluster/cli/test_update_cluster.py | 4 +--- cli/tests/pcluster/config/test_config_patch.py | 1 + 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/cli/tests/pcluster/cli/test_update_cluster.py b/cli/tests/pcluster/cli/test_update_cluster.py index 6c0cf52203..69a9630797 100644 --- a/cli/tests/pcluster/cli/test_update_cluster.py +++ b/cli/tests/pcluster/cli/test_update_cluster.py @@ -247,9 +247,7 @@ def test_no_nodejs_error(self, mocker, test_datadir): assert_that(exc_info.value.data.get("message")).matches("Node.js is required") def test_validate_update_request(self, mocker): - """ - Tests that instance type attribute of the old configuration is never retrieved during an update. - """ + """Tests that instance type attribute of the old configuration is never retrieved during an update.""" new_configuration = """ Image: Os: alinux2 diff --git a/cli/tests/pcluster/config/test_config_patch.py b/cli/tests/pcluster/config/test_config_patch.py index 5cc06f5a09..de6ac9f7f4 100644 --- a/cli/tests/pcluster/config/test_config_patch.py +++ b/cli/tests/pcluster/config/test_config_patch.py @@ -281,6 +281,7 @@ def test_load_config_for_capacity_reservation( ): """ This test checks that when loading a configuration, describe_capacity_reservations does not cause a failure. + This ensures that when the old configuration is loaded during an update, it does not matter whether the capacity reservation is accessible. The existence of the capacity reservation should only be checked during the validation phase. This should happen