Multi instance launch (#736)

* launching specified num of instance, accespting instance_count as an extra args

* check quota for multi instances launch

* update comments for launch_instance()

* validate instance_count in _pre_launch_validation()

* Make a new path for create instance request when instance_count is passed as extra parameter

* validate instance_count before use

* New exception type for invalid instance_count

* return a list of instances in the response when instance_count exist as extra parameter

* log error, when one serialzed instance of the multi-instance-launch is invalid

* yapf on changed files

* travis yapf formatting

* update CHANGELOG

Co-authored-by: Nancy Purcell <npurcella@users.noreply.github.com>

CHANGELOG.md

@@ -25,6 +25,10 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/)
 -->
 
 ## [Unreleased](https://github.com/cyverse/atmosphere/compare/v36-6...HEAD) - YYYY-MM-DD
+### Added
+  - Add support for multi-instance-launch, 1 request can launch multiple instances by passing `instance_count` as an extra parameter
+    ([#736](https://github.com/cyverse/atmosphere/pull/736))
+
 ### Changed
   - Update and add Jetstream scripts
   ([#735](https://github.com/cyverse/atmosphere/pull/735))

api/v2/views/instance.py

@@ -352,6 +352,13 @@ def create(self, request):
             allocation_source = AllocationSource.objects.get(
                 uuid=allocation_source_id
             )
+
+            if 'instance_count' in extra:
+                return self._multi_create(
+                    user, identity_uuid, size_alias, source_alias, name, deploy,
+                    allocation_source, project, boot_scripts, extra
+                )
+
             core_instance = launch_instance(
                 user,
                 identity_uuid,
@@ -407,3 +414,53 @@ def create(self, request):
                 "Returning 409-CONFLICT"
             )
             return failure_response(status.HTTP_409_CONFLICT, str(exc.message))
+
+    def _multi_create(
+        self, user, identity_uuid, size_alias, source_alias, name, deploy,
+        allocation_source, project, boot_scripts, extra
+    ):
+        """
+        1. Launch multiple instances
+        2. Serialize the launched instances
+        3. Return a list of serialized instances
+        """
+
+        core_instances = launch_instance(
+            user,
+            identity_uuid,
+            size_alias,
+            source_alias,
+            name,
+            deploy,
+            allocation_source=allocation_source,
+            **extra
+        )
+
+        serialized_data = []
+
+        # Serialize all instances launched
+        for core_instance in core_instances:
+            # Faking a 'partial update of nothing' to allow call to 'is_valid'
+            serialized_instance = InstanceSerializer(
+                core_instance,
+                context={'request': self.request},
+                data={},
+                partial=True
+            )
+            if not serialized_instance.is_valid():
+                logger.error(
+                    "multi-instance-launch, serialized instance is invalid, {}".
+                    format(serialized_instance)
+                )
+            instance = serialized_instance.save()
+            instance.project = project
+            instance.save()
+            if boot_scripts:
+                _save_scripts_to_instance(instance, boot_scripts)
+            instance.change_allocation_source(allocation_source)
+
+            # append to result
+            serialized_data.append(serialized_instance.data)
+
+        # return a list of instances in the response
+        return Response(serialized_data, status=status.HTTP_201_CREATED)

service/exceptions.py

@@ -211,3 +211,7 @@ class NonZeroDeploymentException(Exception):
 
 class TimeoutError(Exception):
     pass
+
+
+class BadInstanceCount(ServiceException):
+    pass

service/instance.py

@@ -53,7 +53,7 @@
     OverAllocationError, AllocationBlacklistedError, OverQuotaError,
     SizeNotAvailable, SecurityGroupNotCreated, VolumeAttachConflict,
     VolumeDetachConflict, UnderThresholdError, ActionNotAllowed,
-    InstanceDoesNotExist, InstanceLaunchConflict, Unauthorized
+    InstanceDoesNotExist, InstanceLaunchConflict, Unauthorized, BadInstanceCount
 )
 
 from service.accounts.openstack_manager import AccountDriver as OSAccountDriver
@@ -704,15 +704,31 @@ def update_status(esh_driver, instance_id, provider_uuid, identity_uuid, user):
 
 
 def _pre_launch_validation(
-    username, esh_driver, identity_uuid, boot_source, size, allocation_source
+    username,
+    esh_driver,
+    identity_uuid,
+    boot_source,
+    size,
+    allocation_source,
+    instance_count=1
 ):
     """
     Used BEFORE launching a volume/instance .. Raise exceptions here to be dealt with by the caller.
     """
+    # Raise BadInstanceCount Error if not int or non-positive
+    if not isinstance(instance_count, int) or instance_count < 1:
+        raise BadInstanceCount("Bad instance count: %s" % instance_count)
+
     identity = CoreIdentity.objects.get(uuid=identity_uuid)
 
     # May raise OverQuotaError
-    check_quota(username, identity_uuid, size, include_networking=True)
+    check_quota(
+        username,
+        identity_uuid,
+        size,
+        include_networking=True,
+        instance_count=instance_count
+    )
 
     # May raise OverAllocationError, AllocationBlacklistedError
     check_allocation(username, allocation_source)
@@ -742,12 +758,15 @@ def launch_instance(
     Initialization point --> launch_*_instance --> ..
     Required arguments will launch the instance, extras will do
     provider-specific modifications.
+    pass in number of instance to launch_kwargs['instance_count'].
 
     1. Test for available Size (on specific driver!)
     2. Test user has Quota/Allocation (on our DB)
     3. Test user is launching appropriate size (Not below Thresholds)
-    4. Perform an 'Instance launch' depending on Boot Source
-    5. Return CORE Instance with new 'esh' objects attached.
+    4. Perform an 'Instance launch' depending on Boot Source OR Perform
+    multiple instance launch from machine source if instance_count is passed in via launch_kwargs
+    5. Return CORE Instance with new 'esh' objects attached OR a list of
+    CORE instances if instance_count is passed in via launch_kwargs
     """
     now_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
     status_logger.debug(
@@ -768,6 +787,74 @@ def launch_instance(
     # May raise Exception("Size not available")
     size = check_size(esh_driver, size_alias, provider, boot_source)
 
+    # Checking if instance_count is passed as an arg
+    if 'instance_count' in launch_kwargs:
+        instance_count = launch_kwargs['instance_count']
+        del launch_kwargs['instance_count']
+        logger.debug(launch_kwargs)
+
+        return _launch_multiple_instances(
+            user, esh_driver, identity_uuid, boot_source, size, name, deploy,
+            instance_count, launch_kwargs
+        )
+    else:
+        return _launch_one_instance(
+            user, esh_driver, identity_uuid, boot_source, size, name, deploy,
+            launch_kwargs
+        )
+
+
+def _launch_multiple_instances(
+    user, esh_driver, identity_uuid, boot_source, size, name, deploy,
+    instance_count, launch_kwargs
+):
+    """
+    Launching multiple instances, all from the same machine,
+    can NOT boot from volume
+    """
+    # Raise any other exceptions before launching here
+    _pre_launch_validation(
+        user.username,
+        esh_driver,
+        identity_uuid,
+        boot_source,
+        size,
+        launch_kwargs.get('allocation_source'),
+        instance_count=instance_count
+    )
+
+    # checking boot source
+    if boot_source.is_volume():
+        raise Exception(
+            "Multi instance launch does not support volume as boot source"
+        )
+    elif not boot_source.is_machine():
+        raise Exception("Boot source is of an unknown type")
+
+    identity = CoreIdentity.objects.get(uuid=identity_uuid)
+
+    machine = _retrieve_source(esh_driver, boot_source.identifier, "machine")
+    core_instances = launch_multiple_machine_instances(
+        esh_driver,
+        user,
+        identity,
+        machine,
+        size,
+        name,
+        deploy=deploy,
+        instance_count=instance_count,
+        **launch_kwargs
+    )
+    return core_instances
+
+
+def _launch_one_instance(
+    user, esh_driver, identity_uuid, boot_source, size, name, deploy,
+    launch_kwargs
+):
+    """
+    Launching just a single instance
+    """
     # Raise any other exceptions before launching here
     _pre_launch_validation(
         user.username, esh_driver, identity_uuid, boot_source, size,
@@ -784,6 +871,7 @@ def launch_instance(
         deploy=deploy,
         **launch_kwargs
     )
+
     return core_instance
 
 
@@ -907,6 +995,47 @@ def launch_machine_instance(
     )
 
 
+def launch_multiple_machine_instances(
+    driver,
+    user,
+    identity,
+    machine,
+    size,
+    name,
+    deploy=True,
+    instance_count=1,
+    **kwargs
+):
+    """
+    Launch multiple instances off an existing machine
+    """
+    prep_kwargs, userdata, network = _pre_launch_instance(
+        driver, user, identity, size, name, **kwargs
+    )
+    kwargs.update(prep_kwargs)
+
+    core_instances = []
+
+    logger.debug(
+        "multi-instance-launch, launching {} instances".format(instance_count)
+    )
+    # launch specified number of instances
+    for i in range(instance_count):
+        instance, token, password = _launch_machine(
+            driver, identity, machine, size, name, userdata, network, **kwargs
+        )
+        core_instance = _complete_launch_instance(
+            driver, identity, instance, user, token, password, deploy=deploy
+        )
+        core_instances.append(core_instance)
+        logger.debug("multi-instance-launch, #{}, {}".format(i, core_instance))
+
+    logger.debug("multi-instance-launch, result {}".format(core_instances))
+
+    # return all instances
+    return core_instances
+
+
 def _boot_volume(
     driver,
     identity,
@@ -1308,14 +1437,21 @@ def check_allocation(username, allocation_source):
         )
 
 
-def check_quota(username, identity_uuid, esh_size, include_networking=False):
+def check_quota(
+    username,
+    identity_uuid,
+    esh_size,
+    include_networking=False,
+    instance_count=1
+):
     from service.quota import check_over_instance_quota
     try:
         check_over_instance_quota(
             username,
             identity_uuid,
             esh_size,
-            include_networking=include_networking
+            include_networking=include_networking,
+            instance_count=instance_count
         )
     except ValidationError as bad_quota:
         raise OverQuotaError(message=bad_quota.message)

service/quota.py

@@ -17,13 +17,15 @@ def check_over_instance_quota(
     identity_uuid,
     esh_size=None,
     include_networking=False,
-    raise_exc=True
+    raise_exc=True,
+    instance_count=1
 ):
     """
     Checks quota based on current limits (and an instance of size, if passed).
     param - esh_size - if included, update the CPU and Memory totals & increase instance_count
     param - launch_networking - if True, increase floating_ip_count
     param - raise_exc - if True, raise ValidationError, otherwise return False
+    param - instance_count - number of instance to be launched with the same size, default to 1
 
     return True if passing
     return False if ValidationError occurs and raise_exc=False
@@ -42,12 +44,12 @@ def check_over_instance_quota(
     driver = get_cached_driver(identity=identity)
     new_port = new_floating_ip = new_instance = new_cpu = new_ram = 0
     if esh_size:
-        new_cpu += esh_size.cpu
-        new_ram += esh_size.ram
-        new_instance += 1
-        new_port += 1
+        new_cpu += esh_size.cpu * instance_count
+        new_ram += esh_size.ram * instance_count
+        new_instance += instance_count
+        new_port += instance_count
     if include_networking:
-        new_floating_ip += 1
+        new_floating_ip += instance_count
     # Will throw ValidationError if false.
     try:
         has_cpu_quota(driver, quota, new_cpu)