Add new parameters to storage creation command (#43)

* options added to the storage create command

* removed pool_name from settings for testing purposes

* refactor imports

* fix migrate command

* validate none uuids into list comprehension

src/hdx_cli/cli_interface/migrate/commands_v2.py

@@ -8,7 +8,7 @@
 from hdx_cli.cli_interface.migrate.helpers import MigrationData, get_catalog
 from hdx_cli.cli_interface.migrate.resources import get_resources, creating_resources
 from hdx_cli.cli_interface.migrate.validator import validates
-from hdx_cli.library_api.utility.decorators import report_error_and_exit
+from hdx_cli.library_api.utility.decorators import report_error_and_exit, ensure_logged_in
 from hdx_cli.library_api.common.logging import get_logger
 
 logger = get_logger()
@@ -62,6 +62,7 @@ def validate_tablename_format(ctx, param, value):
               help='Number of worker threads to use for migrating partitions. Default is 10.')
 @click.pass_context
 @report_error_and_exit(exctype=Exception)
+@ensure_logged_in
 def migrate(ctx: click.Context, source_table: str, target_table: str, target_profile_name: str,
             target_hostname: str, target_username: str, target_password: str,
             target_uri_scheme: str, allow_merge: bool, only: str, min_timestamp: datetime,

src/hdx_cli/cli_interface/storage/commands.py

@@ -1,20 +1,46 @@
 import json
 from functools import partial
+
 import click
 
 from ..common.migration import migrate_a_storage
-from ...library_api.utility.decorators import (report_error_and_exit,
-                                               dynamic_confirmation_prompt, ensure_logged_in)
+from ..common.rest_operations import list_ as command_list, show as command_show
+from ..common.undecorated_click_commands import (
+    basic_create_from_dict_body,
+    basic_delete,
+    basic_settings,
+    get_resource_list
+)
 from ...library_api.common.context import ProfileUserContext
 from ...library_api.common.logging import get_logger
-from ..common.undecorated_click_commands import basic_create_with_body_from_string, basic_create_from_dict_body
-from ..common.undecorated_click_commands import basic_delete, basic_settings
-from ..common.rest_operations import (list_ as command_list,
-                                      show as command_show)
+from ...library_api.utility.decorators import (
+    report_error_and_exit,
+    dynamic_confirmation_prompt,
+    ensure_logged_in
+)
 
 logger = get_logger()
 
 
+@report_error_and_exit(exctype=Exception)
+def get_credential_id(ctx, param, value):
+    if value is None:
+        return value
+
+    user_profile = ctx.parent.obj.get('usercontext')
+    org_id = user_profile.org_id
+    resource_path = f'/config/v1/orgs/{org_id}/credentials/'
+    credentials_list = get_resource_list(user_profile, resource_path)
+    credential_id = [
+        c.get('uuid')
+        for c in credentials_list
+        if c.get('name') == value and c.get('uuid') is not None
+    ]
+    if not credential_id:
+        raise click.BadParameter(f"Credential name '{value}' not found.")
+    return credential_id[0]
+
+
 @click.group(help="Storage-related operations")
 @click.option('--storage', 'storage_name', metavar='STORAGENAME', default=None,
               help='Perform operation on the passed storage.')
@@ -43,6 +69,14 @@ def storage(ctx: click.Context, storage_name: str):
               help='Region for the storage bucket.')
 @click.option('-c', '--cloud', default=None, required=False,
               help='Type of cloud storage (e.g., aws, gcp).')
+@click.option('-E', '--endpoint', default=None, required=False,
+              help='Endpoint for the storage bucket.')
+@click.option('-C', '--credential-name', 'credential_id', default=None, required=False,
+              callback=get_credential_id,
+              help='Name of the credential to use for the storage bucket.')
+@click.option('-M', '--io-perf-mode', default=None, required=False,
+              type=click.Choice(['aggressive', 'moderate', 'relaxed']),
+              help='I/O performance mode for the storage bucket.')
 @click.pass_context
 @report_error_and_exit(exctype=Exception)
 def create(ctx: click.Context,
@@ -51,30 +85,37 @@ def create(ctx: click.Context,
            bucket_path: str,
            bucket_name: str,
            region: str,
-           cloud: str):
+           cloud: str,
+           endpoint: str,
+           credential_id: str,
+           io_perf_mode: str):
     if not settings_filename and not all((bucket_path, bucket_name, region, cloud)):
-        raise click.BadParameter("You must specify either a settings file or the bucket path, name, region, and cloud.")
+        raise click.BadParameter(
+            "You must specify either a settings file or the bucket path, name, region, and cloud."
+        )
 
     user_profile = ctx.parent.obj.get('usercontext')
     resource_path = ctx.parent.obj.get('resource_path')
 
-    if settings_filename:
-        with open(settings_filename, "r", encoding="utf-8") as file:
-            basic_create_with_body_from_string(user_profile,
-                                               resource_path,
-                                               storage_name,
-                                               file.read())
-    else:
-        storage_settings_list = [('bucket_path', bucket_path),
-                                 ('bucket_name', bucket_name),
-                                 ('region', region),
-                                 ('cloud', cloud)]
+    if not settings_filename:
+        storage_settings = {
+            'bucket_path': bucket_path,
+            'bucket_name': bucket_name,
+            'region': region,
+            'cloud': cloud,
+            'endpoint': endpoint,
+            'credential_id': credential_id,
+            'io_perf_mode': io_perf_mode
+        }
         body = {
-            'name': storage_name,
-            'settings': {key: value for key, value in storage_settings_list}
+            'settings': {key: value for key, value in storage_settings.items() if value is not None},
         }
-        basic_create_from_dict_body(user_profile, resource_path, body)
+    else:
+        with open(settings_filename, "r", encoding="utf-8") as file:
+            body = json.load(file)
 
+    body['name'] = storage_name
+    basic_create_from_dict_body(user_profile, resource_path, body)
     logger.info(f'Created storage {storage_name}')
 
 

tests/command_line_interface/tests_data/sources/kafka_source_settings.json

@@ -11,7 +11,6 @@
             "test_ci_topic"
         ]
     },
-    "pool_name": "my-kafka-pool",
     "k8s_deployment": {
         "cpu": 1,
         "replicas": 1,

tests/command_line_interface/tests_data/sources/kinesis_source_settings.json

@@ -1,5 +1,4 @@
 {
-    "pool_name": "kinesis-pool",
     "k8s_deployment": {
         "cpu": 1,
         "memory": "10Gi",

tests/command_line_interface/tests_data/sources/siem_source_settings.json

@@ -1,7 +1,6 @@
 {
     "transform": "test_ci_transform",
     "table": "test_ci_project.test_ci_table",
-    "pool_name": "siem12-pool",
     "k8s_deployment": {
         "replicas": 1,
         "cpu": 1,