Added functionality for storages and SIEM sources

src/hdx_cli/cli_interface/common/cached_operations.py

@@ -14,13 +14,21 @@ def find_kafka(user_ctx: ProfileUserContext):
                      ('tables', user_ctx.tablename),
                      ('sources/kafka', None)])
 
+
 def find_kinesis(user_ctx: ProfileUserContext):
     return access_resource(user_ctx,
                     [('projects', user_ctx.projectname),
                      ('tables', user_ctx.tablename),
                      ('sources/kinesis', None)])
 
 
+def find_siem(user_ctx: ProfileUserContext):
+    return access_resource(user_ctx,
+                    [('projects', user_ctx.projectname),
+                     ('tables', user_ctx.tablename),
+                     ('sources/siem', None)])
+
+
 def find_projects(user_ctx: ProfileUserContext):
     token = user_ctx.auth
     hostname = user_ctx.hostname
@@ -111,3 +119,14 @@ def find_transform_id(user_ctx, transform_name):
     return [t["uuid"] for t in transforms if t["name"] == transform_name]
 
 
+def find_storages(user_ctx: ProfileUserContext):
+    token = user_ctx.auth
+    hostname = user_ctx.hostname
+    scheme = user_ctx.scheme
+    url = f"{scheme}://{hostname}/config/v1/orgs/{user_ctx.org_id}/storages/"
+    headers = {"Authorization": f"{token.token_type} {token.token}",
+               "Accept": "application/json"}
+    result = requests.get(url, headers=headers, timeout=30)
+    if result.status_code != 200:
+        raise HdxCliException(f"Error getting storages.")
+    return json.loads(result.content)

src/hdx_cli/cli_interface/common/rest_operations.py

@@ -97,13 +97,15 @@ def _heuristically_get_resource_kind(resource_path) -> Tuple[str, str]:
 
 @click.command(help='Show resource. If not resource_name is provided, it will show the default if there is one.')
 @click.pass_context
+@click.option("--indent", type=int, help='Number of spaces for indentation in the output.')
 @report_error_and_exit(exctype=Exception)
-def show(ctx: click.Context):
+def show(ctx: click.Context,
+         indent: int):
     profile = ctx.parent.obj['usercontext']
     _, resource_kind = _heuristically_get_resource_kind(ctx.parent.obj['resource_path'])
     resource_name = getattr(profile, resource_kind + 'name')
 
     resource_path = ctx.parent.obj['resource_path']
     profile = ctx.parent.obj['usercontext']
     print(basic_show(profile, resource_path,
-                     resource_name))
+                     resource_name, indent))

src/hdx_cli/cli_interface/common/undecorated_click_commands.py

@@ -4,14 +4,15 @@
 import click
 
 from hdx_cli.library_api.common.context import ProfileUserContext
-from hdx_cli.library_api.common.exceptions import LogicException, HdxCliException
-
+from hdx_cli.library_api.common.exceptions import LogicException, HdxCliException, TransformNotFoundException
 
 from .cached_operations import * #pylint:disable=wildcard-import,unused-wildcard-import
 from ...library_api.common import rest_operations as rest_ops
+from ...library_api.userdata.token import AuthInfo
 
 _MAX_TIMEOUT = 30
 
+
 def basic_create(profile,
                  resource_path,
                  resource_name: str,
@@ -74,8 +75,10 @@ def basic_create_with_body_from_string(profile,
     rest_ops.create(url, body=body, headers=headers, body_type=body_from_string_type)
 
 
-
-def basic_show(profile, resource_path, resource_name):
+def basic_show(profile,
+               resource_path,
+               resource_name,
+               indent: Optional[int] = None):
     hostname = profile.hostname
     scheme = profile.scheme
     list_url = f'{scheme}://{hostname}{resource_path}'
@@ -85,7 +88,7 @@ def basic_show(profile, resource_path, resource_name):
     resources = rest_ops.list(list_url, headers=headers)
     for resource in resources:
         if resource['name'] == resource_name:
-            return json.dumps(resource)
+            return json.dumps(resource, indent=indent)
 
 
 def basic_transform(ctx: click.Context):
@@ -162,6 +165,7 @@ def _do_create_dict_from_dotted_key_and_value(split_key, value,
                                                   value,
                                                   the_dict[split_key[0]])
 
+
 def _create_dict_from_dotted_key_and_value(dotted_key, value):
     the_dict = {}
     split_key = dotted_key.split(".")
@@ -312,7 +316,8 @@ def _settings_update(resource: Dict[str, Any],
 def basic_settings(profile,
                    resource_path,
                    key,
-                   value):
+                   value,
+                   force_operation: Optional[bool] = False):
     """Given a resource type, it returns the settings that can be used for it"""
     hostname = profile.hostname
     scheme = profile.scheme
@@ -346,6 +351,7 @@ def basic_settings(profile,
     else:
         try:
             this_resource_url = f'{settings_url}{resource["uuid"]}'
+            this_resource_url += '?force_operation=true' if force_operation else ''
             resource = _settings_update(resource, key, value)
             rest_ops.update_with_put(
                 this_resource_url,
@@ -354,14 +360,18 @@ def basic_settings(profile,
         except:
             patch_data = _create_dict_from_dotted_key_and_value(key, value)
             this_resource_url = f'{settings_url}{resource["uuid"]}'
+            this_resource_url += '?force_operation=true' if force_operation else ''
             rest_ops.update_with_patch(
                 this_resource_url,
                 headers=headers,
                 body=patch_data)
         print(f'Updated {resource["name"]} {key}')
 
 
-def basic_delete(profile, resource_path, resource_name: str):
+def basic_delete(profile,
+                 resource_path,
+                 resource_name: str,
+                 force_operation: Optional[bool] = False):
     hostname = profile.hostname
     scheme = profile.scheme
     list_url = f'{scheme}://{hostname}{resource_path}'
@@ -380,6 +390,7 @@ def basic_delete(profile, resource_path, resource_name: str):
     if not url:
         return False
     else:
+        url += '?force_operation=true' if force_operation else ''
         rest_ops.delete(url, headers=headers)
         return True
 

src/hdx_cli/cli_interface/set/commands.py

@@ -39,7 +39,9 @@ def set(ctx, projectname, tablename, scheme=None):
         profile.tablename = tablename
     _serialize_to_config_file(profile, profile.profile_config_file)
 
-
+# MINOR CHANGE
+# This can be improved. In some situation, you can just change the table name and not the project name.
+# So, this command, need to be able to receive the parameter "table", to unset just the table name.
 @click.command(help='Remove any set projects/tables')
 @click.pass_context
 def unset(ctx):

src/hdx_cli/cli_interface/sources/commands.py

@@ -8,22 +8,17 @@
 from ...library_api.utility.decorators import report_error_and_exit
 from ...library_api.common.exceptions import HdxCliException, LogicException
 
-from ..common.rest_operations import (create as command_create,
-                                      delete as command_delete,
-                                      list_ as command_list,
-                                      show as command_show)
-
-from ..common.misc_operations import settings as command_settings
-from .kafkakinesis import (kafka as command_kafka,
-                           kinesis as command_kinesis)
+from .kafka import kafka as command_kafka
+from .kinesis import kinesis as command_kinesis
 from .summary import summary as command_summary
+from .siem import siem as command_siem
 
 
 @click.group(help="Sources-related operations")
 @click.pass_context
 @report_error_and_exit(exctype=Exception)
 def sources(ctx: click.Context):
-    profile_info : ProfileUserContext = ctx.obj['usercontext']
+    profile_info: ProfileUserContext = ctx.obj['usercontext']
     project_name, table_name = profile_info.projectname, profile_info.tablename
     if not project_name:
         raise HdxCliException("Error. No project name provided and no 'projectname' set in profile")
@@ -55,3 +50,4 @@ def sources(ctx: click.Context):
 sources.add_command(command_kafka)
 sources.add_command(command_kinesis)
 sources.add_command(command_summary)
+sources.add_command(command_siem)

src/hdx_cli/cli_interface/sources/common_source_actions.py

@@ -0,0 +1,28 @@
+import click
+
+from ...library_api.utility.decorators import report_error_and_exit
+from ..common.undecorated_click_commands import basic_create_with_body_from_string
+
+
+def _any_source_impl(ctx: click.Context, source_name):
+    profileinfo = ctx.parent.obj['usercontext']
+    sources_path = ctx.parent.obj['resource_path']
+    sourcename_resource_path = f'{sources_path}{source_name}/'
+    ctx.obj = {'usercontext': profileinfo,
+                'resource_path': sourcename_resource_path}
+
+
+@click.command(help="Create source. 'source_filename' contains the settings. name in settings will be replaced by 'source_name'")
+@click.argument('source_filename')
+@click.argument('source_name')
+@click.pass_context
+@report_error_and_exit(exctype=Exception)
+def create(ctx: click.Context,
+           source_filename: str,
+           source_name: str):
+    user_profile = ctx.parent.obj['usercontext']
+    resource_path = ctx.parent.obj['resource_path']
+    with open(source_filename, "r", encoding="utf-8") as f:
+        basic_create_with_body_from_string(user_profile, resource_path,
+                                           source_name, f.read())
+    print(f'Created source {source_name}.')

src/hdx_cli/cli_interface/sources/kafka.py

@@ -0,0 +1,24 @@
+import click
+
+from ...library_api.utility.decorators import report_error_and_exit
+from ..common.rest_operations import (delete as command_delete,
+                                      list_ as command_list,
+                                      show as command_show)
+
+from ..common.misc_operations import settings as command_settings
+from .common_source_actions import create as command_create
+from .common_source_actions import _any_source_impl as any_source_impl
+
+
+@click.group(help="Kafka source operations")
+@click.pass_context
+@report_error_and_exit(exctype=Exception)
+def kafka(ctx: click.Context):
+    any_source_impl(ctx, 'kafka')
+
+
+kafka.add_command(command_create)
+kafka.add_command(command_delete)
+kafka.add_command(command_list)
+kafka.add_command(command_show)
+kafka.add_command(command_settings)

src/hdx_cli/cli_interface/sources/kinesis.py

@@ -0,0 +1,24 @@
+import click
+
+from ...library_api.utility.decorators import report_error_and_exit
+from ..common.rest_operations import (delete as command_delete,
+                                      list_ as command_list,
+                                      show as command_show)
+
+from ..common.misc_operations import settings as command_settings
+from .common_source_actions import create as command_create
+from .common_source_actions import _any_source_impl as any_source_impl
+
+
+@click.group(help="Kinesis source operations")
+@click.pass_context
+@report_error_and_exit(exctype=Exception)
+def kinesis(ctx: click.Context):
+    any_source_impl(ctx, 'kinesis')
+
+
+kinesis.add_command(command_create)
+kinesis.add_command(command_delete)
+kinesis.add_command(command_list)
+kinesis.add_command(command_show)
+kinesis.add_command(command_settings)

src/hdx_cli/cli_interface/sources/siem.py

@@ -0,0 +1,24 @@
+import click
+
+from ...library_api.utility.decorators import report_error_and_exit
+from ..common.rest_operations import (delete as command_delete,
+                                      list_ as command_list,
+                                      show as command_show)
+
+from ..common.misc_operations import settings as command_settings
+from .common_source_actions import create as command_create
+from .common_source_actions import _any_source_impl as any_source_impl
+
+
+@click.group(help="SIEM source operations")
+@click.pass_context
+@report_error_and_exit(exctype=Exception)
+def siem(ctx: click.Context):
+    any_source_impl(ctx, 'siem')
+
+
+siem.add_command(command_create)
+siem.add_command(command_delete)
+siem.add_command(command_list)
+siem.add_command(command_show)
+siem.add_command(command_settings)

src/hdx_cli/cli_interface/sources/summary.py

@@ -1,31 +1,20 @@
 import click
 
-from ...library_api.common import rest_operations as rest_ops
 from ...library_api.utility.decorators import report_error_and_exit
-from ...library_api.common.exceptions import HdxCliException
-
-
 from ..common.rest_operations import (create as command_create,
                                       delete as command_delete,
                                       list_ as command_list,
                                       show as command_show)
 
 from ..common.misc_operations import settings as command_settings
-
-
-def _any_source_impl(ctx: click.Context, source_name):
-    profileinfo = ctx.parent.obj['usercontext']
-    sources_path = ctx.parent.obj['resource_path']
-    sourcename_resource_path =  f'{sources_path}{source_name}/'
-    ctx.obj = {'usercontext': profileinfo,
-                'resource_path': sourcename_resource_path}
+from .common_source_actions import _any_source_impl as any_source_impl
 
 
 @click.group(help="Summary source operations")
 @click.pass_context
 @report_error_and_exit(exctype=Exception)
 def summary(ctx: click.Context):
-    _any_source_impl(ctx, 'summary')
+    any_source_impl(ctx, 'summary')
 
 
 summary.add_command(command_create)