Add feature to upload Docker images to AWS [WIP]

mlcube/mlcube/__main__.py

@@ -49,7 +49,7 @@ def parser_process(value: t.Text, state: click.parser.ParsingState):
 
 
 def _parse_cli_args(ctx: t.Optional[click.core.Context], mlcube: t.Text, platform: t.Optional[t.Text],
-                    workspace: t.Optional[t.Text],
+                    workspace: t.Optional[t.Text], aws: t.Optional[t.Text], 
                     resolve: bool) -> t.Tuple[t.Optional[t.Type[Runner]], DictConfig]:
     """
     Args:
@@ -74,7 +74,7 @@ def _parse_cli_args(ctx: t.Optional[click.core.Context], mlcube: t.Text, platfor
     else:
         runner_cls, runner_config = None, None
     mlcube_config = MLCubeConfig.create_mlcube_config(
-        os.path.join(mlcube_inst.path, mlcube_inst.file), mlcube_cli_args, task_cli_args, runner_config, workspace,
+        os.path.join(mlcube_inst.path, mlcube_inst.file), mlcube_cli_args, task_cli_args, runner_config, workspace, aws,
         resolve=resolve, runner_cls=runner_cls
     )
     return runner_cls, mlcube_config
@@ -103,6 +103,10 @@ def _parse_cli_args(ctx: t.Optional[click.core.Context], mlcube: t.Text, platfor
     '--workspace', required=False, type=str, default=None,
     help="Workspace location that is used to store input/output artifacts of MLCube tasks."
 )
+aws_option = click.option(
+    '--aws', required=False, type=str, default=os.path.join(os.getcwd(), 'aws_credentials.json'),
+    help="Aws config file location that contains credentials."
+)
 
 
 @click.group(name='mlcube', help="MLCube 📦 is a packaging tool for ML models")
@@ -132,7 +136,7 @@ def show_config(ctx: click.core.Context, mlcube: t.Text, platform: t.Text, works
         workspace: Workspace path to use. If not specified, default workspace inside MLCube directory is used.
         resolve: if True, compute values in MLCube configuration.
     """
-    _, mlcube_config = _parse_cli_args(ctx, mlcube, platform, workspace, resolve)
+    _, mlcube_config = _parse_cli_args(ctx, mlcube, platform, workspace, resolve, aws=None)
     print(OmegaConf.to_yaml(mlcube_config))
 
 
@@ -148,11 +152,29 @@ def configure(ctx: click.core.Context, mlcube: t.Text, platform: t.Text) -> None
         mlcube: Path to MLCube root directory or mlcube.yaml file.
         platform: Platform to use to configure this MLCube for (docker, singularity, gcp, k8s etc).
     """
-    runner_cls, mlcube_config = _parse_cli_args(ctx, mlcube, platform, workspace=None, resolve=True)
+    runner_cls, mlcube_config = _parse_cli_args(ctx, mlcube, platform, workspace=None, aws=None, resolve=True)
     docker_runner = runner_cls(mlcube_config, task=None)
     docker_runner.configure()
 
 
+@cli.command(name='upload', help='upload MLCube image.',
+             context_settings=dict(ignore_unknown_options=True, allow_extra_args=True))
+@mlcube_option
+@platform_option
+@aws_option
+@click.pass_context
+def upload(ctx: click.core.Context, mlcube: t.Text, platform: t.Text, aws: t.Text) -> None:
+    """
+    Args:
+        ctx: Click context. We need this to get access to extra CLI arguments.
+        mlcube: Path to MLCube root directory or mlcube.yaml file.
+        platform: Platform to use to configure this MLCube for (docker, singularity, gcp, k8s etc).
+    """
+    runner_cls, mlcube_config = _parse_cli_args(ctx, mlcube, platform, workspace=None, aws=aws, resolve=True)
+    docker_runner = runner_cls(mlcube_config, task=None)
+    docker_runner.upload()
+
+
 @cli.command(name='run', help='Run MLCube ML task.',
              context_settings=dict(ignore_unknown_options=True, allow_extra_args=True))
 @mlcube_option
@@ -169,7 +191,7 @@ def run(ctx: click.core.Context, mlcube: t.Text, platform: t.Text, task: t.Text,
         task: Comma separated list of tasks to run.
         workspace: Workspace path to use. If not specified, default workspace inside MLCube directory is used.
     """
-    runner_cls, mlcube_config = _parse_cli_args(ctx, mlcube, platform, workspace, resolve=True)
+    runner_cls, mlcube_config = _parse_cli_args(ctx, mlcube, platform, workspace, aws=None, resolve=True)
     mlcube_tasks: t.List[str] = list((mlcube_config.get('tasks', None) or {}).keys())  # Tasks in this MLCube.
     tasks: t.List[str] = CliParser.parse_list_arg(task, default=None)                  # Requested tasks.
 
@@ -197,7 +219,7 @@ def run(ctx: click.core.Context, mlcube: t.Text, platform: t.Text, task: t.Text,
 @cli.command(name='describe', help='Describe MLCube.')
 @mlcube_option
 def describe(mlcube: t.Text) -> None:
-    _, mlcube_config = _parse_cli_args(None, mlcube, None, None, resolve=True)
+    _, mlcube_config = _parse_cli_args(None, mlcube, None, None, None, resolve=True)
     print(f"MLCube")
     print(f"  path = {mlcube_config.runtime.root}")
     print(f"  name = {mlcube_config.name}:{mlcube_config.get('version', 'latest')}")

mlcube/mlcube/config.py

@@ -48,7 +48,7 @@ def get_uri(value: t.Text) -> t.Text:
     @staticmethod
     def create_mlcube_config(mlcube_config_file: t.Text, mlcube_cli_args: t.Optional[DictConfig] = None,
                              task_cli_args: t.Optional[t.Dict] = None, runner_config: t.Optional[DictConfig] = None,
-                             workspace: t.Optional[t.Text] = None, resolve: bool = True,
+                             workspace: t.Optional[t.Text] = None, aws: t.Optional[t.Text] = None, resolve: bool = True,
                              runner_cls: t.Optional[t.Type[Runner]] = None) -> DictConfig:
         """ Create MLCube mlcube merging different configs - base, global, local and cli.
         Args:
@@ -91,6 +91,7 @@ def create_mlcube_config(mlcube_config_file: t.Text, mlcube_cli_args: t.Optional
         # located inside workspace (internal or custom), users are encouraged not to use ${runtime.workspace} or
         # ${workspace} in their MLCube configuration files.
         mlcube_config['workspace'] = actual_workspace
+        mlcube_config['aws'] = aws
         # Merge, for instance, docker runner config from system settings with docker config from MLCube config.
         if runner_cls:
             runner_cls.CONFIG.merge(mlcube_config)

mlcube/mlcube/runner.py

@@ -50,5 +50,8 @@ def __init__(self, mlcube: t.Union[DictConfig, t.Dict], task: t.Optional[t.Text]
     def configure(self) -> None:
         ...
 
+    def upload(self) -> None:
+        ...
+
     def run(self) -> None:
         ...