🚑 Add warning for new download error to script

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "zod"
-version = "0.2.9"
+version = "0.2.10"
 description = "Zenseact Open Dataset"
 authors = ["Zenseact <opendataset@zenseact.com>"]
 license = "MIT"

zod/cli/download_zod.py → zod/cli/download.py

@@ -84,6 +84,7 @@ class DownloadSettings:
     extract: bool
     extract_already_downloaded: bool
     parallel: bool
+    max_workers: int = 8
 
     def __str__(self):
         return "\n".join([f"    {key}: {value}" for key, value in self.__dict__.items()])
@@ -253,6 +254,12 @@ def _filter_entry(entry: dropbox.files.Metadata, settings: FilterSettings) -> bo
 def _download_dataset(dl_settings: DownloadSettings, filter_settings: FilterSettings, dirname: str):
     dbx = ResumableDropbox(app_key=APP_KEY, oauth2_refresh_token=REFRESH_TOKEN, timeout=TIMEOUT)
     url, entries = _list_folder(dl_settings.url, dbx, dirname)
+    if not entries:
+        typer.echo(
+            "Warning! No files found. Check the url, but this could be a known dropbox error.\n"
+            "We are working on a fix, so please try again layer. Sorry for the inconvenience."
+        )
+        return
     dl_dir = osp.join(dl_settings.output_dir, "downloads", dirname)
     files_to_download = [
         DownloadExtractInfo(
@@ -277,14 +284,14 @@ def _download_dataset(dl_settings: DownloadSettings, filter_settings: FilterSett
         os.makedirs(dl_settings.output_dir, exist_ok=True)
         os.makedirs(dl_dir, exist_ok=True)
     if dl_settings.parallel:
-        with ThreadPoolExecutor() as pool:
+        with ThreadPoolExecutor(max_workers=dl_settings.max_workers) as pool:
             pool.map(lambda info: _download_and_extract(dbx, info), files_to_download)
     else:
         for info in files_to_download:
             _download_and_extract(dbx, info)
 
 
-def _list_folder(url, dbx, path):
+def _list_folder(url, dbx: ResumableDropbox, path: str):
     shared_link = dropbox.files.SharedLink(url=url)
     try:
         res = dbx.files_list_folder(path=f"/{path}", shared_link=shared_link)
@@ -367,6 +374,9 @@ def download(
         False, help="Extract already downloaded archives", rich_help_panel=GEN
     ),
     parallel: bool = typer.Option(True, help="Download files in parallel", rich_help_panel=GEN),
+    max_workers: int = typer.Option(
+        8, help="Max number of workers for parallel downloads", rich_help_panel=GEN
+    ),
     no_confirm: bool = typer.Option(
         False,
         "-y",
@@ -401,6 +411,7 @@ def download(
         extract=extract,
         extract_already_downloaded=extract_already_downloaded,
         parallel=parallel,
+        max_workers=max_workers,
     )
     filter_settings = FilterSettings(
         version=version,

zod/cli/main.py

@@ -1,13 +1,10 @@
-from pathlib import Path
-from typing import Tuple
-
 try:
     import typer
 except ImportError:
     print('Warning! zod is installed without the CLI dependencies:\npip install "zod[cli]"')
     exit(1)
 
-from zod.cli.download_zod import app as download_app
+from zod.cli.download import app as download_app
 from zod.cli.extract_tsr_patches import cli_dummy as tsr_dummy
 from zod.cli.generate_coco_json import convert_to_coco
 from zod.cli.visualize_lidar import app as visualize_lidar_app

zod/cli/utils.py

@@ -17,11 +17,11 @@ def folder(self) -> str:
 
     @property
     def dataset_cls(self):
-        if self == SubDataset.FRAMES:
-            return ZodFrames
-        elif self == SubDataset.SEQUENCES:
-            return ZodSequences
-        elif self == SubDataset.DRIVES:
-            return ZodDrives
-        else:
-            raise ValueError(f"Unknown subdataset: {self}")
+        return _cls_map[self]
+
+
+_cls_map = {
+    SubDataset.FRAMES: ZodFrames,
+    SubDataset.SEQUENCES: ZodSequences,
+    SubDataset.DRIVES: ZodDrives,
+}