Merge branch 'dev'

.github/workflows/fetch-bi-data.yaml

@@ -1,5 +1,5 @@
 name:
-  Extract data with CS Tools.
+  Extract BI Server with CS Tools.
 
 on:
   workflow_dispatch:
@@ -26,7 +26,7 @@ jobs:
         run: echo "days_ago_1=$(date -d "-1 days" +'%Y-%m-%d')" >> $GITHUB_ENV
 
       - name: Check out the repository main branch
-      - uses: actions/checkout@v4
+        uses: actions/checkout@v4
 
       - name: Set up Python 3.12
         uses: actions/setup-python@v4

.github/workflows/fetch-metdata.yaml

@@ -1,5 +1,5 @@
 name:
-  Extract data with CS Tools.
+  Extract Metadata with CS Tools.
 
 on:
   workflow_dispatch:
@@ -19,7 +19,8 @@ jobs:
 
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4
+      - name: Check out the repository main branch
+        uses: actions/checkout@v4
 
       - name: Set up Python 3.12
         uses: actions/setup-python@v4
@@ -31,4 +32,4 @@ jobs:
 
       # --config ENV:   tells CS Tools to pull the information from environment variables.
       - name: Refresh Metadata from ThoughtSpot
-        run: "cs_tools tools seachable metadata --syncer ${{ secrets.SYNCER_DECLARATIVE_STRING }} --config ENV:"
+        run: "cs_tools tools searchable metadata --syncer ${{ secrets.SYNCER_DECLARATIVE_STRING }} --config ENV:"

.github/workflows/test-bootstrapper.yaml

@@ -1,10 +1,10 @@
 name: Test CS Tools Bootstrapper
 
 on:
+  workflow_dispatch:
   push:
     branches:
       - master
-      - dev
 
 jobs:
  test-bootstrapper:

.gitignore

@@ -44,6 +44,9 @@ venv.bak/
 # ignore coverage file
 .coverage
 
+# ignore linting
+.mypy_cache/
+
 # generated documentation
 docs/terminal-screenshots/*.svg
 

cs_tools/__project__.py

@@ -1,4 +1,4 @@
-__version__ = "1.5.1"
+__version__ = "1.5.2"
 __docs__ = "https://thoughtspot.github.io/cs_tools/"
 __repo__ = "https://github.com/thoughtspot/cs_tools"
 __help__ = f"{__repo__}/discussions/"

cs_tools/api/_utils.py

@@ -34,7 +34,7 @@ def is_valid_guid(to_test: str) -> bool:
     return str(guid) == to_test
 
 
-def scrub_undefined_sentinel(inp: Any, *, null: Union[UNDEFINED, None]) -> Any:
+def scrub_undefined_sentinel(inp: Any, *, null: Union[type[UNDEFINED], None]) -> Any:
     """
     Remove sentinel values from input parameters.
 
@@ -59,7 +59,7 @@ def obfuscate_sensitive_data(request_query: httpx.QueryParams) -> dict[str, Any]
     httpx.QueryParams.items() returns only the first specified parameter. If the user
     specifies the parameter multiple times, we'd have to switch to .multi_items().
     """
-    SAFEWORDS = ("password", "access_token")
+    SAFEWORDS = ("auth_token", "secret_key", "password", "access_token")
 
     # don't modify the actual keywords we want to build into the request
     secure = copy.deepcopy({k: v for k, v in request_query.items() if k not in ("file", "files")})

cs_tools/api/middlewares/logical_table.py

@@ -3,7 +3,6 @@
 from typing import TYPE_CHECKING, Optional, Union
 import logging
 
-from cs_tools import utils
 from cs_tools.api import _utils
 from cs_tools.errors import ContentDoesNotExist
 from cs_tools.types import GUID, MetadataCategory, TableRowsFormat
@@ -75,6 +74,12 @@ def all(  # noqa: A003
             if exclude_system_content:
                 to_extend = [table for table in to_extend if table.get("authorName") not in _utils.SYSTEM_USERS]
 
+            # Fake the .type for Models.
+            to_extend = [
+                {**table, "type": "MODEL" if table.get("worksheetVersion") == "V2" else table["type"]}
+                for table in to_extend
+            ]
+
             tables.extend([{"metadata_type": "LOGICAL_TABLE", **table} for table in to_extend])
 
             if not tables and raise_on_error:
@@ -98,7 +103,9 @@ def all(  # noqa: A003
         if include_data_source:
             for table in tables:
                 connection_guid = self.ts.metadata.find_data_source_of_logical_table(guid=table["id"])
-                source_details = self.ts.metadata.fetch_data_source_info(guid=connection_guid)
+                source_details = self.ts.metadata.fetch_header_and_extras(
+                    metadata_type="DATA_SOURCE", guid=connection_guid
+                )
                 table["data_source"] = source_details["header"]
                 table["data_source"]["type"] = source_details["type"]
 
@@ -108,8 +115,9 @@ def columns(self, guids: list[GUID], *, include_hidden: bool = False, chunksize:
         """ """
         columns = []
 
-        for chunk in utils.batched(guids, n=chunksize):
-            r = self.ts.api.v1.metadata_details(guids=chunk, show_hidden=include_hidden)
+        # for chunk in utils.batched(guids, n=chunksize):
+        for guid in guids:
+            r = self.ts.metadata.fetch_header_and_extras(metadata_type="LOGICAL_TABLE", guid=guid)
 
             for logical_table in r.json()["storables"]:
                 for column in logical_table.get("columns", []):

cs_tools/api/middlewares/metadata.py

@@ -1,14 +1,13 @@
 from __future__ import annotations
 
-from typing import TYPE_CHECKING, Optional, Union
+from typing import TYPE_CHECKING, Optional
 import functools as ft
 import logging
 
 from cs_tools import utils
 from cs_tools.errors import CSToolsError
 from cs_tools.types import (
     GUID,
-    MetadataObjectSubtype,
     MetadataObjectType,
     MetadataParent,
     PermissionType,
@@ -32,31 +31,18 @@ def permissions(
         self,
         guids: list[GUID],
         *,
-        type: Union[MetadataObjectType, MetadataObjectSubtype],  # noqa: A002
+        metadata_type: MetadataObjectType,
         permission_type: PermissionType = PermissionType.explicit,
         chunksize: int = 25,
     ) -> TableRowsFormat:
-        """ """
-        type_to_supertype = {
-            "FORMULA": "LOGICAL_COLUMN",
-            "CALENDAR_TABLE": "LOGICAL_COLUMN",
-            "LOGICAL_COLUMN": "LOGICAL_COLUMN",
-            "QUESTION_ANSWER_BOOK": "QUESTION_ANSWER_BOOK",
-            "PINBOARD_ANSWER_BOOK": "PINBOARD_ANSWER_BOOK",
-            "ONE_TO_ONE_LOGICAL": "LOGICAL_TABLE",
-            "USER_DEFINED": "LOGICAL_TABLE",
-            "WORKSHEET": "LOGICAL_TABLE",
-            "AGGR_WORKSHEET": "LOGICAL_TABLE",
-            "MATERIALIZED_VIEW": "LOGICAL_TABLE",
-            "SQL_VIEW": "LOGICAL_TABLE",
-            "LOGICAL_TABLE": "LOGICAL_TABLE",
-        }
-
+        """
+        Fetch permissions for a given object type.
+        """
         sharing_access = []
         group_guids = [group["id"] for group in self.ts.group.all()]
 
         for chunk in utils.batched(guids, n=chunksize):
-            r = self.ts.api.v1.security_metadata_permissions(metadata_type=type_to_supertype[type], guids=chunk)
+            r = self.ts.api.v1.security_metadata_permissions(metadata_type=metadata_type, guids=chunk)
 
             for data in r.json().values():
                 for shared_to_principal_guid, permission in data["permissions"].items():
@@ -233,23 +219,36 @@ def objects_exist(self, metadata_type: MetadataObjectType, guids: list[GUID]) ->
         existence = {header["id"] for header in r.json()["headers"]}
         return {guid: guid in existence for guid in guids}
 
-    @ft.lru_cache(maxsize=1000)  # noqa: B019
-    def fetch_data_source_info(self, guid: GUID) -> GUID:
+    @ft.cache  # noqa: B019
+    def fetch_header_and_extras(self, metadata_type: MetadataObjectType, guid: GUID) -> dict:
         """
         METADATA DETAILS is expensive. Here's our shortcut.
         """
-        r = self.ts.api.v1.metadata_details(metadata_type="DATA_SOURCE", guids=[guid], show_hidden=True)
-        return r.json()["storables"][0]
+        r = self.ts.api.v1.metadata_details(metadata_type=metadata_type, guids=[guid], show_hidden=True)
+
+        d = r.json()["storables"][0]
+
+        header_and_extras = {
+            "metadata_type": metadata_type,
+            "header": d["header"],
+            "type": d.get("type"),  # READ: .subtype  (eg. ONE_TO_ONE_LOGICAL, WORKSHEET, etc..)
+            # LOGICAL_TABLE extras
+            "dataSourceId": d.get("dataSourceId"),
+            "columns": d.get("columns"),
+            # VIZ extras (answer, liveboard)
+            "reportContent": d.get("reportContent"),
+        }
+
+        return header_and_extras
 
-    @ft.lru_cache(maxsize=1000)  # noqa: B019
     def find_data_source_of_logical_table(self, guid: GUID) -> GUID:
         """
         METADATA DETAILS is expensive. Here's our shortcut.
         """
-        r = self.ts.api.v1.metadata_details(metadata_type="LOGICAL_TABLE", guids=[guid], show_hidden=True)
-        storable = r.json()["storables"][0]
-        return storable["dataSourceId"]
+        info = self.fetch_header_and_extras(metadata_type="LOGICAL_TABLE", guid=guid)
+        return info["dataSourceId"]
 
+    @ft.cache  # noqa: B019
     def table_references(self, guid: GUID, *, tml_type: str, hidden: bool = False) -> list[MetadataParent]:
         """
         Returns a mapping of parent LOGICAL_TABLEs
@@ -265,7 +264,7 @@ def table_references(self, guid: GUID, *, tml_type: str, hidden: bool = False) -
 
         """
         metadata_type = TMLSupportedContent.from_friendly_type(tml_type)
-        r = self.ts.api.v1.metadata_details(metadata_type=metadata_type, guids=[guid], show_hidden=hidden)
+        r = self.fetch_header_and_extras(metadata_type=metadata_type, guids=guid, show_hidden=hidden)
         mappings: list[MetadataParent] = []
 
         if "storables" not in r.json():

cs_tools/cli/commands/config.py

@@ -40,7 +40,9 @@ def create(
     config: str = typer.Option(..., help="config file identifier", metavar="NAME"),
     url: str = typer.Option(..., help="your thoughtspot url or IP"),
     username: str = typer.Option(..., help="username when logging into ThoughtSpot"),
-    password: str = typer.Option(None, help="the password you type when using the ThoughtSpot login screen"),
+    password: str = typer.Option(
+        None, help="the password you type on the ThoughtSpot login screen, use [b magenta]prompt[/] to type it hidden"
+    ),
     secret: str = typer.Option(None, help="the trusted authentication secret key, found in the developer tab"),
     token: str = typer.Option(None, help="the V2 API bearer token"),
     default_org: int = typer.Option(None, help="org ID to sign into by default"),
@@ -50,22 +52,26 @@ def create(
     disable_ssl: bool = typer.Option(
         False, "--disable-ssl", help="whether or not to turn off checking the SSL certificate"
     ),
+    proxy: str = typer.Option(None, help="proxy server to use to connect to ThoughtSpot"),
     default: bool = typer.Option(False, "--default", help="whether or not to make this the default configuration"),
     verbose: bool = typer.Option(False, "--verbose", "-v", help="enable verbose logging"),
 ):
     """
     Create a new config file.
     """
 
+    if not any((password, secret, token)):
+        log.error("You must specify at least one authentication method (--password, --secret, or --token)")
+        raise typer.Exit()
+
     if CSToolsConfig.exists(name=config):
         log.warning(f'[b yellow]Configuration file "{config}" already exists.')
 
         if not Confirm.ask("\nDo you want to overwrite it?", console=rich_console):
             raise typer.Abort()
 
-    if all(safe is None for safe in (password, secret, token)):
-        log.error("You must specify at least one authentication method")
-        return -1
+    if password == "prompt":
+        password = rich_console.input("\nType your password [b yellow](your input is hidden)\n", password=True)
 
     data = {
         "name": config,
@@ -77,6 +83,7 @@ def create(
             "bearer_token": token,
             "default_org": default_org,
             "disable_ssl": disable_ssl,
+            "proxy": proxy,
         },
         "verbose": verbose,
         "temp_dir": temp_dir or cs_tools_venv.tmp_dir,
@@ -112,13 +119,19 @@ def modify(
     config: str = typer.Option(None, help="config file identifier", metavar="NAME"),
     url: str = typer.Option(None, help="your thoughtspot server"),
     username: str = typer.Option(None, help="username when logging into ThoughtSpot"),
-    password: str = typer.Option(None, help="the password you type when using the ThoughtSpot login screen"),
+    password: str = typer.Option(
+        None, help="the password you type on the ThoughtSpot login screen, use [b magenta]prompt[/] to type it hidden"
+    ),
     secret: str = typer.Option(None, help="the trusted authentication secret key"),
     token: str = typer.Option(None, help="the V2 API bearer token"),
+    temp_dir: pathlib.Path = typer.Option(
+        None, help="the temporary directory to use for uploading files", click_type=Directory()
+    ),
     disable_ssl: bool = typer.Option(
         None, "--disable-ssl", help="whether or not to turn off checking the SSL certificate"
     ),
     default_org: int = typer.Option(None, help="org ID to sign into by default"),
+    proxy: str = typer.Option(None, help="proxy server to use to connect to ThoughtSpot"),
     default: bool = typer.Option(
         None,
         "--default / --remove-default",
@@ -140,7 +153,7 @@ def modify(
         data["thoughtspot"]["default_org"] = default_org
 
     if password == "prompt":
-        password = rich_console.input("[b yellow]Type your password (your input is hidden)\n", password=True)
+        password = rich_console.input("\nType your password [b yellow](your input is hidden)\n", password=True)
 
     if password is not None:
         data["thoughtspot"]["password"] = password
@@ -151,9 +164,15 @@ def modify(
     if token is not None:
         data["thoughtspot"]["bearer_token"] = token
 
+    if temp_dir is not None:
+        data["temp_dir"] = temp_dir
+
     if disable_ssl is not None:
         data["thoughtspot"]["disable_ssl"] = disable_ssl
 
+    if proxy is not None:
+        data["thoughtspot"]["proxy"] = proxy
+
     if default is not None:
         meta.default_config_name = config
         meta.save()

cs_tools/cli/commands/main.py

@@ -172,7 +172,7 @@ def run() -> int:
     this_run = _analytics.CommandExecution.validated_init(**this_run_data, context=app.info.context_settings["obj"])
 
     # Add the analytics to the local database
-    if not (CURRENT_RUNTIME.is_ci or CURRENT_RUNTIME.is_dev):
+    if not CURRENT_RUNTIME.is_dev:
         try:
             with db.begin() as transaction:
                 stmt = sa.insert(_analytics.CommandExecution).values([this_run.model_dump()])

cs_tools/cli/commands/self.py

@@ -28,6 +28,7 @@
 
     {meta.newer_version_string()}
     """,
+    no_args_is_help=True,
     invoke_without_command=True,
 )
 

cs_tools/cli/commands/tools.py

@@ -18,6 +18,7 @@
     which aren't native to ThoughtSpot or advanced functionality for
     clients who have a well-adopted platform.
     """,
+    no_args_is_help=True,
     subcommand_metavar="<tool>",
     invoke_without_command=True,
     epilog=(

cs_tools/cli/dependencies/thoughtspot.py

@@ -115,7 +115,8 @@ def __enter__(self):
         command_params = [p.name for p in ctx.command.params]
         overrides = {k: ctx.params.pop(k) for k in ctx.params.copy() if k not in command_params}
 
-        log.debug(f"Command Overrides: {' '.join(overrides)}")
+        if overrides:
+            log.debug(f"Command Overrides: {' '.join(overrides)}")
 
         cfg = CSToolsConfig.from_name(config, **overrides, automigrate=True)
 
@@ -128,7 +129,7 @@ def __enter__(self):
 
     def _send_analytics_in_background(self) -> None:
         """Send analyics in the background."""
-        if meta.analytics.is_opted_in is not True or meta.environment.is_dev:
+        if meta.analytics.is_opted_in or meta.environment.is_dev or meta.environment.is_ci:
             return
 
         # AVOID CIRCULAR IMPORTS WITH cli.ux