From 669c5b842ea76ead97620f7dbf0ecbaa404934c6 Mon Sep 17 00:00:00 2001 From: Colin Rogers <111200756+colin-rogers-dbt@users.noreply.github.com> Date: Wed, 31 Jul 2024 08:53:48 -0700 Subject: [PATCH 1/5] Update setup.py to allow for dbt-adapters ~1.3 (#748) Co-authored-by: Kshitij Aranke Co-authored-by: Ben Cassell --- dbt/adapters/databricks/impl.py | 8 ++++++-- requirements.txt | 6 +++--- setup.py | 4 ++-- 3 files changed, 11 insertions(+), 7 deletions(-) diff --git a/dbt/adapters/databricks/impl.py b/dbt/adapters/databricks/impl.py index 4c9ba36e..2f4f4a87 100644 --- a/dbt/adapters/databricks/impl.py +++ b/dbt/adapters/databricks/impl.py @@ -172,14 +172,18 @@ def acquire_connection( # override @contextmanager - def connection_named(self, name: str, query_header_context: Any = None) -> Iterator[None]: + def connection_named( + self, name: str, query_header_context: Any = None, should_release_connection: bool = True + ) -> Iterator[None]: try: if self.connections.query_header is not None: self.connections.query_header.set(name, query_header_context) self.acquire_connection(name, query_header_context) yield finally: - self.release_connection() + if should_release_connection: + self.release_connection() + if self.connections.query_header is not None: self.connections.query_header.reset() diff --git a/requirements.txt b/requirements.txt index 900a4cdf..4398f0d6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,8 +1,8 @@ databricks-sql-connector>=3.1.0, <3.2.0 dbt-spark~=1.8.0 -dbt-core~=1.8.0 -dbt-adapters~=1.2.0 +dbt-core>=1.8.0, <2.0 +dbt-adapters>=1.3.0, <2.0 databricks-sdk==0.17.0 keyring>=23.13.0 pandas<2.2.0 -protobuf<5.0.0 \ No newline at end of file +protobuf<5.0.0 diff --git a/setup.py b/setup.py index 98ea497b..975db05d 100644 --- a/setup.py +++ b/setup.py @@ -55,8 +55,8 @@ def _get_plugin_version() -> str: include_package_data=True, install_requires=[ "dbt-spark>=1.8.0, <2.0", - "dbt-core~=1.8.0", - "dbt-adapters~=1.2.0", + "dbt-core>=1.8.0, <2.0", + "dbt-adapters>=1.3.0, <2.0", "databricks-sql-connector>=3.1.0, <3.2.0", "databricks-sdk==0.17.0", "keyring>=23.13.0", From 38aefbf86371341f09f39b2ae93224ba2f21270d Mon Sep 17 00:00:00 2001 From: Juri Krainjukov Date: Wed, 31 Jul 2024 22:51:51 +0300 Subject: [PATCH 2/5] fix: race condition on cluster creation (#751) Signed-off-by: Juri Krainjukov --- CHANGELOG.md | 1 + dbt/adapters/databricks/python_submissions.py | 4 +++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cc84e06c..350be897 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ - Always use lower case when gathering metadata (since objects are stored internally as lower case regardless of how we create them) ([742](https://github.com/databricks/dbt-databricks/pull/742)) - Persist table comments for python models ([743](https://github.com/databricks/dbt-databricks/pull/743)) - Stop cursor destructor warnings ([744](https://github.com/databricks/dbt-databricks/pull/744)) +- Race condition on cluster creation. (thanks @jurasan!) ([751](https://github.com/databricks/dbt-databricks/pull/751)) ## dbt-databricks 1.8.4 (July 17, 2024) diff --git a/dbt/adapters/databricks/python_submissions.py b/dbt/adapters/databricks/python_submissions.py index 56bd290c..b725191e 100644 --- a/dbt/adapters/databricks/python_submissions.py +++ b/dbt/adapters/databricks/python_submissions.py @@ -395,7 +395,9 @@ def start_cluster(self) -> None: json={"cluster_id": self.cluster_id}, ) if response.status_code != 200: - raise DbtRuntimeError(f"Error starting terminated cluster.\n {response.content!r}") + current_status = self.get_cluster_status().get("state", "").upper() + if current_status not in ["RUNNING", "PENDING"]: + raise DbtRuntimeError(f"Error starting terminated cluster.\n {response.content!r}") self._wait_for_cluster_to_start() From 1b89b97daa67c863da34e4a3ed40fe2e0e71b97c Mon Sep 17 00:00:00 2001 From: Ben Cassell <98852248+benc-db@users.noreply.github.com> Date: Wed, 31 Jul 2024 15:43:58 -0700 Subject: [PATCH 3/5] Allow auth type to be configured for tests (#754) --- tests/profiles.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/profiles.py b/tests/profiles.py index 41768861..940d2e2a 100644 --- a/tests/profiles.py +++ b/tests/profiles.py @@ -31,7 +31,7 @@ def _build_databricks_cluster_target( "connect_retries": 3, "connect_timeout": 5, "retry_all": True, - "auth_type": "oauth", + "auth_type": os.getenv("DBT_DATABRICKS_AUTH_TYPE", "oauth"), } if catalog is not None: profile["catalog"] = catalog From b1eb5e2ece869cd938a06a846389b314d3d5ef07 Mon Sep 17 00:00:00 2001 From: Ben Cassell <98852248+benc-db@users.noreply.github.com> Date: Thu, 1 Aug 2024 09:56:25 -0700 Subject: [PATCH 4/5] Allow configuring port for tests (#755) --- tests/profiles.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/profiles.py b/tests/profiles.py index 940d2e2a..2c80af44 100644 --- a/tests/profiles.py +++ b/tests/profiles.py @@ -39,6 +39,8 @@ def _build_databricks_cluster_target( profile["schema"] = schema if session_properties is not None: profile["session_properties"] = session_properties + if os.getenv("DBT_DATABRICKS_PORT"): + profile["connection_parameters"] = {"_port": os.getenv("DBT_DATABRICKS_PORT")} return profile From 25807a8376e4cafa0e73c89be15be93052fa66af Mon Sep 17 00:00:00 2001 From: yunbodeng-db <104732431+yunbodeng-db@users.noreply.github.com> Date: Tue, 6 Aug 2024 15:18:52 -0700 Subject: [PATCH 5/5] Prep for 1.8.5 release (#757) --- CHANGELOG.md | 2 +- dbt/adapters/databricks/__version__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 350be897..68423f4c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,4 @@ -## dbt-databricks next +## dbt-databricks 1.8.5 (August 6, 2024) ### Fixes diff --git a/dbt/adapters/databricks/__version__.py b/dbt/adapters/databricks/__version__.py index dc09b9b7..4db8df56 100644 --- a/dbt/adapters/databricks/__version__.py +++ b/dbt/adapters/databricks/__version__.py @@ -1 +1 @@ -version: str = "1.8.4" +version: str = "1.8.5"