From b9930c7813e06863802af8ffc80f41374e2b84e5 Mon Sep 17 00:00:00 2001 From: Drew Banin Date: Wed, 16 Aug 2017 16:52:04 -0400 Subject: [PATCH 1/9] make it possible to issue hooks outside of a transaction --- dbt/adapters/default.py | 10 +++++++--- dbt/adapters/postgres.py | 4 ++-- dbt/adapters/redshift.py | 4 ++-- dbt/adapters/snowflake.py | 6 +++--- dbt/context/common.py | 5 +++++ dbt/contracts/graph/parsed.py | 2 ++ dbt/include/global_project/macros/core.sql | 4 ++-- .../global_project/macros/materializations/helpers.sql | 4 ++-- .../macros/materializations/incremental.sql | 3 ++- .../global_project/macros/materializations/table.sql | 3 +++ .../global_project/macros/materializations/view.sql | 3 +++ dbt/model.py | 4 ++-- dbt/utils.py | 2 ++ 13 files changed, 37 insertions(+), 17 deletions(-) diff --git a/dbt/adapters/default.py b/dbt/adapters/default.py index c21e1158cd5..6ccdfcf842b 100644 --- a/dbt/adapters/default.py +++ b/dbt/adapters/default.py @@ -396,6 +396,10 @@ def reload(cls, connection): def add_begin_query(cls, profile, name): return cls.add_query(profile, 'BEGIN', name, auto_begin=False) + @classmethod + def add_commit_query(cls, profile, name): + return cls.add_query(profile, 'COMMIT', name, auto_begin=False) + @classmethod def begin(cls, profile, name='master'): global connections_in_use @@ -428,10 +432,10 @@ def commit_if_has_connection(cls, profile, name): connection = cls.get_connection(profile, name, False) - return cls.commit(connection) + return cls.commit(profile, connection) @classmethod - def commit(cls, connection): + def commit(cls, profile, connection): global connections_in_use if dbt.flags.STRICT_MODE: @@ -445,7 +449,7 @@ def commit(cls, connection): 'it does not have one open!'.format(connection.get('name'))) logger.debug('On {}: COMMIT'.format(connection.get('name'))) - connection.get('handle').commit() + cls.add_commit_query(profile, connection.get('name')) connection['transaction_open'] = False connections_in_use[connection.get('name')] = connection diff --git a/dbt/adapters/postgres.py b/dbt/adapters/postgres.py index 04491a88c16..cb85b1a80e8 100644 --- a/dbt/adapters/postgres.py +++ b/dbt/adapters/postgres.py @@ -23,14 +23,14 @@ def exception_handler(cls, profile, sql, model_name=None, except psycopg2.DatabaseError as e: logger.debug('Postgres error: {}'.format(str(e))) - cls.rollback(connection) + cls.release_connection(profile, connection_name) raise dbt.exceptions.DatabaseException( dbt.compat.to_string(e).strip()) except Exception as e: logger.debug("Error running SQL: %s", sql) logger.debug("Rolling back transaction.") - cls.rollback(connection) + cls.release_connection(profile, connection_name) raise dbt.exceptions.RuntimeException(e) @classmethod diff --git a/dbt/adapters/redshift.py b/dbt/adapters/redshift.py index 5430873d5f4..2768f565e45 100644 --- a/dbt/adapters/redshift.py +++ b/dbt/adapters/redshift.py @@ -29,14 +29,14 @@ def drop(cls, profile, relation, relation_type, model_name=None): connection = cls.get_connection(profile, model_name) if connection.get('transaction_open'): - cls.commit(connection) + cls.commit(profile, connection) cls.begin(profile, connection.get('name')) to_return = super(PostgresAdapter, cls).drop( profile, relation, relation_type, model_name) - cls.commit(connection) + cls.commit(profile, connection) cls.begin(profile, connection.get('name')) return to_return diff --git a/dbt/adapters/snowflake.py b/dbt/adapters/snowflake.py index 0f30531a081..59f63b7139e 100644 --- a/dbt/adapters/snowflake.py +++ b/dbt/adapters/snowflake.py @@ -34,7 +34,7 @@ def exception_handler(cls, profile, sql, model_name=None, if 'Empty SQL statement' in msg: logger.debug("got empty sql statement, moving on") elif 'This session does not have a current database' in msg: - cls.rollback(connection) + cls.release_connection(profile, connection_name) raise dbt.exceptions.FailedToConnectException( ('{}\n\nThis error sometimes occurs when invalid ' 'credentials are provided, or when your default role ' @@ -42,12 +42,12 @@ def exception_handler(cls, profile, sql, model_name=None, 'Please double check your profile and try again.') .format(msg)) else: - cls.rollback(connection) + cls.release_connection(profile, connection_name) raise dbt.exceptions.DatabaseException(msg) except Exception as e: logger.debug("Error running SQL: %s", sql) logger.debug("Rolling back transaction.") - cls.rollback(connection) + cls.release_connection(profile, connection_name) raise dbt.exceptions.RuntimeException(e.msg) @classmethod diff --git a/dbt/context/common.py b/dbt/context/common.py index f6976e8e4ea..74fb43aa4de 100644 --- a/dbt/context/common.py +++ b/dbt/context/common.py @@ -251,6 +251,9 @@ def generate(model, project, flat_graph, provider=None): pre_hooks = get_hooks(model, context, 'pre-hook') post_hooks = get_hooks(model, context, 'post-hook') + pre_transaction_hooks = get_hooks(model, context, 'pre-transaction-hook') + post_transaction_hooks = get_hooks(model, context, 'post-transaction-hook') + db_wrapper = DatabaseWrapper(model, adapter, profile) context = dbt.utils.merge(context, { @@ -266,6 +269,8 @@ def generate(model, project, flat_graph, provider=None): "model": model, "post_hooks": post_hooks, "pre_hooks": pre_hooks, + "pre_transaction_hooks": pre_transaction_hooks, + "post_transaction_hooks": post_transaction_hooks, "ref": provider.ref(model, project, profile, schema, flat_graph), "schema": schema, "sql": model.get('injected_sql'), diff --git a/dbt/contracts/graph/parsed.py b/dbt/contracts/graph/parsed.py index e291fc0422c..360093466bf 100644 --- a/dbt/contracts/graph/parsed.py +++ b/dbt/contracts/graph/parsed.py @@ -18,6 +18,8 @@ Required('materialized'): basestring, Required('post-hook'): list, Required('pre-hook'): list, + Required('post-transaction-hook'): list, + Required('pre-transaction-hook'): list, Required('vars'): dict, }, extra=ALLOW_EXTRA) diff --git a/dbt/include/global_project/macros/core.sql b/dbt/include/global_project/macros/core.sql index dd33b4674f5..430ebd20f98 100644 --- a/dbt/include/global_project/macros/core.sql +++ b/dbt/include/global_project/macros/core.sql @@ -1,4 +1,4 @@ -{% macro statement(name=None, fetch_result=False) -%} +{% macro statement(name=None, fetch_result=False, auto_begin=True) -%} {%- if execute: -%} {%- set sql = render(caller()) -%} @@ -7,7 +7,7 @@ {{ write(sql) }} {%- endif -%} - {%- set _, cursor = adapter.add_query(sql) -%} + {%- set _, cursor = adapter.add_query(sql, auto_begin=auto_begin) -%} {%- if name is not none -%} {%- set result = [] if not fetch_result else adapter.get_result_from_cursor(cursor) -%} {{ store_result(name, status=adapter.get_status(cursor), data=result) }} diff --git a/dbt/include/global_project/macros/materializations/helpers.sql b/dbt/include/global_project/macros/materializations/helpers.sql index d81c1c046f3..470f52972c5 100644 --- a/dbt/include/global_project/macros/materializations/helpers.sql +++ b/dbt/include/global_project/macros/materializations/helpers.sql @@ -1,6 +1,6 @@ -{% macro run_hooks(hooks) %} +{% macro run_hooks(hooks, auto_begin=True) %} {% for hook in hooks %} - {% call statement() %} + {% call statement(auto_begin=auto_begin) %} {{ hook }}; {% endcall %} {% endfor %} diff --git a/dbt/include/global_project/macros/materializations/incremental.sql b/dbt/include/global_project/macros/materializations/incremental.sql index aa148ba3cb5..52180c67fe0 100644 --- a/dbt/include/global_project/macros/materializations/incremental.sql +++ b/dbt/include/global_project/macros/materializations/incremental.sql @@ -38,6 +38,7 @@ {{ adapter.drop(identifier, existing_type) }} {%- endif %} + {{ run_hooks(pre_transaction_hooks, auto_begin=False) }} {{ run_hooks(pre_hooks) }} -- build model @@ -80,7 +81,7 @@ {%- endif %} {{ run_hooks(post_hooks) }} - {{ adapter.commit() }} + {{ run_hooks(post_transaction_hooks, auto_begin=False) }} {%- endmaterialization %} diff --git a/dbt/include/global_project/macros/materializations/table.sql b/dbt/include/global_project/macros/materializations/table.sql index eb15b989ff3..2f677ec1eee 100644 --- a/dbt/include/global_project/macros/materializations/table.sql +++ b/dbt/include/global_project/macros/materializations/table.sql @@ -14,6 +14,7 @@ {%- endif %} {%- endif %} + {{ run_hooks(pre_transaction_hooks, auto_begin=False) }} {{ run_hooks(pre_hooks) }} -- build model @@ -51,4 +52,6 @@ {%- endif %} {{ adapter.commit() }} + + {{ run_hooks(post_transaction_hooks, auto_begin=False) }} {% endmaterialization %} diff --git a/dbt/include/global_project/macros/materializations/view.sql b/dbt/include/global_project/macros/materializations/view.sql index eeb0c181e3c..59386e26537 100644 --- a/dbt/include/global_project/macros/materializations/view.sql +++ b/dbt/include/global_project/macros/materializations/view.sql @@ -6,6 +6,7 @@ {%- set existing = adapter.query_for_existing(schema) -%} {%- set existing_type = existing.get(identifier) -%} + {{ run_hooks(pre_transaction_hooks, auto_begin=False) }} {{ run_hooks(pre_hooks) }} -- build model @@ -32,4 +33,6 @@ {{ adapter.commit() }} + {{ run_hooks(post_transaction_hooks, auto_begin=False) }} + {%- endmaterialization -%} diff --git a/dbt/model.py b/dbt/model.py index e3f8e11e206..b4a148fa16a 100644 --- a/dbt/model.py +++ b/dbt/model.py @@ -10,7 +10,7 @@ class SourceConfig(object): ConfigKeys = DBTConfigKeys - AppendListFields = ['pre-hook', 'post-hook'] + AppendListFields = ['pre-hook', 'post-hook', 'pre-transaction-hook', 'post-transaction-hook'] ExtendDictFields = ['vars'] ClobberFields = [ 'enabled', @@ -83,7 +83,7 @@ def update_in_model_config(self, config): # make sure we're not clobbering an array of hooks with a single hook # string - hook_fields = ['pre-hook', 'post-hook'] + hook_fields = ['pre-hook', 'post-hook', 'pre-transaction-hook', 'post-transaction-hook'] for hook_field in hook_fields: if hook_field in config: config[hook_field] = self.__get_hooks(config, hook_field) diff --git a/dbt/utils.py b/dbt/utils.py index c3bfd3764c3..9f9c64dc521 100644 --- a/dbt/utils.py +++ b/dbt/utils.py @@ -20,6 +20,8 @@ 'sort_type', 'pre-hook', 'post-hook', + 'pre-transaction-hook', + 'post-transaction-hook', 'vars' ] From 0e6044508def2e9901a1d964861c0b678be28617 Mon Sep 17 00:00:00 2001 From: Drew Banin Date: Wed, 16 Aug 2017 20:06:57 -0400 Subject: [PATCH 2/9] wip --- dbt/adapters/default.py | 4 +-- dbt/context/common.py | 25 ++++++++++---- dbt/contracts/graph/parsed.py | 2 -- .../macros/materializations/helpers.sql | 34 ++++++++++++++++--- .../macros/materializations/table.sql | 11 +++--- .../macros/materializations/view.sql | 8 ++--- dbt/model.py | 19 +++-------- dbt/utils.py | 2 -- 8 files changed, 66 insertions(+), 39 deletions(-) diff --git a/dbt/adapters/default.py b/dbt/adapters/default.py index 6ccdfcf842b..f9db8b021e3 100644 --- a/dbt/adapters/default.py +++ b/dbt/adapters/default.py @@ -32,13 +32,13 @@ class DefaultAdapter(object): "truncate", "add_query", "expand_target_column_types", + "quote_schema_and_table", ] raw_functions = [ "get_status", "get_result_from_cursor", "quote", - "quote_schema_and_table", ] ### @@ -580,6 +580,6 @@ def quote(cls, identifier): return '"{}"'.format(identifier) @classmethod - def quote_schema_and_table(cls, profile, schema, table): + def quote_schema_and_table(cls, profile, schema, table, model_name=None): return '{}.{}'.format(cls.quote(schema), cls.quote(table)) diff --git a/dbt/context/common.py b/dbt/context/common.py index 74fb43aa4de..664135d5916 100644 --- a/dbt/context/common.py +++ b/dbt/context/common.py @@ -17,10 +17,16 @@ def get_hooks(model, context, hook_key): hooks = model.get('config', {}).get(hook_key, []) - if isinstance(hooks, basestring): + if not isinstance(hooks, (list, tuple)): hooks = [hooks] - return hooks + wrapped_hooks = [] + for hook in hooks: + if isinstance(hook, dict): + hook = json.dumps(hook) + wrapped_hooks.append(hook) + + return wrapped_hooks class DatabaseWrapper(object): @@ -227,6 +233,15 @@ def fn(string): return fn +def fromjson(node): + def fn(string, default=None): + try: + return json.loads(string) + except ValueError as e: + return default + return fn + + def generate(model, project, flat_graph, provider=None): """ Not meant to be called directly. Call with either: @@ -251,9 +266,6 @@ def generate(model, project, flat_graph, provider=None): pre_hooks = get_hooks(model, context, 'pre-hook') post_hooks = get_hooks(model, context, 'post-hook') - pre_transaction_hooks = get_hooks(model, context, 'pre-transaction-hook') - post_transaction_hooks = get_hooks(model, context, 'post-transaction-hook') - db_wrapper = DatabaseWrapper(model, adapter, profile) context = dbt.utils.merge(context, { @@ -269,12 +281,11 @@ def generate(model, project, flat_graph, provider=None): "model": model, "post_hooks": post_hooks, "pre_hooks": pre_hooks, - "pre_transaction_hooks": pre_transaction_hooks, - "post_transaction_hooks": post_transaction_hooks, "ref": provider.ref(model, project, profile, schema, flat_graph), "schema": schema, "sql": model.get('injected_sql'), "sql_now": adapter.date_function(), + "fromjson": fromjson(model), "target": target, "this": dbt.utils.This( schema, diff --git a/dbt/contracts/graph/parsed.py b/dbt/contracts/graph/parsed.py index 360093466bf..e291fc0422c 100644 --- a/dbt/contracts/graph/parsed.py +++ b/dbt/contracts/graph/parsed.py @@ -18,8 +18,6 @@ Required('materialized'): basestring, Required('post-hook'): list, Required('pre-hook'): list, - Required('post-transaction-hook'): list, - Required('pre-transaction-hook'): list, Required('vars'): dict, }, extra=ALLOW_EXTRA) diff --git a/dbt/include/global_project/macros/materializations/helpers.sql b/dbt/include/global_project/macros/materializations/helpers.sql index 470f52972c5..2c2ca19110a 100644 --- a/dbt/include/global_project/macros/materializations/helpers.sql +++ b/dbt/include/global_project/macros/materializations/helpers.sql @@ -1,8 +1,14 @@ -{% macro run_hooks(hooks, auto_begin=True) %} +{% macro run_hooks(hooks, inside_transaction=True) %} {% for hook in hooks %} - {% call statement(auto_begin=auto_begin) %} - {{ hook }}; - {% endcall %} + {%- set hook_data = fromjson(render(hook), {}) -%} + {%- set hook_is_in_transaction = hook_data.get('transaction', True) -%}; + {%- set hook_sql = hook_data.get('sql', hook) -%}; + + {%- if hook_is_in_transaction == inside_transaction -%} + {% call statement(auto_begin=inside_transaction) %} + {{ hook_sql }} + {% endcall %} + {%- endif -%} {% endfor %} {% endmacro %} @@ -19,3 +25,23 @@ "{{ col.name }}" {{ col.data_type }} {%- if not loop.last %},{% endif %} {% endfor -%} {% endmacro %} + + +{% macro make_hook_config(sql, inside_transaction) %} + {{ {"sql": sql, "transaction": inside_transaction} | tojson }} +{% endmacro %} + + +{% macro before_begin(sql) %} + {{ make_hook_config(sql, inside_transaction=False) }} +{% endmacro %} + + +{% macro after_commit(sql) %} + {{ make_hook_config(sql, inside_transaction=False) }} +{% endmacro %} + + +{% macro vacuum(tbl) %} + {{ after_commit('vacuum ' ~ adapter.quote_schema_and_table(tbl.schema, tbl.name)) }} +{% endmacro %} diff --git a/dbt/include/global_project/macros/materializations/table.sql b/dbt/include/global_project/macros/materializations/table.sql index 2f677ec1eee..295497890ac 100644 --- a/dbt/include/global_project/macros/materializations/table.sql +++ b/dbt/include/global_project/macros/materializations/table.sql @@ -14,8 +14,10 @@ {%- endif %} {%- endif %} - {{ run_hooks(pre_transaction_hooks, auto_begin=False) }} - {{ run_hooks(pre_hooks) }} + {{ run_hooks(pre_hooks, inside_transaction=False) }} + + -- `BEGIN` happens here: + {{ run_hooks(pre_hooks, inside_transaction=True) }} -- build model {% call statement('main') -%} @@ -38,7 +40,7 @@ {%- endif -%} {%- endcall %} - {{ run_hooks(post_hooks) }} + {{ run_hooks(post_hooks, inside_transaction=True) }} -- cleanup {% if non_destructive_mode -%} @@ -51,7 +53,8 @@ {{ adapter.rename(tmp_identifier, identifier) }} {%- endif %} + -- `COMMIT` happens here {{ adapter.commit() }} - {{ run_hooks(post_transaction_hooks, auto_begin=False) }} + {{ run_hooks(post_hooks, inside_transaction=False) }} {% endmaterialization %} diff --git a/dbt/include/global_project/macros/materializations/view.sql b/dbt/include/global_project/macros/materializations/view.sql index 59386e26537..dfae7fb52a0 100644 --- a/dbt/include/global_project/macros/materializations/view.sql +++ b/dbt/include/global_project/macros/materializations/view.sql @@ -6,8 +6,8 @@ {%- set existing = adapter.query_for_existing(schema) -%} {%- set existing_type = existing.get(identifier) -%} - {{ run_hooks(pre_transaction_hooks, auto_begin=False) }} - {{ run_hooks(pre_hooks) }} + {{ run_hooks(pre_hooks | rejectattr('transaction'), auto_begin=False) }} + {{ run_hooks(pre_hooks | selectattr('transaction')) }} -- build model {% if non_destructive_mode and existing_type == 'view' -%} @@ -18,7 +18,7 @@ {%- endcall %} {%- endif %} - {{ run_hooks(post_hooks) }} + {{ run_hooks(post_hooks | selectattr('transaction')) }} -- cleanup {% if non_destructive_mode and existing_type == 'view' -%} @@ -33,6 +33,6 @@ {{ adapter.commit() }} - {{ run_hooks(post_transaction_hooks, auto_begin=False) }} + {{ run_hooks(post_hooks | rejectattr('transaction'), auto_begin=False) }} {%- endmaterialization -%} diff --git a/dbt/model.py b/dbt/model.py index b4a148fa16a..c0a509b122f 100644 --- a/dbt/model.py +++ b/dbt/model.py @@ -10,7 +10,7 @@ class SourceConfig(object): ConfigKeys = DBTConfigKeys - AppendListFields = ['pre-hook', 'post-hook', 'pre-transaction-hook', 'post-transaction-hook'] + AppendListFields = ['pre-hook', 'post-hook'] ExtendDictFields = ['vars'] ClobberFields = [ 'enabled', @@ -83,7 +83,7 @@ def update_in_model_config(self, config): # make sure we're not clobbering an array of hooks with a single hook # string - hook_fields = ['pre-hook', 'post-hook', 'pre-transaction-hook', 'post-transaction-hook'] + hook_fields = ['pre-hook', 'post-hook'] for hook_field in hook_fields: if hook_field in config: config[hook_field] = self.__get_hooks(config, hook_field) @@ -91,22 +91,13 @@ def update_in_model_config(self, config): self.in_model_config.update(config) def __get_hooks(self, relevant_configs, key): - hooks = [] - if key not in relevant_configs: return [] - new_hooks = relevant_configs[key] - if type(new_hooks) not in [list, tuple]: - new_hooks = [new_hooks] - - for hook in new_hooks: - if not isinstance(hook, basestring): - name = ".".join(self.fqn) - dbt.exceptions.raise_compiler_error( - "{} for model {} is not a string!".format(key, name)) + hooks = relevant_configs[key] + if not isinstance(hooks, (list, tuple)): + hooks = [hooks] - hooks.append(hook) return hooks def smart_update(self, mutable_config, new_configs): diff --git a/dbt/utils.py b/dbt/utils.py index 9f9c64dc521..c3bfd3764c3 100644 --- a/dbt/utils.py +++ b/dbt/utils.py @@ -20,8 +20,6 @@ 'sort_type', 'pre-hook', 'post-hook', - 'pre-transaction-hook', - 'post-transaction-hook', 'vars' ] From 7f6136634220406ada49ac590730c6dfb8becbee Mon Sep 17 00:00:00 2001 From: Drew Banin Date: Mon, 21 Aug 2017 15:15:21 -0400 Subject: [PATCH 3/9] fix incremental materializations --- .../macros/materializations/incremental.sql | 13 +++++++++---- .../global_project/macros/materializations/view.sql | 11 +++++++---- 2 files changed, 16 insertions(+), 8 deletions(-) diff --git a/dbt/include/global_project/macros/materializations/incremental.sql b/dbt/include/global_project/macros/materializations/incremental.sql index 52180c67fe0..b9c422b66cf 100644 --- a/dbt/include/global_project/macros/materializations/incremental.sql +++ b/dbt/include/global_project/macros/materializations/incremental.sql @@ -38,8 +38,10 @@ {{ adapter.drop(identifier, existing_type) }} {%- endif %} - {{ run_hooks(pre_transaction_hooks, auto_begin=False) }} - {{ run_hooks(pre_hooks) }} + {{ run_hooks(pre_hooks, inside_transaction=False) }} + + -- `BEGIN` happens here: + {{ run_hooks(pre_hooks, inside_transaction=True) }} -- build model {% if force_create or not adapter.already_exists(schema, identifier) -%} @@ -80,8 +82,11 @@ {% endcall %} {%- endif %} - {{ run_hooks(post_hooks) }} + {{ run_hooks(post_hooks, inside_transaction=True) }} + + -- `COMMIT` happens here {{ adapter.commit() }} - {{ run_hooks(post_transaction_hooks, auto_begin=False) }} + + {{ run_hooks(post_hooks, inside_transaction=False) }} {%- endmaterialization %} diff --git a/dbt/include/global_project/macros/materializations/view.sql b/dbt/include/global_project/macros/materializations/view.sql index dfae7fb52a0..098d4c269f5 100644 --- a/dbt/include/global_project/macros/materializations/view.sql +++ b/dbt/include/global_project/macros/materializations/view.sql @@ -6,8 +6,10 @@ {%- set existing = adapter.query_for_existing(schema) -%} {%- set existing_type = existing.get(identifier) -%} - {{ run_hooks(pre_hooks | rejectattr('transaction'), auto_begin=False) }} - {{ run_hooks(pre_hooks | selectattr('transaction')) }} + {{ run_hooks(pre_hooks, inside_transaction=False) }} + + -- `BEGIN` happens here: + {{ run_hooks(pre_hooks, inside_transaction=True) }} -- build model {% if non_destructive_mode and existing_type == 'view' -%} @@ -18,7 +20,7 @@ {%- endcall %} {%- endif %} - {{ run_hooks(post_hooks | selectattr('transaction')) }} + {{ run_hooks(post_hooks, inside_transaction=True) }} -- cleanup {% if non_destructive_mode and existing_type == 'view' -%} @@ -31,8 +33,9 @@ {{ adapter.rename(tmp_identifier, identifier) }} {%- endif %} + -- `COMMIT` happens here {{ adapter.commit() }} - {{ run_hooks(post_hooks | rejectattr('transaction'), auto_begin=False) }} + {{ run_hooks(post_hooks, inside_transaction=False) }} {%- endmaterialization -%} From f250d1f57e27e5fdda6a660f156d094c7f99496e Mon Sep 17 00:00:00 2001 From: Drew Banin Date: Mon, 28 Aug 2017 16:44:19 -0400 Subject: [PATCH 4/9] hook contract for on-run-start/on-run-end --- dbt/context/common.py | 23 ++------- dbt/contracts/graph/parsed.py | 8 +++- dbt/hooks.py | 42 +++++++++++++++++ .../macros/materializations/helpers.sql | 22 ++++----- dbt/loader.py | 47 ++++++++++++------- dbt/node_runners.py | 25 +++++++--- dbt/parser.py | 37 +++++++++++---- 7 files changed, 137 insertions(+), 67 deletions(-) create mode 100644 dbt/hooks.py diff --git a/dbt/context/common.py b/dbt/context/common.py index 664135d5916..0140b06f525 100644 --- a/dbt/context/common.py +++ b/dbt/context/common.py @@ -3,7 +3,7 @@ import voluptuous from dbt.adapters.factory import get_adapter -from dbt.compat import basestring +from dbt.compat import basestring, to_string import dbt.clients.jinja import dbt.flags @@ -11,22 +11,9 @@ import dbt.tracking import dbt.utils -from dbt.logger import GLOBAL_LOGGER as logger # noqa - - -def get_hooks(model, context, hook_key): - hooks = model.get('config', {}).get(hook_key, []) +import dbt.hooks - if not isinstance(hooks, (list, tuple)): - hooks = [hooks] - - wrapped_hooks = [] - for hook in hooks: - if isinstance(hook, dict): - hook = json.dumps(hook) - wrapped_hooks.append(hook) - - return wrapped_hooks +from dbt.logger import GLOBAL_LOGGER as logger # noqa class DatabaseWrapper(object): @@ -263,8 +250,8 @@ def generate(model, project, flat_graph, provider=None): context = {'env': target} schema = profile.get('schema', 'public') - pre_hooks = get_hooks(model, context, 'pre-hook') - post_hooks = get_hooks(model, context, 'post-hook') + pre_hooks = model.get('config', {}).get('pre-hook') + post_hooks = model.get('config', {}).get('post-hook') db_wrapper = DatabaseWrapper(model, adapter, profile) diff --git a/dbt/contracts/graph/parsed.py b/dbt/contracts/graph/parsed.py index e291fc0422c..da686f5af7d 100644 --- a/dbt/contracts/graph/parsed.py +++ b/dbt/contracts/graph/parsed.py @@ -12,12 +12,16 @@ from dbt.logger import GLOBAL_LOGGER as logger # noqa +hook_contract = Schema({ + Required('sql'): basestring, + Required('transaction'): bool, +}) config_contract = Schema({ Required('enabled'): bool, Required('materialized'): basestring, - Required('post-hook'): list, - Required('pre-hook'): list, + Required('post-hook'): [hook_contract], + Required('pre-hook'): [hook_contract], Required('vars'): dict, }, extra=ALLOW_EXTRA) diff --git a/dbt/hooks.py b/dbt/hooks.py new file mode 100644 index 00000000000..48a5bbf5489 --- /dev/null +++ b/dbt/hooks.py @@ -0,0 +1,42 @@ + +import json +from dbt.compat import to_string + + +class ModelHookType: + PreHook = 'pre-hook' + PostHook = 'post-hook' + Both = [PreHook, PostHook] + + +def _parse_hook_to_dict(hook_string): + try: + hook_dict = json.loads(hook_string) + except ValueError as e: + hook_dict = {"sql": hook_string} + + if 'transaction' not in hook_dict: + hook_dict['transaction'] = True + + return hook_dict + + +def get_hook_dict(hook): + if isinstance(hook, dict): + hook_dict = hook + else: + hook_dict = _parse_hook_to_dict(to_string(hook)) + + return hook_dict + + +def get_hooks(model, hook_key): + hooks = model.get('config', {}).get(hook_key, []) + + if not isinstance(hooks, (list, tuple)): + hooks = [hooks] + + wrapped = [get_hook_dict(hook) for hook in hooks] + return wrapped + + diff --git a/dbt/include/global_project/macros/materializations/helpers.sql b/dbt/include/global_project/macros/materializations/helpers.sql index 2c2ca19110a..d37a6e3130d 100644 --- a/dbt/include/global_project/macros/materializations/helpers.sql +++ b/dbt/include/global_project/macros/materializations/helpers.sql @@ -1,14 +1,8 @@ {% macro run_hooks(hooks, inside_transaction=True) %} - {% for hook in hooks %} - {%- set hook_data = fromjson(render(hook), {}) -%} - {%- set hook_is_in_transaction = hook_data.get('transaction', True) -%}; - {%- set hook_sql = hook_data.get('sql', hook) -%}; - - {%- if hook_is_in_transaction == inside_transaction -%} - {% call statement(auto_begin=inside_transaction) %} - {{ hook_sql }} - {% endcall %} - {%- endif -%} + {% for hook in hooks | selectattr('transaction', 'equalto', inside_transaction) %} + {% call statement(auto_begin=inside_transaction) %} + {{ hook.get('sql') }} + {% endcall %} {% endfor %} {% endmacro %} @@ -37,11 +31,11 @@ {% endmacro %} -{% macro after_commit(sql) %} - {{ make_hook_config(sql, inside_transaction=False) }} +{% macro in_transaction(sql) %} + {{ make_hook_config(sql, inside_transaction=True) }} {% endmacro %} -{% macro vacuum(tbl) %} - {{ after_commit('vacuum ' ~ adapter.quote_schema_and_table(tbl.schema, tbl.name)) }} +{% macro after_commit(sql) %} + {{ make_hook_config(sql, inside_transaction=False) }} {% endmacro %} diff --git a/dbt/loader.py b/dbt/loader.py index 4f63d305aba..57472351b1b 100644 --- a/dbt/loader.py +++ b/dbt/loader.py @@ -14,15 +14,17 @@ def load_all(cls, root_project, all_projects): subgraphs = ['nodes', 'macros'] + macros = MacroLoader.load_all(root_project, all_projects) for subgraph in subgraphs: subgraph_nodes = {} for loader in cls._LOADERS[subgraph]: subgraph_nodes.update( - loader.load_all(root_project, all_projects)) + loader.load_all(root_project, all_projects, macros)) to_return[subgraph] = subgraph_nodes + to_return['macros'] = macros return to_return @classmethod @@ -38,17 +40,18 @@ def register(cls, loader, subgraph='nodes'): class ResourceLoader(object): @classmethod - def load_all(cls, root_project, all_projects): + def load_all(cls, root_project, all_projects, macros=None): to_return = {} for project_name, project in all_projects.items(): to_return.update(cls.load_project(root_project, all_projects, - project, project_name)) + project, project_name, macros)) return to_return @classmethod - def load_project(root_project, all_projects, project, project_name): + def load_project(cls, root_project, all_projects, project, project_name, + macros): raise dbt.exceptions.NotImplementedException( 'load_project is not implemented for this loader!') @@ -56,7 +59,8 @@ def load_project(root_project, all_projects, project, project_name): class MacroLoader(ResourceLoader): @classmethod - def load_project(cls, root_project, all_projects, project, project_name): + def load_project(cls, root_project, all_projects, project, project_name, + macros): return dbt.parser.load_and_parse_macros( package_name=project_name, root_project=root_project, @@ -69,33 +73,38 @@ def load_project(cls, root_project, all_projects, project, project_name): class ModelLoader(ResourceLoader): @classmethod - def load_project(cls, root_project, all_projects, project, project_name): + def load_project(cls, root_project, all_projects, project, project_name, + macros): return dbt.parser.load_and_parse_sql( package_name=project_name, root_project=root_project, all_projects=all_projects, root_dir=project.get('project-root'), relative_dirs=project.get('source-paths', []), - resource_type=NodeType.Model) + resource_type=NodeType.Model, + macros=macros) class AnalysisLoader(ResourceLoader): @classmethod - def load_project(cls, root_project, all_projects, project, project_name): + def load_project(cls, root_project, all_projects, project, project_name, + macros): return dbt.parser.load_and_parse_sql( package_name=project_name, root_project=root_project, all_projects=all_projects, root_dir=project.get('project-root'), relative_dirs=project.get('analysis-paths', []), - resource_type=NodeType.Analysis) + resource_type=NodeType.Analysis, + macros=macros) class SchemaTestLoader(ResourceLoader): @classmethod - def load_project(cls, root_project, all_projects, project, project_name): + def load_project(cls, root_project, all_projects, project, project_name, + macros): return dbt.parser.load_and_parse_yml( package_name=project_name, root_project=root_project, @@ -107,7 +116,8 @@ def load_project(cls, root_project, all_projects, project, project_name): class DataTestLoader(ResourceLoader): @classmethod - def load_project(cls, root_project, all_projects, project, project_name): + def load_project(cls, root_project, all_projects, project, project_name, + macros): return dbt.parser.load_and_parse_sql( package_name=project_name, root_project=root_project, @@ -115,13 +125,15 @@ def load_project(cls, root_project, all_projects, project, project_name): root_dir=project.get('project-root'), relative_dirs=project.get('test-paths', []), resource_type=NodeType.Test, - tags={'data'}) + tags={'data'}, + macros=macros) class ArchiveLoader(ResourceLoader): @classmethod - def load_project(cls, root_project, all_projects, project, project_name): + def load_project(cls, root_project, all_projects, project, project_name, + macros): return dbt.parser.parse_archives_from_projects(root_project, all_projects) @@ -129,12 +141,11 @@ def load_project(cls, root_project, all_projects, project, project_name): class RunHookLoader(ResourceLoader): @classmethod - def load_project(cls, root_project, all_projects, project, project_name): - return dbt.parser.load_and_parse_run_hooks(root_project, all_projects) - + def load_project(cls, root_project, all_projects, project, project_name, + macros): + return dbt.parser.load_and_parse_run_hooks(root_project, all_projects, + macros) -# macro loaders -GraphLoader.register(MacroLoader, 'macros') # node loaders GraphLoader.register(ModelLoader, 'nodes') diff --git a/dbt/node_runners.py b/dbt/node_runners.py index d7614c431e8..72d23250b0e 100644 --- a/dbt/node_runners.py +++ b/dbt/node_runners.py @@ -275,13 +275,26 @@ def run_hooks(cls, project, adapter, flat_graph, hook_type): nodes = flat_graph.get('nodes', {}).values() hooks = get_nodes_by_tags(nodes, {hook_type}, NodeType.Operation) - for hook in hooks: - compiled = cls.compile_node(adapter, project, hook, flat_graph) + # TODO : hook_blob is a ;-delimited list of statements. Make this a list + # (in the parser) instead + for hook_blob in hooks: + compiled = cls.compile_node(adapter, project, hook_blob, flat_graph) model_name = compiled.get('name') - sql = compiled['wrapped_sql'] - adapter.execute_one(profile, sql, model_name=model_name, - auto_begin=True) - adapter.commit_if_has_connection(profile, model_name) + statements = compiled['wrapped_sql'].split(";") + + # make sure there isn't an open transaction + conn = adapter.begin(profile, model_name) + adapter.commit(profile, conn) + + for statement in statements: + hook = dbt.hooks.get_hook_dict(statement) + sql = hook.get('sql', '').strip() + + if len(sql) == 0: + continue + + adapter.execute_one(profile, sql, model_name=model_name, + auto_begin=False) @classmethod def safe_run_hooks(cls, project, adapter, flat_graph, hook_type): diff --git a/dbt/parser.py b/dbt/parser.py index a8ba769f81d..8215c245434 100644 --- a/dbt/parser.py +++ b/dbt/parser.py @@ -5,6 +5,7 @@ import dbt.flags import dbt.model import dbt.utils +import dbt.hooks import jinja2.runtime import dbt.clients.jinja @@ -179,7 +180,7 @@ def parse_macro_file(macro_file_path, def parse_node(node, node_path, root_project_config, package_project_config, - all_projects, tags=None, fqn_extra=None): + all_projects, tags=None, fqn_extra=None, macros=None): logger.debug("Parsing {}".format(node_path)) node = copy.deepcopy(node) @@ -189,6 +190,9 @@ def parse_node(node, node_path, root_project_config, package_project_config, if fqn_extra is None: fqn_extra = [] + if macros is None: + macros = {} + node.update({ 'refs': [], 'depends_on': { @@ -215,7 +219,8 @@ def parse_node(node, node_path, root_project_config, package_project_config, config_dict.update(config.config) node['config'] = config_dict - context = dbt.context.parser.generate(node, package_project_config, {}) + context = dbt.context.parser.generate(node, package_project_config, + {"macros": macros}) dbt.clients.jinja.get_rendered( node.get('raw_sql'), context, node, @@ -232,15 +237,21 @@ def parse_node(node, node_path, root_project_config, package_project_config, config_dict.update(config.config) node['config'] = config_dict + for hook_type in dbt.hooks.ModelHookType.Both: + node['config'][hook_type] = dbt.hooks.get_hooks(node, hook_type) + del node['config_reference'] return node -def parse_sql_nodes(nodes, root_project, projects, tags=None): +def parse_sql_nodes(nodes, root_project, projects, tags=None, macros=None): if tags is None: tags = set() + if macros is None: + macros = {} + to_return = {} dbt.contracts.graph.unparsed.validate_nodes(nodes) @@ -258,7 +269,8 @@ def parse_sql_nodes(nodes, root_project, projects, tags=None): root_project, projects.get(package_name), projects, - tags=tags) + tags=tags, + macros=macros) dbt.contracts.graph.parsed.validate_nodes(to_return) @@ -266,12 +278,15 @@ def parse_sql_nodes(nodes, root_project, projects, tags=None): def load_and_parse_sql(package_name, root_project, all_projects, root_dir, - relative_dirs, resource_type, tags=None): + relative_dirs, resource_type, tags=None, macros=None): extension = "[!.#~]*.sql" if tags is None: tags = set() + if macros is None: + macros = {} + if dbt.flags.STRICT_MODE: dbt.contracts.project.validate_list(all_projects) @@ -311,7 +326,7 @@ def load_and_parse_sql(package_name, root_project, all_projects, root_dir, 'raw_sql': file_contents }) - return parse_sql_nodes(result, root_project, all_projects, tags) + return parse_sql_nodes(result, root_project, all_projects, tags, macros) def get_hooks_from_project(project_cfg, hook_type): @@ -335,7 +350,8 @@ def get_hooks(all_projects, hook_type): return project_hooks -def load_and_parse_run_hook_type(root_project, all_projects, hook_type): +def load_and_parse_run_hook_type(root_project, all_projects, hook_type, + macros): if dbt.flags.STRICT_MODE: dbt.contracts.project.validate_list(all_projects) @@ -361,13 +377,16 @@ def load_and_parse_run_hook_type(root_project, all_projects, hook_type): tags = {hook_type} return parse_sql_nodes(result, root_project, all_projects, tags=tags) +def load_and_parse_run_hooks(root_project, all_projects, macros=None): + if macros is None: + macros = {} -def load_and_parse_run_hooks(root_project, all_projects): hook_nodes = {} for hook_type in RunHookType.Both: project_hooks = load_and_parse_run_hook_type(root_project, all_projects, - hook_type) + hook_type, + macros=macros) hook_nodes.update(project_hooks) return hook_nodes From f681f7711c0959cdbed81c52e4b275cb525339af Mon Sep 17 00:00:00 2001 From: Drew Banin Date: Tue, 29 Aug 2017 13:40:27 -0400 Subject: [PATCH 5/9] make on-run-* hooks work more sanely --- dbt/adapters/default.py | 6 ++++++ dbt/contracts/graph/parsed.py | 4 ++++ dbt/node_runners.py | 37 +++++++++++++++++++++-------------- dbt/parser.py | 35 ++++++++++++++++++--------------- 4 files changed, 51 insertions(+), 31 deletions(-) diff --git a/dbt/adapters/default.py b/dbt/adapters/default.py index f9db8b021e3..9681d084ac9 100644 --- a/dbt/adapters/default.py +++ b/dbt/adapters/default.py @@ -516,6 +516,12 @@ def add_query(cls, profile, sql, model_name=None, auto_begin=True): return connection, cursor + @classmethod + def clear_transaction(cls, profile, conn_name='master'): + conn = cls.begin(profile, conn_name) + cls.commit(profile, conn) + return conn_name + @classmethod def execute_one(cls, profile, sql, model_name=None, auto_begin=False): cls.get_connection(profile, model_name) diff --git a/dbt/contracts/graph/parsed.py b/dbt/contracts/graph/parsed.py index da686f5af7d..511129a7914 100644 --- a/dbt/contracts/graph/parsed.py +++ b/dbt/contracts/graph/parsed.py @@ -73,6 +73,10 @@ }) +def validate_hook(hook): + validate_with(hook_contract, hooks) + + def validate_nodes(parsed_nodes): validate_with(parsed_nodes_contract, parsed_nodes) diff --git a/dbt/node_runners.py b/dbt/node_runners.py index 72d23250b0e..b3412809d32 100644 --- a/dbt/node_runners.py +++ b/dbt/node_runners.py @@ -275,26 +275,33 @@ def run_hooks(cls, project, adapter, flat_graph, hook_type): nodes = flat_graph.get('nodes', {}).values() hooks = get_nodes_by_tags(nodes, {hook_type}, NodeType.Operation) - # TODO : hook_blob is a ;-delimited list of statements. Make this a list - # (in the parser) instead - for hook_blob in hooks: - compiled = cls.compile_node(adapter, project, hook_blob, flat_graph) + # This will clear out an open transaction if there is one. + # on-run-* hooks should run outside of a transaction. This happens b/c + # psycopg2 automatically begins a transaction when a connection is + # created. TODO : Move transaction logic out of here, and implement + # a for-loop over these sql statements in jinja-land. Also, consider + # configuring psycopg2 (and other adapters?) to ensure that a + # transaction is only created if dbt initiates it. + conn_name = adapter.clear_transaction(profile) + + compiled_hooks = [] + for hook in hooks: + compiled = cls.compile_node(adapter, project, hook, flat_graph) model_name = compiled.get('name') - statements = compiled['wrapped_sql'].split(";") + statement = compiled['wrapped_sql'] - # make sure there isn't an open transaction - conn = adapter.begin(profile, model_name) - adapter.commit(profile, conn) + hook_dict = dbt.hooks.get_hook_dict(statement) + compiled_hooks.append(hook_dict) - for statement in statements: - hook = dbt.hooks.get_hook_dict(statement) - sql = hook.get('sql', '').strip() + for hook in compiled_hooks: - if len(sql) == 0: - continue + if dbt.flags.STRICT_MODE: + dbt.contracts.graph.parsed.validate_hook(hook) - adapter.execute_one(profile, sql, model_name=model_name, - auto_begin=False) + sql = hook.get('sql', '') + adapter.execute_one(profile, sql, model_name=conn_name, + auto_begin=False) + adapter.release_connection(profile, conn_name) @classmethod def safe_run_hooks(cls, project, adapter, flat_graph, hook_type): diff --git a/dbt/parser.py b/dbt/parser.py index 1b24ecf4009..6c97cdbb31c 100644 --- a/dbt/parser.py +++ b/dbt/parser.py @@ -2,6 +2,7 @@ import os import re import hashlib +import collections import dbt.flags import dbt.model @@ -342,13 +343,11 @@ def get_hooks_from_project(project_cfg, hook_type): def get_hooks(all_projects, hook_type): - project_hooks = {} + project_hooks = collections.defaultdict(list) for project_name, project in all_projects.items(): hooks = get_hooks_from_project(project, hook_type) - - if len(hooks) > 0: - project_hooks[project_name] = ";\n".join(hooks) + project_hooks[project_name].extend(hooks) return project_hooks @@ -365,20 +364,24 @@ def load_and_parse_run_hook_type(root_project, all_projects, hook_type, for project_name, hooks in project_hooks.items(): project = all_projects[project_name] - hook_path = dbt.utils.get_pseudo_hook_path(hook_type) - - result.append({ - 'name': hook_type, - 'root_path': "{}/dbt_project.yml".format(project_name), - 'resource_type': NodeType.Operation, - 'path': hook_path, - 'original_file_path': hook_path, - 'package_name': project_name, - 'raw_sql': hooks - }) + for i, hook in enumerate(hooks): + hook_name = '{}-{}-{}'.format(project_name, hook_type, i) + hook_path = dbt.utils.get_pseudo_hook_path(hook_name) + + result.append({ + 'name': hook_name, + 'root_path': "{}/dbt_project.yml".format(project_name), + 'resource_type': NodeType.Operation, + 'path': hook_path, + 'original_file_path': hook_path, + 'package_name': project_name, + 'raw_sql': hook + }) tags = {hook_type} - return parse_sql_nodes(result, root_project, all_projects, tags=tags) + return parse_sql_nodes(result, root_project, all_projects, tags=tags, + macros=macros) + def load_and_parse_run_hooks(root_project, all_projects, macros=None): if macros is None: From 2c206898b3c1ccabbf9c9060d859ea986aeff83e Mon Sep 17 00:00:00 2001 From: Drew Banin Date: Tue, 29 Aug 2017 13:42:46 -0400 Subject: [PATCH 6/9] pep8 --- dbt/hooks.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/dbt/hooks.py b/dbt/hooks.py index 48a5bbf5489..7df63354869 100644 --- a/dbt/hooks.py +++ b/dbt/hooks.py @@ -38,5 +38,3 @@ def get_hooks(model, hook_key): wrapped = [get_hook_dict(hook) for hook in hooks] return wrapped - - From 022b3d71712776379ac1440fb62b9d242f754b90 Mon Sep 17 00:00:00 2001 From: Drew Banin Date: Tue, 29 Aug 2017 13:47:49 -0400 Subject: [PATCH 7/9] make codeclimate happy(-ier) --- dbt/graph/selector.py | 9 +-------- dbt/parser.py | 13 ++++--------- dbt/utils.py | 7 +++++++ 3 files changed, 12 insertions(+), 17 deletions(-) diff --git a/dbt/graph/selector.py b/dbt/graph/selector.py index 2974e1954ab..0e4efe49976 100644 --- a/dbt/graph/selector.py +++ b/dbt/graph/selector.py @@ -1,7 +1,7 @@ import networkx as nx from dbt.logger import GLOBAL_LOGGER as logger -from dbt.utils import is_enabled, get_materialization +from dbt.utils import is_enabled, get_materialization, coalesce from dbt.node_types import NodeType SELECTOR_PARENTS = '+' @@ -43,13 +43,6 @@ def parse_spec(node_spec): } -def coalesce(*args): - for arg in args: - if arg is not None: - return arg - return None - - def get_package_names(graph): return set([node.split(".")[1] for node in graph.nodes()]) diff --git a/dbt/parser.py b/dbt/parser.py index 6c97cdbb31c..89504d0c4c6 100644 --- a/dbt/parser.py +++ b/dbt/parser.py @@ -22,7 +22,7 @@ from dbt.node_types import NodeType, RunHookType from dbt.compat import basestring, to_string from dbt.logger import GLOBAL_LOGGER as logger -from dbt.utils import get_pseudo_test_path +from dbt.utils import get_pseudo_test_path, coalesce def get_path(resource_type, package_name, resource_name): @@ -187,14 +187,9 @@ def parse_node(node, node_path, root_project_config, package_project_config, logger.debug("Parsing {}".format(node_path)) node = copy.deepcopy(node) - if tags is None: - tags = set() - - if fqn_extra is None: - fqn_extra = [] - - if macros is None: - macros = {} + tags = coalesce(tags, set()) + fqn_extra = coalesce(tags, []) + macros = coalesce(macros, {}) node.update({ 'refs': [], diff --git a/dbt/utils.py b/dbt/utils.py index c3bfd3764c3..ddb83eb20ed 100644 --- a/dbt/utils.py +++ b/dbt/utils.py @@ -43,6 +43,13 @@ def __repr__(self): return self.schema_table(self.schema, self.table) +def coalesce(*args): + for arg in args: + if arg is not None: + return arg + return None + + def get_model_name_or_none(model): if model is None: name = '' From 98e42270e38ea0004ed5462181950ebb92f474c8 Mon Sep 17 00:00:00 2001 From: Drew Banin Date: Tue, 29 Aug 2017 14:06:49 -0400 Subject: [PATCH 8/9] typo --- dbt/parser.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbt/parser.py b/dbt/parser.py index 89504d0c4c6..f95692f4c61 100644 --- a/dbt/parser.py +++ b/dbt/parser.py @@ -188,7 +188,7 @@ def parse_node(node, node_path, root_project_config, package_project_config, node = copy.deepcopy(node) tags = coalesce(tags, set()) - fqn_extra = coalesce(tags, []) + fqn_extra = coalesce(fqn_extra, []) macros = coalesce(macros, {}) node.update({ From df98bcf6e741dddb56786311c3cec7e7b890e0b6 Mon Sep 17 00:00:00 2001 From: Drew Banin Date: Tue, 29 Aug 2017 15:04:35 -0400 Subject: [PATCH 9/9] fix for bq commit signature --- dbt/adapters/bigquery.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbt/adapters/bigquery.py b/dbt/adapters/bigquery.py index 0a1eeee3e75..ff6949b6360 100644 --- a/dbt/adapters/bigquery.py +++ b/dbt/adapters/bigquery.py @@ -67,7 +67,7 @@ def begin(cls, profile, name='master'): pass @classmethod - def commit(cls, connection): + def commit(cls, profile, connection): pass @classmethod