Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Support transient tables on Snowflake #1252

Merged
merged 2 commits into from
Feb 13, 2019
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions core/dbt/adapters/base/impl.py
Original file line number Diff line number Diff line change
Expand Up @@ -110,6 +110,10 @@ class BaseAdapter(object):
# This should be an implementation of BaseConnectionManager
ConnectionManager = None

# A set of clobber config fields accepted by this adapter
# for use in materializations
AdapterSpecificConfigs = frozenset()

def __init__(self, config):
self.config = config
self.cache = RelationsCache()
Expand Down
4 changes: 2 additions & 2 deletions core/dbt/config/project.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@
from dbt.ui import printer
from dbt.utils import deep_map
from dbt.utils import parse_cli_vars
from dbt.utils import DBTConfigKeys
from dbt.parser.source_config import SourceConfig

from dbt.contracts.project import Project as ProjectContract
from dbt.contracts.project import PackageConfig
Expand Down Expand Up @@ -83,7 +83,7 @@ def _get_config_paths(config, path=(), paths=None):

for key, value in config.items():
if isinstance(value, dict):
if key in DBTConfigKeys:
if key in SourceConfig.ConfigKeys:
if path not in paths:
paths.add(path)
else:
Expand Down
39 changes: 17 additions & 22 deletions core/dbt/parser/source_config.py
Original file line number Diff line number Diff line change
@@ -1,28 +1,25 @@
import dbt.exceptions

from dbt.utils import deep_merge, DBTConfigKeys
from dbt.utils import deep_merge
from dbt.node_types import NodeType
from dbt.adapters.factory import get_adapter_class_by_name


class SourceConfig(object):
ConfigKeys = DBTConfigKeys

AppendListFields = {'pre-hook', 'post-hook', 'tags'}
ExtendDictFields = {'vars', 'column_types', 'quoting'}
ClobberFields = {
'alias',
'schema',
'enabled',
'materialized',
'dist',
'sort',
'sql_where',
'unique_key',
'sort_type',
'bind',
'database',
}

ConfigKeys = AppendListFields | ExtendDictFields | ClobberFields

def __init__(self, active_project, own_project, fqn, node_type):
self._config = None
# active_project is a RuntimeConfig, not a Project
Expand All @@ -31,16 +28,13 @@ def __init__(self, active_project, own_project, fqn, node_type):
self.fqn = fqn
self.node_type = node_type

adapter_type = active_project.credentials.type
adapter_class = get_adapter_class_by_name(adapter_type)
self.AdapterSpecificConfigs = adapter_class.AdapterSpecificConfigs

# the config options defined within the model
self.in_model_config = {}

# make sure we categorize all configs
all_configs = self.AppendListFields | self.ExtendDictFields | \
self.ClobberFields

for config in self.ConfigKeys:
assert config in all_configs, config

def _merge(self, *configs):
merged_config = {}
for config in configs:
Expand Down Expand Up @@ -107,7 +101,7 @@ def update_in_model_config(self, config):
)
current.update(value)
self.in_model_config[key] = current
else: # key in self.ClobberFields
else: # key in self.ClobberFields or self.AdapterSpecificConfigs
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This seems to mean that adapter specific configurations will always be clobber-only. If we wanted to change that in the future, would it be easy/possible without breaking existing adapters?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I gave this a tiny bit of thought, and I'm really not sure what an append/extend config could possibly look like for adapter configs. I think "clobber" will be the most common paradigm for new configs, but I do also like the idea of not needing to change this API in the future. I think it will be easy to just add a couple more class attributes like AdapterSpecificExtendConfigs as the need arises, and 3rd party adapters should just inherit these attributes from the base adapter. Do you buy that? Or do you think there's a better way of doing this?

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yeah, that does sound ok, in the worst case we end up with 6 of these which isn't terrible.

I do think adapter-specific append configs are potentially pretty reasonable - as a lazy example, imagine an adapter where the underlying database supports a concept like tags. It would feel pretty natural for those to append instead of clobbering.

self.in_model_config[key] = value

@staticmethod
Expand All @@ -121,24 +115,25 @@ def __get_as_list(relevant_configs, key):

return items

@classmethod
def smart_update(cls, mutable_config, new_configs):
def smart_update(self, mutable_config, new_configs):
config_keys = self.ConfigKeys | self.AdapterSpecificConfigs

relevant_configs = {
key: new_configs[key] for key
in new_configs if key in cls.ConfigKeys
in new_configs if key in config_keys
}

for key in cls.AppendListFields:
append_fields = cls.__get_as_list(relevant_configs, key)
for key in self.AppendListFields:
append_fields = self.__get_as_list(relevant_configs, key)
mutable_config[key].extend([
f for f in append_fields if f not in mutable_config[key]
])

for key in cls.ExtendDictFields:
for key in self.ExtendDictFields:
dict_val = relevant_configs.get(key, {})
mutable_config[key].update(dict_val)

for key in cls.ClobberFields:
for key in (self.ClobberFields | self.AdapterSpecificConfigs):
if key in relevant_configs:
mutable_config[key] = relevant_configs[key]

Expand Down
21 changes: 0 additions & 21 deletions core/dbt/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,27 +20,6 @@
from dbt.clients import yaml_helper


DBTConfigKeys = [
'alias',
'schema',
'enabled',
'materialized',
'dist',
'sort',
'sql_where',
'unique_key',
'sort_type',
'pre-hook',
'post-hook',
'vars',
'column_types',
'bind',
'quoting',
'tags',
'database',
]


class ExitCodes(object):
Success = 0
ModelError = 1
Expand Down
2 changes: 2 additions & 0 deletions plugins/bigquery/dbt/adapters/bigquery/impl.py
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,8 @@ class BigQueryAdapter(BaseAdapter):
Column = dbt.schema.BigQueryColumn
ConnectionManager = BigQueryConnectionManager

AdapterSpecificConfigs = frozenset({"cluster_by", "partition_by"})

###
# Implementations of abstract methods
###
Expand Down
2 changes: 2 additions & 0 deletions plugins/redshift/dbt/adapters/redshift/impl.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,8 @@
class RedshiftAdapter(PostgresAdapter):
ConnectionManager = RedshiftConnectionManager

AdapterSpecificConfigs = frozenset({"sort_type", "dist", "sort", "bind"})

@classmethod
def date_function(cls):
return 'getdate()'
Expand Down
2 changes: 2 additions & 0 deletions plugins/snowflake/dbt/adapters/snowflake/impl.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,8 @@ class SnowflakeAdapter(SQLAdapter):
Relation = SnowflakeRelation
ConnectionManager = SnowflakeConnectionManager

AdapterSpecificConfigs = frozenset({"transient"})

@classmethod
def date_function(cls):
return 'CURRENT_TIMESTAMP()'
Expand Down
11 changes: 10 additions & 1 deletion plugins/snowflake/dbt/include/snowflake/macros/adapters.sql
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,16 @@
use schema {{ adapter.quote_as_configured(schema, 'schema') }};
{% endif %}

{{ default__create_table_as(temporary, relation, sql) }}
{%- set transient = config.get('transient', default=true) -%}

create {% if temporary -%}
temporary
{%- elif transient -%}
transient
{%- endif %} table {{ relation.include(database=(not temporary), schema=(not temporary)) }}
as (
{{ sql }}
);
{% endmacro %}

{% macro snowflake__create_view_as(relation, sql) -%}
Expand Down
2 changes: 1 addition & 1 deletion test/unit/test_parser.py
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ def setUp(self):
'quoting': {},
'outputs': {
'test': {
'type': 'postgres',
'type': 'redshift',
'host': 'localhost',
'schema': 'analytics',
'user': 'test',
Expand Down