Merge pull request #2143 from fishtown-analytics/feature/generate-dat…

…abase-name

add generate_database_name macro (#1695)

CHANGELOG.md

@@ -2,10 +2,12 @@
 
 ### Breaking changes
 - Arguments to source tests are not parsed in the config-rendering context, and are passed as their literal unparsed values to macros ([#2150](https://github.com/fishtown-analytics/dbt/pull/2150))
+- `generate_schema_name` macros that accept a single argument are no longer supported ([#2143](https://github.com/fishtown-analytics/dbt/pull/2143))
 
 ### Features
 - Add a "docs" field to models, with a "show" subfield ([#1671](https://github.com/fishtown-analytics/dbt/issues/1671), [#2107](https://github.com/fishtown-analytics/dbt/pull/2107))
 - Add a dbt-{dbt_version} user agent field to the bigquery connector ([#2121](https://github.com/fishtown-analytics/dbt/issues/2121), [#2146](https://github.com/fishtown-analytics/dbt/pull/2146))
+- Add support for `generate_database_name` macro ([#1695](https://github.com/fishtown-analytics/dbt/issues/1695), [#2143](https://github.com/fishtown-analytics/dbt/pull/2143))
 
 ### Fixes
 - Fix issue where dbt did not give an error in the presence of duplicate doc names ([#2054](https://github.com/fishtown-analytics/dbt/issues/2054), [#2080](https://github.com/fishtown-analytics/dbt/pull/2080))

core/dbt/contracts/graph/manifest.py

@@ -541,21 +541,21 @@ def filter(candidate: MacroCandidate) -> bool:
         return candidates.last()
 
     def find_generate_macro_by_name(
-        self, name: str, root_project_name: str
+        self, component: str, root_project_name: str
     ) -> Optional[ParsedMacro]:
         """
         The `generate_X_name` macros are similar to regular ones, but ignore
         imported packages.
-            - if there is a `name` macro in the root project, return it
-            - if that does not exist but there is a `name` macro in the 'dbt'
-                internal project (or a plugin), return that
-            - if neither of those exist (unit tests?), return None
+            - if there is a `generate_{component}_name` macro in the root
+              project, return it
+            - return the `generate_{component}_name` macro from the 'dbt'
+              internal project
         """
         def filter(candidate: MacroCandidate) -> bool:
             return candidate.locality != Locality.Imported
 
         candidates: CandidateList = self._find_macros_by_name(
-            name=name,
+            name=f'generate_{component}_name',
             root_project_name=root_project_name,
             # filter out imported packages
             filter=filter,

core/dbt/include/global_project/macros/etc/get_custom_database.sql

@@ -0,0 +1,28 @@
+{#
+    Renders a database name given a custom database name. If the custom
+    database name is none, then the resulting database is just the "database"
+    value in the specified target. If a database override is specified, then
+    the resulting database is the default database concatenated with the
+    custom database.
+
+    This macro can be overriden in projects to define different semantics
+    for rendering a database name.
+
+    Arguments:
+    custom_database_name: The custom database name specified for a model, or none
+    node: The node the database is being generated for
+
+#}
+{% macro generate_database_name(custom_database_name=none, node=none) -%}
+    {%- set default_database = target.database -%}
+    {%- if custom_database_name is none -%}
+
+        {{ default_database }}
+
+    {%- else -%}
+
+        {{ custom_database_name }}
+
+    {%- endif -%}
+
+{%- endmacro %}

core/dbt/parser/base.py

@@ -2,7 +2,7 @@
 import itertools
 import os
 from typing import (
-    List, Dict, Any, Callable, Iterable, Optional, Generic, TypeVar
+    List, Dict, Any, Iterable, Generic, TypeVar
 )
 
 from hologram import ValidationError
@@ -11,18 +11,17 @@
 from dbt.clients.system import load_file_contents
 from dbt.context.providers import generate_parser_model, generate_parser_macro
 import dbt.flags
-from dbt import deprecations
 from dbt import hooks
 from dbt.adapters.factory import get_adapter
 from dbt.clients.jinja import get_rendered
 from dbt.config import Project, RuntimeConfig
 from dbt.contracts.graph.manifest import (
     Manifest, SourceFile, FilePath, FileHash
 )
-from dbt.contracts.graph.parsed import HasUniqueID, ParsedMacro
+from dbt.contracts.graph.parsed import HasUniqueID
 from dbt.contracts.graph.unparsed import UnparsedNode
 from dbt.exceptions import (
-    CompilationException, validator_error_message
+    CompilationException, validator_error_message, InternalException
 )
 from dbt.node_types import NodeType
 from dbt.source_config import SourceConfig
@@ -39,7 +38,6 @@
 FinalNode = TypeVar('FinalNode', bound=ManifestNodes)
 
 
-RelationUpdate = Callable[[Optional[str], IntermediateNode], str]
 ConfiguredBlockType = TypeVar('ConfiguredBlockType', bound=FileBlock)
 
 
@@ -94,6 +92,31 @@ def __init__(
         self.macro_manifest = macro_manifest
 
 
+class RelationUpdate:
+    def __init__(
+        self, config: RuntimeConfig, manifest: Manifest, component: str
+    ) -> None:
+        macro = manifest.find_generate_macro_by_name(
+            component=component,
+            root_project_name=config.project_name,
+        )
+        if macro is None:
+            raise InternalException(
+                f'No macro with name generate_{component}_name found'
+            )
+
+        root_context = generate_parser_macro(macro, config, manifest, None)
+        self.updater = MacroGenerator(macro, root_context)
+        self.component = component
+
+    def __call__(
+        self, parsed_node: Any, config_dict: Dict[str, Any]
+    ) -> None:
+        override = config_dict.get(self.component)
+        new_value = self.updater(override, parsed_node).strip()
+        setattr(parsed_node, self.component, new_value)
+
+
 class ConfiguredParser(
     Parser[FinalNode],
     Generic[ConfiguredBlockType, IntermediateNode, FinalNode],
@@ -106,8 +129,16 @@ def __init__(
         macro_manifest: Manifest,
     ) -> None:
         super().__init__(results, project, root_project, macro_manifest)
-        self._get_schema_func: Optional[RelationUpdate] = None
-        self._get_alias_func: Optional[RelationUpdate] = None
+
+        self._update_node_database = RelationUpdate(
+            manifest=macro_manifest, config=root_project, component='database'
+        )
+        self._update_node_schema = RelationUpdate(
+            manifest=macro_manifest, config=root_project, component='schema'
+        )
+        self._update_node_alias = RelationUpdate(
+            manifest=macro_manifest, config=root_project, component='alias'
+        )
 
     @abc.abstractclassmethod
     def get_compiled_path(cls, block: ConfiguredBlockType) -> str:
@@ -129,69 +160,6 @@ def default_schema(self):
     def default_database(self):
         return self.root_project.credentials.database
 
-    def _build_generate_macro_function(self, macro: ParsedMacro) -> Callable:
-        root_context = generate_parser_macro(
-            macro, self.root_project, self.macro_manifest, None
-        )
-        return MacroGenerator(macro, root_context)
-
-    def get_schema_func(self) -> RelationUpdate:
-        """The get_schema function is set by a few different things:
-            - if there is a 'generate_schema_name' macro in the root project,
-                it will be used.
-            - if that does not exist but there is a 'generate_schema_name'
-                macro in the 'dbt' internal project, that will be used
-            - if neither of those exist (unit tests?), a function that returns
-                the 'default schema' as set in the root project's 'credentials'
-                is used
-        """
-        if self._get_schema_func is not None:
-            return self._get_schema_func
-
-        get_schema_macro = self.macro_manifest.find_generate_macro_by_name(
-            name='generate_schema_name',
-            root_project_name=self.root_project.project_name,
-        )
-        # this is only true in tests!
-        if get_schema_macro is None:
-            def get_schema(custom_schema_name=None, node=None):
-                return self.default_schema
-        else:
-            get_schema = self._build_generate_macro_function(get_schema_macro)
-
-        self._get_schema_func = get_schema
-        return self._get_schema_func
-
-    def get_alias_func(self) -> RelationUpdate:
-        """The get_alias function is set by a few different things:
-            - if there is a 'generate_alias_name' macro in the root project,
-                it will be used.
-            - if that does not exist but there is a 'generate_alias_name'
-                macro in the 'dbt' internal project, that will be used
-            - if neither of those exist (unit tests?), a function that returns
-                the 'default alias' as set in the model's filename or alias
-                configuration.
-        """
-        if self._get_alias_func is not None:
-            return self._get_alias_func
-
-        get_alias_macro = self.macro_manifest.find_generate_macro_by_name(
-            name='generate_alias_name',
-            root_project_name=self.root_project.project_name,
-        )
-        # the generate_alias_name macro might not exist
-        if get_alias_macro is None:
-            def get_alias(custom_alias_name, node):
-                if custom_alias_name is None:
-                    return node.name
-                else:
-                    return custom_alias_name
-        else:
-            get_alias = self._build_generate_macro_function(get_alias_macro)
-
-        self._get_alias_func = get_alias
-        return self._get_alias_func
-
     def get_fqn(self, path: str, name: str) -> List[str]:
         """Get the FQN for the node. This impacts node selection and config
         application.
@@ -297,33 +265,6 @@ def render_with_context(
                 parsed_node.raw_sql, context, parsed_node, capture_macros=True
             )
 
-    def update_parsed_node_schema(
-        self, parsed_node: IntermediateNode, config_dict: Dict[str, Any]
-    ) -> None:
-        # Special macro defined in the global project. Use the root project's
-        # definition, not the current package
-        schema_override = config_dict.get('schema')
-        get_schema = self.get_schema_func()
-        try:
-            schema = get_schema(schema_override, parsed_node)
-        except dbt.exceptions.CompilationException as exc:
-            too_many_args = (
-                "macro 'dbt_macro__generate_schema_name' takes not more than "
-                "1 argument(s)"
-            )
-            if too_many_args not in str(exc):
-                raise
-            deprecations.warn('generate-schema-name-single-arg')
-            schema = get_schema(schema_override)  # type: ignore
-        parsed_node.schema = schema.strip()
-
-    def update_parsed_node_alias(
-        self, parsed_node: IntermediateNode, config_dict: Dict[str, Any]
-    ) -> None:
-        alias_override = config_dict.get('alias')
-        get_alias = self.get_alias_func()
-        parsed_node.alias = get_alias(alias_override, parsed_node).strip()
-
     def update_parsed_node_config(
         self, parsed_node: IntermediateNode, config_dict: Dict[str, Any]
     ) -> None:
@@ -334,6 +275,13 @@ def update_parsed_node_config(
         self._mangle_hooks(final_config_dict)
         parsed_node.config = parsed_node.config.from_dict(final_config_dict)
 
+    def update_parsed_node_name(
+        self, parsed_node: IntermediateNode, config_dict: Dict[str, Any]
+    ) -> None:
+        self._update_node_database(parsed_node, config_dict)
+        self._update_node_schema(parsed_node, config_dict)
+        self._update_node_alias(parsed_node, config_dict)
+
     def update_parsed_node(
         self, parsed_node: IntermediateNode, config: SourceConfig
     ) -> None:
@@ -347,15 +295,10 @@ def update_parsed_node(
         model_tags = config_dict.get('tags', [])
         parsed_node.tags.extend(model_tags)
 
-        # do this once before we parse the node schema/alias, so
+        # do this once before we parse the node database/schema/alias, so
         # parsed_node.config is what it would be if they did nothing
         self.update_parsed_node_config(parsed_node, config_dict)
-
-        parsed_node.database = config_dict.get(
-            'database', self.default_database
-        ).strip()
-        self.update_parsed_node_schema(parsed_node, config_dict)
-        self.update_parsed_node_alias(parsed_node, config_dict)
+        self.update_parsed_node_name(parsed_node, config_dict)
 
         # at this point, we've collected our hooks. Use the node context to
         # render each hook and collect refs/sources

test/integration/006_simple_dependency_test/local_dependency/macros/generate_schema_name.sql

@@ -1,4 +1,15 @@
 {# This should be ignored as it's in a subpackage #}
-{% macro generate_schema_name(custom_schema_name=none) -%}
-  invalid_schema_name
+{% macro generate_schema_name(custom_schema_name=none, node=none) -%}
+  {{ exceptions.raise_compiler_error('invalid', node=node) }}
+{%- endmacro %}
+
+{# This should be ignored as it's in a subpackage #}
+{% macro generate_database_name(custom_database_name=none, node=none) -%}
+  {{ exceptions.raise_compiler_error('invalid', node=node) }}
+{%- endmacro %}
+
+
+{# This should be ignored as it's in a subpackage #}
+{% macro generate_alias_name(custom_alias_name=none, node=none) -%}
+  {{ exceptions.raise_compiler_error('invalid', node=node) }}
 {%- endmacro %}

test/integration/012_deprecation_tests/test_deprecations.py

@@ -35,30 +35,6 @@ def test_postgres_deprecations(self):
         self.assertEqual(expected, deprecations.active_deprecations)
 
 
-class TestMacroDeprecations(BaseTestDeprecations):
-    @property
-    def models(self):
-        return self.dir('boring-models')
-
-    @property
-    def project_config(self):
-        return {
-            'macro-paths': [self.dir('deprecated-macros')],
-        }
-
-    @use_profile('postgres')
-    def test_postgres_deprecations_fail(self):
-        with self.assertRaises(dbt.exceptions.CompilationException):
-            self.run_dbt(strict=True)
-
-    @use_profile('postgres')
-    def test_postgres_deprecations(self):
-        self.assertEqual(deprecations.active_deprecations, set())
-        self.run_dbt(strict=False)
-        expected = {'generate-schema-name-single-arg'}
-        self.assertEqual(expected, deprecations.active_deprecations)
-
-
 class TestMaterializationReturnDeprecation(BaseTestDeprecations):
     @property
     def models(self):

test/integration/024_custom_schema_test/custom-db-macros/custom_db.sql

@@ -0,0 +1,10 @@
+
+{% macro generate_database_name(database_name, node) %}
+    {% if database_name == 'alt' %}
+        {{ env_var('SNOWFLAKE_TEST_ALT_DATABASE') }}
+    {% elif database_name %}
+        {{ database_name }}
+    {% else %}
+        {{ target.database }}
+    {% endif %}
+{% endmacro %}

test/integration/024_custom_schema_test/db-models/view_1.sql

@@ -0,0 +1,3 @@
+
+
+select * from {{ target.schema }}.seed

test/integration/024_custom_schema_test/db-models/view_2.sql

@@ -0,0 +1,2 @@
+{{ config(database='alt') }}
+select * from {{ ref('view_1') }}