Feature/scoped macros

dbt/compilation.py

@@ -7,6 +7,7 @@
 from dbt.source import Source
 from dbt.utils import find_model_by_fqn, find_model_by_name, dependency_projects, split_path, This, Var, compiler_error, to_string
 from dbt.linker import Linker
+from dbt.runtime import RuntimeContext
 import dbt.targets
 import dbt.templates
 import time
@@ -176,6 +177,8 @@ def wrapped_do_ref(*args):
         return wrapped_do_ref
 
     def get_context(self, linker, model,  models, add_dependency=False):
+        runtime = RuntimeContext(model=model)
+
         context = self.project.context()
 
         # built-ins
@@ -192,11 +195,20 @@ def get_context(self, linker, model,  models, add_dependency=False):
         # add in context from run target
         context.update(self.target.context)
 
+        runtime.update_global(context)
+
         # add in macros (can we cache these somehow?)
-        for macro_name, macro in self.macro_generator(context):
-            context[macro_name] = macro
+        for macro_data in self.macro_generator(context):
+            macro = macro_data["macro"]
+            macro_name = macro_data["name"]
+            project = macro_data["project"]
+
+            runtime.update_package(project['name'], {macro_name: macro})
 
-        return context
+            if project['name'] == self.project['name']:
+                runtime.update_global({macro_name: macro})
+
+        return runtime
 
     def compile_model(self, linker, model, models, add_dependency=True):
         try:
@@ -211,6 +223,8 @@ def compile_model(self, linker, model, models, add_dependency=True):
             rendered = template.render(context)
         except jinja2.exceptions.TemplateSyntaxError as e:
             compiler_error(model, str(e))
+        except jinja2.exceptions.UndefinedError as e:
+            compiler_error(model, str(e))
 
         return rendered
 

dbt/deprecations.py

@@ -4,24 +4,29 @@ class DBTDeprecation(object):
     name = None
     description = None
 
-    def show(self):
+    def show(self, *args, **kwargs):
         if self.name not in active_deprecations:
-            print("* Deprecation Warning: {}".format(self.description))
+            desc = self.description.format(**kwargs)
+            print("* Deprecation Warning: {}\n".format(desc))
             active_deprecations.add(self.name)
 
-
 class DBTRunTargetDeprecation(DBTDeprecation):
     name = 'run-target'
     description = """profiles.yml configuration option 'run-target' is deprecated. Please use 'target' instead. 
   The 'run-target' option will be removed (in favor of 'target') in DBT version 0.6.0"""
 
+class DBTInvalidPackageName(DBTDeprecation):
+    name = 'invalid-package-name'
+    description = """The package name '{package_name}' is not valid. Package names must only contain letters and underscores.
+  Packages with invalid names will fail to compile in DBT version 0.6.0"""
+
 
-def warn(name):
+def warn(name, *args, **kwargs):
     if name not in deprecations:
         # this should (hopefully) never happen
         raise RuntimeError("Error showing deprecation warning: {}".format(name))
 
-    deprecations[name].show()
+    deprecations[name].show(*args, **kwargs)
 
 
 # these are globally available
@@ -30,7 +35,8 @@ def warn(name):
 active_deprecations = set()
 
 deprecations_list = [
-    DBTRunTargetDeprecation()
+    DBTRunTargetDeprecation(),
+    DBTInvalidPackageName()
 ]
 
 deprecations = {d.name : d for d in deprecations_list}

dbt/model.py

@@ -623,11 +623,14 @@ def __init__(self, project, target_dir, rel_filepath, own_project):
 
     def get_macros(self, ctx):
         env = jinja2.Environment()
-        template = env.from_string(self.contents, globals=ctx)
+        try:
+            template = env.from_string(self.contents, globals=ctx)
+        except jinja2.exceptions.TemplateSyntaxError as e:
+            compiler_error(self, str(e))
 
         for key, item in template.module.__dict__.items():
             if type(item) == jinja2.runtime.Macro:
-                yield key, item
+                yield {"project": self.own_project, "name": key, "macro": item}
 
     def __repr__(self):
         return "<Macro {}.{}: {}>".format(self.project['name'], self.name, self.filepath)

dbt/project.py

@@ -4,6 +4,7 @@
 import copy
 import sys
 import hashlib
+import re
 import dbt.deprecations
 
 default_project_cfg = {
@@ -75,6 +76,15 @@ def handle_deprecations(self):
             dbt.deprecations.warn('run-target')
             self.cfg['target'] = self.cfg['run-target']
 
+        if not self.is_valid_package_name():
+            dbt.deprecations.warn('invalid-package-name', package_name = self['name'])
+
+    def is_valid_package_name(self):
+        if re.match(r"^[^\d\W]\w*\Z", self['name']):
+            return True
+        else:
+            return False
+
     def run_environment(self):
         target_name = self.cfg['target']
         return self.cfg['outputs'][target_name]

dbt/runtime.py

@@ -0,0 +1,38 @@
+from dbt.utils import compiler_error
+
+class RuntimeContext(dict):
+    def __init__(self, model=None, *args, **kwargs):
+        super(RuntimeContext, self).__init__(*args, **kwargs)
+
+        self.model = model
+
+    def __getattr__(self, attr):
+        if attr in self:
+            return self.get(attr)
+        else:
+            compiler_error(self.model, "'{}' is undefined".format(attr))
+
+    def __setattr__(self, key, value):
+        self.__setitem__(key, value)
+
+    def __setitem__(self, key, value):
+        super(RuntimeContext, self).__setitem__(key, value)
+        self.__dict__.update({key: value})
+
+    def __delattr__(self, item):
+        self.__delitem__(item)
+
+    def __delitem__(self, key):
+        super(RuntimeContext, self).__delitem__(key)
+        del self.__dict__[key]
+
+    def update_global(self, data):
+        self.update(data)
+
+    def update_package(self, pkg_name, data):
+        if pkg_name not in self:
+            ctx = RuntimeContext(model=self.model)
+
+            self[pkg_name] = ctx
+
+        self[pkg_name].update(data)

dbt/source.py

@@ -16,6 +16,10 @@ def __init__(self, project, own_project=None):
     def find(self, source_paths, file_pattern):
         """returns abspath, relpath, filename of files matching file_regex in source_paths"""
         found = []
+
+        if type(source_paths) not in (list, tuple):
+            source_paths = [source_paths]
+
         for source_path in source_paths:
             root_path = os.path.join(self.own_project_root, source_path)
             for root, dirs, files in os.walk(root_path):

test/integration/006_simple_dependency_test/test_simple_dependency_with_configs.py

@@ -20,7 +20,7 @@ class TestSimpleDependencyWithConfigs(BaseTestSimpleDependencyWithConfigs):
     def project_config(self):
         return {
             "models": {
-                "DBT Integration Project": {
+                "dbt_integration_project": {
                     'vars': {
                         'bool_config': True
                     }
@@ -50,7 +50,7 @@ def project_config(self):
         return {
             "models": {
                 # project-level configs
-                "DBT Integration Project": {
+                "dbt_integration_project": {
                     "vars": {
                         "config_1": "abc",
                         "config_2": "def",
@@ -83,7 +83,7 @@ class TestSimpleDependencyWithModelSpecificOverriddenConfigs(BaseTestSimpleDepen
     def project_config(self):
         return {
             "models": {
-                "DBT Integration Project": {
+                "dbt_integration_project": {
                     "config": {
                         # model-level configs
                         "vars": {
@@ -118,7 +118,7 @@ class TestSimpleDependencyWithModelSpecificOverriddenConfigs(BaseTestSimpleDepen
     def project_config(self):
         return {
             "models": {
-                "DBT Integration Project": {
+                "dbt_integration_project": {
                     # disable config model, but supply vars
                     "config": {
                         "enabled": False,

test/integration/016_macro_tests/bad-macros/bad_macros.sql

@@ -0,0 +1,7 @@
+
+{% macro invalid_macro() %}
+
+    select
+        '{{ foo2 }}' as foo2,
+        '{{ bar2 }}' as bar2
+

test/integration/016_macro_tests/bad-models/dep_macro.sql

@@ -0,0 +1,5 @@
+
+-- try to globally reference a dependency macro
+{{
+    do_something("arg1", "arg2")
+}}

test/integration/016_macro_tests/macros/my_macros.sql

@@ -0,0 +1,8 @@
+
+{% macro do_something2(foo2, bar2) %}
+
+    select
+        '{{ foo2 }}' as foo2,
+        '{{ bar2 }}' as bar2
+
+{% endmacro %}

test/integration/016_macro_tests/models/dep_macro.sql

@@ -0,0 +1,4 @@
+
+{{
+    dbt_integration_project.do_something("arg1", "arg2")
+}}

test/integration/016_macro_tests/models/local_macro.sql

@@ -0,0 +1,12 @@
+
+{{
+    do_something2("arg1", "arg2")
+}}
+
+union all
+
+{{
+    test.do_something2("arg3", "arg4")
+}}
+
+

test/integration/016_macro_tests/seed.sql

@@ -0,0 +1,18 @@
+create table test_macros_016.expected_dep_macro (
+	foo TEXT,
+	bar TEXT
+);
+
+create table test_macros_016.expected_local_macro (
+	foo2 TEXT,
+	bar2 TEXT
+);
+
+insert into test_macros_016.expected_dep_macro (foo, bar)
+values ('arg1', 'arg2');
+
+insert into test_macros_016.expected_local_macro (foo2, bar2)
+values ('arg1', 'arg2'), ('arg3', 'arg4');
+
+
+

test/integration/016_macro_tests/test_macros.py

@@ -0,0 +1,90 @@
+from test.integration.base import DBTIntegrationTest
+
+class TestMacros(DBTIntegrationTest):
+
+    def setUp(self):
+        DBTIntegrationTest.setUp(self)
+        self.run_sql_file("test/integration/016_macro_tests/seed.sql")
+
+    @property
+    def schema(self):
+        return "test_macros_016"
+
+    @property
+    def models(self):
+        return "test/integration/016_macro_tests/models"
+
+    @property
+    def project_config(self):
+        return {
+            "macro-paths": ["test/integration/016_macro_tests/macros"],
+            "repositories": [
+                'https://github.com/fishtown-analytics/dbt-integration-project'
+            ]
+        }
+
+    def test_working_macros(self):
+        self.run_dbt(["deps"])
+        self.run_dbt(["run"])
+
+        self.assertTablesEqual("expected_dep_macro","dep_macro")
+        self.assertTablesEqual("expected_local_macro","local_macro")
+
+
+class TestInvalidMacros(DBTIntegrationTest):
+
+    def setUp(self):
+        DBTIntegrationTest.setUp(self)
+
+    @property
+    def schema(self):
+        return "test_macros_016"
+
+    @property
+    def models(self):
+        return "test/integration/016_macro_tests/models"
+
+    @property
+    def project_config(self):
+        return {
+            "macro-paths": ["test/integration/016_macro_tests/bad-macros"]
+        }
+
+    def test_invalid_macro(self):
+
+        try:
+            self.run_dbt(["compile"])
+            self.assertTrue(False, 'compiling bad macro should raise a runtime error')
+        except RuntimeError as e:
+            pass
+
+class TestMisusedMacros(DBTIntegrationTest):
+
+    def setUp(self):
+        DBTIntegrationTest.setUp(self)
+
+    @property
+    def schema(self):
+        return "test_macros_016"
+
+    @property
+    def models(self):
+        return "test/integration/016_macro_tests/bad-models"
+
+    @property
+    def project_config(self):
+        return {
+            "macro-paths": ["test/integration/016_macro_tests/macros"],
+            "repositories": [
+                'https://github.com/fishtown-analytics/dbt-integration-project'
+            ]
+        }
+
+    def test_working_macros(self):
+        self.run_dbt(["deps"])
+
+        try:
+            self.run_dbt(["run"])
+            self.assertTrue(False, 'invoked a package macro from global scope')
+        except RuntimeError as e:
+            pass