Merge pull request #2534 from fishtown-analytics/fix/dbt-modules-in-p…

…roject-dir

Make deps respect --project-dir

CHANGELOG.md

@@ -2,7 +2,7 @@
 
 ### Fixes
 - dbt compile and ls no longer create schemas if they don't already exist ([#2525](https://github.com/fishtown-analytics/dbt/issues/2525), [#2528](https://github.com/fishtown-analytics/dbt/pull/2528))
-
+- `dbt deps` now respects the `--project-dir` flag, so using `dbt deps --project-dir=/some/path` and then `dbt run --project-dir=/some/path` will properly find dependencies ([#2519](https://github.com/fishtown-analytics/dbt/issues/2519), [#2534](https://github.com/fishtown-analytics/dbt/pull/2534))
 
 ## dbt 0.17.0 (June 08, 2020)
 

core/dbt/main.py

@@ -907,6 +907,10 @@ def parse_args(args, cls=DBTArgumentParser):
     if hasattr(parsed, 'profiles_dir'):
         parsed.profiles_dir = os.path.expanduser(parsed.profiles_dir)
 
+    if getattr(parsed, 'project_dir', None) is not None:
+        expanded_user = os.path.expanduser(parsed.project_dir)
+        parsed.project_dir = os.path.abspath(expanded_user)
+
     if not hasattr(parsed, 'which'):
         # the user did not provide a valid subcommand. trigger the help message
         # and exit with a error

core/dbt/task/deps.py

@@ -11,7 +11,7 @@
 from dbt.logger import GLOBAL_LOGGER as logger
 from dbt.clients import system
 
-from dbt.task.base import BaseTask
+from dbt.task.base import BaseTask, move_to_nearest_project_dir
 
 
 class DepsTask(BaseTask):
@@ -65,3 +65,10 @@ def run(self):
                     package_name=package.name,
                     source_type=package.source_type(),
                     version=package.get_version())
+
+    @classmethod
+    def from_args(cls, args):
+        # deps needs to move to the project directory, as it does put files
+        # into the modules directory
+        move_to_nearest_project_dir(args)
+        return super().from_args(args)

core/dbt/task/rpc/deps.py

@@ -9,10 +9,9 @@
 
 
 def _clean_deps(config):
-    modules_dir = os.path.join(config.project_root, config.modules_path)
-    if os.path.exists(modules_dir):
-        shutil.rmtree(modules_dir)
-    os.makedirs(modules_dir)
+    if os.path.exists(config.modules_path):
+        shutil.rmtree(config.modules_path)
+    os.makedirs(config.modules_path)
 
 
 class RemoteDepsTask(

test/integration/004_simple_snapshot_test/test_simple_snapshot.py

@@ -406,6 +406,7 @@ def test__bigquery__snapshot_with_new_field(self):
 
 class TestCrossDBSnapshotFiles(DBTIntegrationTest):
     setup_alternate_db = True
+
     @property
     def schema(self):
         return "simple_snapshot_004"
@@ -465,6 +466,13 @@ def test__bigquery__cross_snapshot(self):
 class TestCrossSchemaSnapshotFiles(DBTIntegrationTest):
     NUM_SNAPSHOT_MODELS = 1
 
+    def setUp(self):
+        super().setUp()
+        self._created_schemas.add(
+            self._get_schema_fqn(self.default_database, self.target_schema()),
+        )
+
+
     @property
     def schema(self):
         return "simple_snapshot_004"

test/integration/005_simple_seed_test/test_simple_seed.py

@@ -54,8 +54,11 @@ def test_postgres_simple_seed_with_drop(self):
 class TestSimpleSeedCustomSchema(DBTIntegrationTest):
 
     def setUp(self):
-        DBTIntegrationTest.setUp(self)
+        super().setUp()
         self.run_sql_file("seed.sql")
+        self._created_schemas.add(
+            self._get_schema_fqn(self.default_database, self.custom_schema_name())
+        )
 
     @property
     def schema(self):
@@ -76,9 +79,12 @@ def project_config(self):
             },
         }
 
+    def custom_schema_name(self):
+        return "{}_{}".format(self.unique_schema(), 'custom_schema')
+
     @use_profile('postgres')
     def test_postgres_simple_seed_with_schema(self):
-        schema_name = "{}_{}".format(self.unique_schema(), 'custom_schema')
+        schema_name = self.custom_schema_name()
 
         results = self.run_dbt(["seed"])
         self.assertEqual(len(results),  1)

test/integration/006_simple_dependency_test/test_local_dependency.py

@@ -25,6 +25,15 @@ def base_schema(self):
     def configured_schema(self):
         return self.unique_schema() + '_configured'
 
+    def setUp(self):
+        super().setUp()
+        self._created_schemas.add(
+            self._get_schema_fqn(self.default_database, self.base_schema())
+        )
+        self._created_schemas.add(
+            self._get_schema_fqn(self.default_database, self.configured_schema())
+        )
+
     @property
     def packages_config(self):
         return {

test/integration/015_cli_invocation_tests/local_dependency/dbt_project.yml

@@ -0,0 +1,3 @@
+name: dependency
+version: '1.0.0'
+config-version: 2

test/integration/015_cli_invocation_tests/local_dependency/macros/my_test_macro.sql

@@ -0,0 +1,2 @@
+{% macro some_macro() %}
+{% endmacro %}

test/integration/015_cli_invocation_tests/models/subdir1/subdir2/model.sql

@@ -4,4 +4,7 @@
     )
 }}
 
+{# we don't care what, just do anything that will fail without "dbt deps" #}
+{% do dependency.some_macro() %}
+
 select 1 as id

test/integration/015_cli_invocation_tests/test_cli_invocation.py

@@ -6,16 +6,28 @@
 
 
 class ModelCopyingIntegrationTest(DBTIntegrationTest):
+
     def _symlink_test_folders(self):
         # dbt's normal symlink behavior breaks this test, so special-case it
         for entry in os.listdir(self.test_original_source_path):
             src = os.path.join(self.test_original_source_path, entry)
             tst = os.path.join(self.test_root_dir, entry)
             if entry == 'models':
                 shutil.copytree(src, tst)
+            elif entry == 'local_dependency':
+                continue
             elif os.path.isdir(entry) or entry.endswith('.sql'):
                 os.symlink(src, tst)
 
+    @property
+    def packages_config(self):
+        path = os.path.join(self.test_original_source_path, 'local_dependency')
+        return {
+            'packages': [{
+                'local': path,
+            }],
+        }
+
 
 class TestCLIInvocation(ModelCopyingIntegrationTest):
 
@@ -33,13 +45,15 @@ def models(self):
 
     @use_profile('postgres')
     def test_postgres_toplevel_dbt_run(self):
+        self.run_dbt(['deps'])
         results = self.run_dbt(['run'])
         self.assertEqual(len(results), 1)
         self.assertTablesEqual("seed", "model")
 
     @use_profile('postgres')
     def test_postgres_subdir_dbt_run(self):
         os.chdir(os.path.join(self.models, "subdir1"))
+        self.run_dbt(['deps'])
 
         results = self.run_dbt(['run'])
         self.assertEqual(len(results), 1)
@@ -51,8 +65,8 @@ class TestCLIInvocationWithProfilesDir(ModelCopyingIntegrationTest):
     def setUp(self):
         super().setUp()
 
-        self.run_sql("DROP SCHEMA IF EXISTS {} CASCADE;".format(self.custom_schema))
-        self.run_sql("CREATE SCHEMA {};".format(self.custom_schema))
+        self.run_sql(f"DROP SCHEMA IF EXISTS {self.custom_schema} CASCADE;")
+        self.run_sql(f"CREATE SCHEMA {self.custom_schema};")
 
         # the test framework will remove this in teardown for us.
         if not os.path.exists('./dbt-profile'):
@@ -63,6 +77,10 @@ def setUp(self):
 
         self.run_sql_file("seed_custom.sql")
 
+    def tearDown(self):
+        self.run_sql(f"DROP SCHEMA IF EXISTS {self.custom_schema} CASCADE;")
+        super().tearDown()
+
     def custom_profile_config(self):
         return {
             'config': {
@@ -99,6 +117,7 @@ def models(self):
 
     @use_profile('postgres')
     def test_postgres_toplevel_dbt_run_with_profile_dir_arg(self):
+        self.run_dbt(['deps'])
         results = self.run_dbt(['run', '--profiles-dir', 'dbt-profile'], profiles_dir=False)
         self.assertEqual(len(results), 1)
 
@@ -136,6 +155,14 @@ def test_postgres_dbt_commands_with_randomdir_as_project_dir(self):
             self._run_simple_dbt_commands(workdir)
             os.chdir(workdir)
 
+    @use_profile('postgres')
+    def test_postgres_dbt_commands_with_relative_dir_as_project_dir(self):
+        workdir = os.getcwd()
+        with tempfile.TemporaryDirectory() as tmpdir:
+            os.chdir(tmpdir)
+            self._run_simple_dbt_commands(os.path.relpath(workdir, tmpdir))
+            os.chdir(workdir)
+
     def _run_simple_dbt_commands(self, project_dir):
         self.run_dbt(['deps', '--project-dir', project_dir])
         self.run_dbt(['seed', '--project-dir', project_dir])