From 8145832b7b6ced1b4202f931f04f8e546ce25b35 Mon Sep 17 00:00:00 2001 From: Emily Rockman Date: Fri, 25 Mar 2022 13:35:51 -0500 Subject: [PATCH 01/21] initial pass at source config test w/o overrides --- .../functional/sources/test_source_configs.py | 594 ++++++++++++++++++ 1 file changed, 594 insertions(+) create mode 100644 tests/functional/sources/test_source_configs.py diff --git a/tests/functional/sources/test_source_configs.py b/tests/functional/sources/test_source_configs.py new file mode 100644 index 00000000000..f4b3aabff50 --- /dev/null +++ b/tests/functional/sources/test_source_configs.py @@ -0,0 +1,594 @@ +import pytest +from dbt.contracts.graph.model_config import SourceConfig +from dbt.contracts.graph.unparsed import FreshnessThreshold, Time, TimePeriod, Quoting +from dbt.exceptions import CompilationException + +from dbt.tests.util import run_dbt, update_config_file, get_manifest + + +class SourceConfigTests: + @pytest.fixture(scope="class", autouse=True) + def setUp(self): + pytest.expected_config = SourceConfig( + enabled=True, + # TODO: uncomment all this once it's added to SourceConfig, throws error right now + # quoting = Quoting(database=False, schema=False, identifier=False, column=False) + # freshness = FreshnessThreshold( + # warn_after=Time(count=12, period=TimePeriod.hour), + # error_after=Time(count=24, period=TimePeriod.hour), + # filter=None + # ) + # loader = "a_loader" + # loaded_at_field = some_column + # database = custom_database + # schema = custom_schema + # # identifier = "seed" #this doesnt seems to be supported now? + # meta = {'languages': ['python']} + # tags = ["important_tag"] + ) + + +models__schema_yml = """version: 2 + +sources: + - name: test_source + tables: + - name: test_table + - name: other_source + tables: + - name: test_table +""" + + +# Test enabled config in dbt_project.yml +# expect pass, already implemented +class TestSourceEnabledConfigProjectLevel(SourceConfigTests): + @pytest.fixture(scope="class") + def models(self): + return { + "schema.yml": models__schema_yml, + } + + @pytest.fixture(scope="class") + def project_config_update(self): + return { + "sources": { + "test": { + "test_source": { + "enabled": True, + }, + } + } + } + + def test_enabled_source_config_dbt_project(self, project): + run_dbt(["compile"]) + manifest = get_manifest(project.project_root) + assert "source.test.test_source.test_table" in manifest.sources + + new_enabled_config = { + "sources": { + "test": { + "test_source": { + "enabled": False, + }, + } + } + } + update_config_file(new_enabled_config, project.project_root, "dbt_project.yml") + run_dbt(["compile"]) + manifest = get_manifest(project.project_root) + + assert ( + "source.test.test_source.test_table" not in manifest.sources + ) # or should it be there with enabled: false?? + assert "source.test.other_source.test_table" in manifest.sources + + +disabled_source_level__schema_yml = """version: 2 + +sources: + - name: test_source + config: + enabled: False + tables: + - name: test_table + - name: disabled_test_table +""" + + +# Test enabled config at sources level in yml file +# expect fail - not implemented +class TestConfigYamlSourceLevel(SourceConfigTests): + @pytest.fixture(scope="class") + def models(self): + return { + "schema.yml": disabled_source_level__schema_yml, + } + + @pytest.mark.xfail + def test_source_config_yaml_source_level(self, project): + run_dbt(["compile"]) + manifest = get_manifest(project.project_root) + assert "source.test.test_source.test_table" not in manifest.sources + assert "source.test.test_source.disabled_test_table" not in manifest.sources + + +disabled_source_table__schema_yml = """version: 2 + +sources: + - name: test_source + tables: + - name: test_table + - name: disabled_test_table + config: + enabled: False +""" + + +# Test enabled config at source table level in yaml file +# expect fail - not implemented +class TestConfigYamlSourceTable(SourceConfigTests): + @pytest.fixture(scope="class") + def models(self): + return { + "schema.yml": disabled_source_table__schema_yml, + } + + @pytest.mark.xfail + def test_source_config_yaml_source_table(self, project): + run_dbt(["compile"]) + manifest = get_manifest(project.project_root) + assert "source.test.test_source.test_table" in manifest.sources + assert "source.test.test_source.disabled_test_table" not in manifest.sources + + +# Test all configs other than enabled in dbt_project.yml +# expect fail - not implemented +class TestAllConfigsProjectLevel(SourceConfigTests): + @pytest.fixture(scope="class") + def models(self): + return {"schema.yml": models__schema_yml} + + @pytest.fixture(scope="class") + def project_config_update(self): + return { + "sources": { + "enabled": True, + "quoting": { + "database": False, + "schema": False, + "identifier": False, + "column": False, + }, + "freshness": { + "error_after": {"count": 24, "period": "hour"}, + "warn_after": {"count": 12, "period": "hour"}, + }, + "loader": "a_loader", + "loaded_at_field": "some_column", + "database": "custom_database", + "schema": "custom_schema", + # identifier: "seed" #this doesnt seems to be supported now? + "meta": {"languages": ["python"]}, + "tags": ["important_tag"], + } + } + + @pytest.mark.xfail + def test_source_all_configs_dbt_project(self, project): + run_dbt(["compile"]) + manifest = get_manifest(project.project_root) + assert "source.test.test_source.test_table" in manifest.sources + config = manifest.sources.get("source.test.test_source.test_table").config + + assert isinstance(config, SourceConfig) + + assert config == pytest.expected_config + + +configs_source_level__schema_yml = """version: 2 + +sources: + - name: test_source + config: + enabled: True, + quoting: + database: False + schema: False + identifier: False + column: False + freshness: + error_after: {count: 24, period: hour} + warn_after: {count: 12, period: hour} + loader: "a_loader" + loaded_at_field: some_column + database: custom_database + schema: custom_schema + # identifier: "seed" #this doesnt seems to be supported now? + meta: {'languages': ['python']} + tags: ["important_tag"] + tables: + - name: test_table + - name: other_test_table +""" + + +# Test configs other than enabled at sources level in yaml file +# **currently passes since enabled is all that ends up in the +# node.config since it's the only thing implemented +class TestAllConfigsSourceLevel(SourceConfigTests): + @pytest.fixture(scope="class") + def models(self): + return {"schema.yml": configs_source_level__schema_yml} + + def test_source_all_configs_source_level(self, project): + run_dbt(["compile"]) + manifest = get_manifest(project.project_root) + assert "source.test.test_source.test_table" in manifest.sources + assert "source.test.test_source.other_test_table" in manifest.sources + config_test_table = manifest.sources.get("source.test.test_source.test_table").config + config_other_test_table = manifest.sources.get( + "source.test.test_source.other_test_table" + ).config + + assert isinstance(config_test_table, SourceConfig) + assert isinstance(config_other_test_table, SourceConfig) + + assert config_test_table == config_other_test_table + assert config_test_table == pytest.expected_config + + +configs_source_table__schema_yml = """version: 2 + +sources: + - name: test_source + tables: + - name: test_table + config: + enabled: True, + quoting: + database: False + schema: False + identifier: False + column: False + freshness: + error_after: {count: 24, period: hour} + warn_after: {count: 12, period: hour} + loader: "a_loader" + loaded_at_field: some_column + database: custom_database + schema: custom_schema + # identifier: "seed" #this doesnt seems to be supported now? + meta: {'languages': ['python']} + tags: ["important_tag"] + - name: other_test_table +""" + + +# Test configs other than enabled at source table level in yml file +# expect fail - not implemented +class TestSourceAllConfigsSourceTable(SourceConfigTests): + @pytest.fixture(scope="class") + def models(self): + return {"schema.yml": configs_source_table__schema_yml} + + @pytest.mark.xfail + def test_source_all_configs_source_table(self, project): + run_dbt(["compile"]) + manifest = get_manifest(project.project_root) + assert "source.test.test_source.test_table" in manifest.sources + assert "source.test.test_source.other_test_table" in manifest.sources + config_test_table = manifest.sources.get("source.test.test_source.test_table").config + config_other_test_table = manifest.sources.get( + "source.test.test_source.other_test_table" + ).config + + assert isinstance(config_test_table, SourceConfig) + assert isinstance(config_other_test_table, SourceConfig) + + assert config_test_table != config_other_test_table + assert config_test_table == pytest.expected_config + + +all_configs_everywhere__schema_yml = """version: 2 + +sources: + - name: test_source + config: + tags: "source_level_important_tag", + tables: + - name: test_table + config: + enabled: True, + quoting: + database: False + schema: False + identifier: False + column: False + freshness: + error_after: {count: 24, period: hour} + warn_after: {count: 12, period: hour} + loader: "a_loader" + loaded_at_field: some_column + database: custom_database + schema: custom_schema + # identifier: "seed" #this doesnt seems to be supported now? + meta: {'languages': ['python']} + tags: ["important_tag"] + - name: other_test_table +""" + + +# Test inheritence - set configs atproject, source, and source-table level - expect source-table level to win +# expect fail - not implemented +class TestSourceConfigsInheritence1(SourceConfigTests): + @pytest.fixture(scope="class") + def models(self): + return {"schema.yml": all_configs_everywhere__schema_yml} + + @pytest.fixture(scope="class") + def project_config_update(self): + return {"sources": {"tags": "project_level_important_tag"}} + + @pytest.mark.xfail + def test_source_all_configs_source_table(self, project): + run_dbt(["compile"]) + manifest = get_manifest(project.project_root) + assert "source.test.test_source.test_table" in manifest.sources + assert "source.test.test_source.other_test_table" in manifest.sources + config_test_table = manifest.sources.get("source.test.test_source.test_table").config + config_other_test_table = manifest.sources.get( + "source.test.test_source.other_test_table" + ).config + + assert isinstance(config_test_table, SourceConfig) + assert isinstance(config_other_test_table, SourceConfig) + + assert config_test_table != config_other_test_table + assert config_test_table == pytest.expected_config + + expected_source_level_config = SourceConfig( + enabled=True, + # "tags" = "source_level_important_tag" #TODO: update after SourceConfigs gets updated + ) + + assert config_other_test_table == expected_source_level_config + + +all_configs_not_table_schema_yml = """version: 2 + +sources: + - name: test_source + config: + enabled: True, + quoting: + database: False + schema: False + identifier: False + column: False + freshness: + error_after: {count: 24, period: hour} + warn_after: {count: 12, period: hour} + loader: "a_loader" + loaded_at_field: some_column + database: custom_database + schema: custom_schema + # identifier: "seed" #this doesnt seems to be supported now? + meta: {'languages': ['python']} + tags: ["important_tag"] + tables: + - name: test_table + - name: other_test_table +""" + + +# Test inheritence - set configs at project and source level - expect source level to win +# expect fail - not implemented +class TestSourceConfigsInheritence2(SourceConfigTests): + @pytest.fixture(scope="class") + def models(self): + return {"schema.yml": all_configs_not_table_schema_yml} + + @pytest.fixture(scope="class") + def project_config_update(self): + return {"sources": {"tags": "project_level_important_tag"}} + + @pytest.mark.xfail + def test_source_two_configs_source_level(self, project): + run_dbt(["compile"]) + manifest = get_manifest(project.project_root) + assert "source.test.test_source.test_table" in manifest.sources + assert "source.test.test_source.other_test_table" in manifest.sources + config_test_table = manifest.sources.get("source.test.test_source.test_table").config + config_other_test_table = manifest.sources.get( + "source.test.test_source.other_test_table" + ).config + + assert isinstance(config_test_table, SourceConfig) + assert isinstance(config_other_test_table, SourceConfig) + + assert config_test_table == config_other_test_table + assert config_test_table == pytest.expected_config + + +all_configs_everywhere__schema_yml = """version: 2 + +sources: + - name: test_source + tables: + - name: test_table + config: + enabled: True, + quoting: + database: False + schema: False + identifier: False + column: False + freshness: + error_after: {count: 24, period: hour} + warn_after: {count: 12, period: hour} + loader: "a_loader" + loaded_at_field: some_column + database: custom_database + schema: custom_schema + # identifier: "seed" #this doesnt seems to be supported now? + meta: {'languages': ['python']} + tags: ["important_tag"] + - name: other_test_table +""" + + +# Test inheritence - set configs at project and source-table level - expect source-table level to win +# expect fail - not implemented +class TestSourceConfigsInheritence3(SourceConfigTests): + @pytest.fixture(scope="class") + def models(self): + return {"schema.yml": all_configs_everywhere__schema_yml} + + @pytest.fixture(scope="class") + def project_config_update(self): + return {"sources": {"tags": "project_level_important_tag"}} + + @pytest.mark.xfail + def test_source_two_configs_source_table(self, project): + run_dbt(["compile"]) + manifest = get_manifest(project.project_root) + assert "source.test.test_source.test_table" in manifest.sources + assert "source.test.test_source.other_test_table" in manifest.sources + config_test_table = manifest.sources.get("source.test.test_source.test_table").config + config_other_test_table = manifest.sources.get( + "source.test.test_source.other_test_table" + ).config + + assert isinstance(config_test_table, SourceConfig) + assert isinstance(config_other_test_table, SourceConfig) + + assert config_test_table != config_other_test_table + assert config_test_table == pytest.expected_config + + expected_project_level_config = SourceConfig( + enabled=True, + # tags = "project_level_important_tag", # TODO: uncomment these once SourceConfig is updated + ) + + assert config_other_test_table == expected_project_level_config + + +configs_as_properties__schema_yml = """version: 2 + +sources: + - name: test_source + quoting: + database: False + schema: False + identifier: False + column: False + freshness: + error_after: {count: 24, period: hour} + warn_after: {count: 12, period: hour} + loader: "a_loader" + loaded_at_field: some_column + database: custom_database + schema: custom_schema + # identifier: "seed" #this doesnt seems to be supported now? + meta: {'languages': ['python']} + tags: ["important_tag"] + tables: + - name: test_table +""" + + +# Check backwards compatibility of setting configs as properties at top level +# expect pass since the properties don't get copied to the node.config yet +class TestSourceBackwardsCompatibility(SourceConfigTests): + @pytest.fixture(scope="class") + def models(self): + return {"schema.yml": configs_as_properties__schema_yml} + + def test_source_configs_as_properties(self, project): + run_dbt(["compile"]) + manifest = get_manifest(project.project_root) + assert "source.test.test_source.test_table" in manifest.sources + config_test_table = manifest.sources.get("source.test.test_source.test_table").config + + # this is new functionality - but it currently passes since SourceConfig is not updated + # and is commented out in the setup becuse it is not updated with new configs + # even when properties are defined at teh top level they should end up on the node.config + assert isinstance(config_test_table, SourceConfig) + assert config_test_table == pytest.expected_config + + # this is existing functionality - should always pass + properties_test_table = manifest.sources.get("source.test.test_source.test_table") + assert properties_test_table.quoting == Quoting( + database=False, schema=False, identifier=False, column=False + ) + assert properties_test_table.freshness == FreshnessThreshold( + warn_after=Time(count=12, period=TimePeriod.hour), + error_after=Time(count=24, period=TimePeriod.hour), + filter=None, + ) + assert properties_test_table.loader == "a_loader" + assert properties_test_table.loaded_at_field == "some_column" + assert properties_test_table.database == "custom_database" + assert properties_test_table.schema == "custom_schema" + # assert properties_test_table.identifier == "seed" + assert properties_test_table.meta == {} # TODO: why is this blank + assert properties_test_table.tags == ["important_tag"] + + +configs_properites__schema_yml = """version: 2 + +sources: + - name: test_source + quoting: + database: False + schema: False + identifier: False + column: False + freshness: + error_after: {count: 24, period: hour} + warn_after: {count: 12, period: hour} + loader: "a_loader" + loaded_at_field: some_column + database: custom_database + schema: custom_schema + # identifier: "seed" #this doesnt seems to be supported now? + meta: {'languages': ['python']} + tags: ["important_tag"] + config: + enabled: True, + quoting: + database: False + schema: False + identifier: False + column: False + freshness: + error_after: {count: 24, period: hour} + warn_after: {count: 12, period: hour} + loader: "a_loader" + loaded_at_field: some_column + database: custom_database + schema: custom_schema + # identifier: "seed" #this doesnt seems to be supported now? + meta: {'languages': ['python']} + tags: ["important_tag"] + tables: + - name: test_table + - name: other_test_table +""" + + +# Raise an error when properties are set at top level and also as configs +class TestErrorSourceConfigProperty(SourceConfigTests): + @pytest.fixture(scope="class") + def models(self): + return {"schema.yml": configs_properites__schema_yml} + + @pytest.mark.xfail + def test_error_source_configs_properties(self, project): + # TODO: update below with correct exception/text/etc. This is placeholder. + with pytest.raises(CompilationException) as exc: + run_dbt(["compile"]) + + assert "???" in str(exc.value) From c22b4249d26e3217f62cf3c34f6fb972b880f1da Mon Sep 17 00:00:00 2001 From: Emily Rockman Date: Sat, 26 Mar 2022 09:01:12 -0500 Subject: [PATCH 02/21] Update tests/functional/sources/test_source_configs.py Co-authored-by: Jeremy Cohen --- tests/functional/sources/test_source_configs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/functional/sources/test_source_configs.py b/tests/functional/sources/test_source_configs.py index f4b3aabff50..67468172797 100644 --- a/tests/functional/sources/test_source_configs.py +++ b/tests/functional/sources/test_source_configs.py @@ -361,7 +361,7 @@ def test_source_all_configs_source_table(self, project): sources: - name: test_source config: - enabled: True, + enabled: True quoting: database: False schema: False From 8f71d292ad4dae3405cab849580be6081c45b0cf Mon Sep 17 00:00:00 2001 From: Emily Rockman Date: Sat, 26 Mar 2022 09:01:25 -0500 Subject: [PATCH 03/21] Update tests/functional/sources/test_source_configs.py Co-authored-by: Jeremy Cohen --- tests/functional/sources/test_source_configs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/functional/sources/test_source_configs.py b/tests/functional/sources/test_source_configs.py index 67468172797..32808cadb55 100644 --- a/tests/functional/sources/test_source_configs.py +++ b/tests/functional/sources/test_source_configs.py @@ -300,7 +300,7 @@ def test_source_all_configs_source_table(self, project): tables: - name: test_table config: - enabled: True, + enabled: True quoting: database: False schema: False From ca01d5c6db4eb660aff2ea54bb2ff707a8992a62 Mon Sep 17 00:00:00 2001 From: Emily Rockman Date: Sat, 26 Mar 2022 11:26:05 -0500 Subject: [PATCH 04/21] tweaks from feedback --- .../functional/sources/test_source_configs.py | 216 +++++++++++++----- 1 file changed, 157 insertions(+), 59 deletions(-) diff --git a/tests/functional/sources/test_source_configs.py b/tests/functional/sources/test_source_configs.py index 32808cadb55..1205dbfdf0a 100644 --- a/tests/functional/sources/test_source_configs.py +++ b/tests/functional/sources/test_source_configs.py @@ -22,7 +22,6 @@ def setUp(self): # loaded_at_field = some_column # database = custom_database # schema = custom_schema - # # identifier = "seed" #this doesnt seems to be supported now? # meta = {'languages': ['python']} # tags = ["important_tag"] ) @@ -62,7 +61,7 @@ def project_config_update(self): } def test_enabled_source_config_dbt_project(self, project): - run_dbt(["compile"]) + run_dbt(["parse"]) manifest = get_manifest(project.project_root) assert "source.test.test_source.test_table" in manifest.sources @@ -76,7 +75,7 @@ def test_enabled_source_config_dbt_project(self, project): } } update_config_file(new_enabled_config, project.project_root, "dbt_project.yml") - run_dbt(["compile"]) + run_dbt(["parse"]) manifest = get_manifest(project.project_root) assert ( @@ -108,7 +107,7 @@ def models(self): @pytest.mark.xfail def test_source_config_yaml_source_level(self, project): - run_dbt(["compile"]) + run_dbt(["parse"]) manifest = get_manifest(project.project_root) assert "source.test.test_source.test_table" not in manifest.sources assert "source.test.test_source.disabled_test_table" not in manifest.sources @@ -137,7 +136,7 @@ def models(self): @pytest.mark.xfail def test_source_config_yaml_source_table(self, project): - run_dbt(["compile"]) + run_dbt(["parse"]) manifest = get_manifest(project.project_root) assert "source.test.test_source.test_table" in manifest.sources assert "source.test.test_source.disabled_test_table" not in manifest.sources @@ -169,7 +168,6 @@ def project_config_update(self): "loaded_at_field": "some_column", "database": "custom_database", "schema": "custom_schema", - # identifier: "seed" #this doesnt seems to be supported now? "meta": {"languages": ["python"]}, "tags": ["important_tag"], } @@ -177,7 +175,7 @@ def project_config_update(self): @pytest.mark.xfail def test_source_all_configs_dbt_project(self, project): - run_dbt(["compile"]) + run_dbt(["parse"]) manifest = get_manifest(project.project_root) assert "source.test.test_source.test_table" in manifest.sources config = manifest.sources.get("source.test.test_source.test_table").config @@ -192,7 +190,7 @@ def test_source_all_configs_dbt_project(self, project): sources: - name: test_source config: - enabled: True, + enabled: True quoting: database: False schema: False @@ -205,7 +203,6 @@ def test_source_all_configs_dbt_project(self, project): loaded_at_field: some_column database: custom_database schema: custom_schema - # identifier: "seed" #this doesnt seems to be supported now? meta: {'languages': ['python']} tags: ["important_tag"] tables: @@ -223,7 +220,7 @@ def models(self): return {"schema.yml": configs_source_level__schema_yml} def test_source_all_configs_source_level(self, project): - run_dbt(["compile"]) + run_dbt(["parse"]) manifest = get_manifest(project.project_root) assert "source.test.test_source.test_table" in manifest.sources assert "source.test.test_source.other_test_table" in manifest.sources @@ -246,7 +243,7 @@ def test_source_all_configs_source_level(self, project): tables: - name: test_table config: - enabled: True, + enabled: True quoting: database: False schema: False @@ -259,7 +256,6 @@ def test_source_all_configs_source_level(self, project): loaded_at_field: some_column database: custom_database schema: custom_schema - # identifier: "seed" #this doesnt seems to be supported now? meta: {'languages': ['python']} tags: ["important_tag"] - name: other_test_table @@ -275,7 +271,7 @@ def models(self): @pytest.mark.xfail def test_source_all_configs_source_table(self, project): - run_dbt(["compile"]) + run_dbt(["parse"]) manifest = get_manifest(project.project_root) assert "source.test.test_source.test_table" in manifest.sources assert "source.test.test_source.other_test_table" in manifest.sources @@ -296,7 +292,7 @@ def test_source_all_configs_source_table(self, project): sources: - name: test_source config: - tags: "source_level_important_tag", + tags: ["source_level_important_tag"] tables: - name: test_table config: @@ -313,14 +309,13 @@ def test_source_all_configs_source_table(self, project): loaded_at_field: some_column database: custom_database schema: custom_schema - # identifier: "seed" #this doesnt seems to be supported now? meta: {'languages': ['python']} tags: ["important_tag"] - name: other_test_table """ -# Test inheritence - set configs atproject, source, and source-table level - expect source-table level to win +# Test inheritence - set configs at project, source, and source-table level - expect source-table level to win # expect fail - not implemented class TestSourceConfigsInheritence1(SourceConfigTests): @pytest.fixture(scope="class") @@ -333,7 +328,7 @@ def project_config_update(self): @pytest.mark.xfail def test_source_all_configs_source_table(self, project): - run_dbt(["compile"]) + run_dbt(["parse"]) manifest = get_manifest(project.project_root) assert "source.test.test_source.test_table" in manifest.sources assert "source.test.test_source.other_test_table" in manifest.sources @@ -374,7 +369,6 @@ def test_source_all_configs_source_table(self, project): loaded_at_field: some_column database: custom_database schema: custom_schema - # identifier: "seed" #this doesnt seems to be supported now? meta: {'languages': ['python']} tags: ["important_tag"] tables: @@ -396,7 +390,7 @@ def project_config_update(self): @pytest.mark.xfail def test_source_two_configs_source_level(self, project): - run_dbt(["compile"]) + run_dbt(["parse"]) manifest = get_manifest(project.project_root) assert "source.test.test_source.test_table" in manifest.sources assert "source.test.test_source.other_test_table" in manifest.sources @@ -412,14 +406,14 @@ def test_source_two_configs_source_level(self, project): assert config_test_table == pytest.expected_config -all_configs_everywhere__schema_yml = """version: 2 +all_configs_project_source__schema_yml = """version: 2 sources: - name: test_source tables: - name: test_table config: - enabled: True, + enabled: True quoting: database: False schema: False @@ -432,7 +426,6 @@ def test_source_two_configs_source_level(self, project): loaded_at_field: some_column database: custom_database schema: custom_schema - # identifier: "seed" #this doesnt seems to be supported now? meta: {'languages': ['python']} tags: ["important_tag"] - name: other_test_table @@ -444,7 +437,7 @@ def test_source_two_configs_source_level(self, project): class TestSourceConfigsInheritence3(SourceConfigTests): @pytest.fixture(scope="class") def models(self): - return {"schema.yml": all_configs_everywhere__schema_yml} + return {"schema.yml": all_configs_project_source__schema_yml} @pytest.fixture(scope="class") def project_config_update(self): @@ -452,7 +445,7 @@ def project_config_update(self): @pytest.mark.xfail def test_source_two_configs_source_table(self, project): - run_dbt(["compile"]) + run_dbt(["parse"]) manifest = get_manifest(project.project_root) assert "source.test.test_source.test_table" in manifest.sources assert "source.test.test_source.other_test_table" in manifest.sources @@ -475,7 +468,39 @@ def test_source_two_configs_source_table(self, project): assert config_other_test_table == expected_project_level_config -configs_as_properties__schema_yml = """version: 2 +class SourceBackwardsCompatibility(SourceConfigTests): + @pytest.mark.xfail + def check_source_configs_and_properties(self, project): + run_dbt(["parse"]) + manifest = get_manifest(project.project_root) + assert "source.test.test_source.test_table" in manifest.sources + config_test_table = manifest.sources.get("source.test.test_source.test_table").config + + # this is new functionality - but it currently passes since SourceConfig is not updated + # and is commented out in the setup becuse it is not updated with new configs + # even when properties are defined at the top level they should end up on the node.config + assert isinstance(config_test_table, SourceConfig) + assert config_test_table == pytest.expected_config + + # this is existing functionality - should always pass + properties_test_table = manifest.sources.get("source.test.test_source.test_table") + assert properties_test_table.quoting == Quoting( + database=False, schema=False, identifier=False, column=False + ) + assert properties_test_table.freshness == FreshnessThreshold( + warn_after=Time(count=12, period=TimePeriod.hour), + error_after=Time(count=24, period=TimePeriod.hour), + filter=None, + ) + assert properties_test_table.loader == "a_loader" + assert properties_test_table.loaded_at_field == "some_column" + assert properties_test_table.database == "custom_database" + assert properties_test_table.schema == "custom_schema" + assert properties_test_table.meta == {} # TODO: why is this empty + assert properties_test_table.tags == ["important_tag"] + + +properties_as_configs__schema_yml = """version: 2 sources: - name: test_source @@ -491,7 +516,6 @@ def test_source_two_configs_source_table(self, project): loaded_at_field: some_column database: custom_database schema: custom_schema - # identifier: "seed" #this doesnt seems to be supported now? meta: {'languages': ['python']} tags: ["important_tag"] tables: @@ -500,41 +524,51 @@ def test_source_two_configs_source_table(self, project): # Check backwards compatibility of setting configs as properties at top level -# expect pass since the properties don't get copied to the node.config yet -class TestSourceBackwardsCompatibility(SourceConfigTests): +# expect pass since the properties don't get copied to the node.config yet so +# the values match since we haven't build the full SourceConfig yet +class TestPropertiesAsConfigs(SourceBackwardsCompatibility): @pytest.fixture(scope="class") def models(self): - return {"schema.yml": configs_as_properties__schema_yml} + return {"schema.yml": properties_as_configs__schema_yml} def test_source_configs_as_properties(self, project): - run_dbt(["compile"]) - manifest = get_manifest(project.project_root) - assert "source.test.test_source.test_table" in manifest.sources - config_test_table = manifest.sources.get("source.test.test_source.test_table").config + self.check_source_configs_and_properties(project) - # this is new functionality - but it currently passes since SourceConfig is not updated - # and is commented out in the setup becuse it is not updated with new configs - # even when properties are defined at teh top level they should end up on the node.config - assert isinstance(config_test_table, SourceConfig) - assert config_test_table == pytest.expected_config - # this is existing functionality - should always pass - properties_test_table = manifest.sources.get("source.test.test_source.test_table") - assert properties_test_table.quoting == Quoting( - database=False, schema=False, identifier=False, column=False - ) - assert properties_test_table.freshness == FreshnessThreshold( - warn_after=Time(count=12, period=TimePeriod.hour), - error_after=Time(count=24, period=TimePeriod.hour), - filter=None, - ) - assert properties_test_table.loader == "a_loader" - assert properties_test_table.loaded_at_field == "some_column" - assert properties_test_table.database == "custom_database" - assert properties_test_table.schema == "custom_schema" - # assert properties_test_table.identifier == "seed" - assert properties_test_table.meta == {} # TODO: why is this blank - assert properties_test_table.tags == ["important_tag"] +configs_as_properties__schema_yml = """version: 2 + +sources: + - name: test_source + config: + quoting: + database: False + schema: False + identifier: False + column: False + freshness: + error_after: {count: 24, period: hour} + warn_after: {count: 12, period: hour} + loader: "a_loader" + loaded_at_field: some_column + database: custom_database + schema: custom_schema + meta: {'languages': ['python']} + tags: ["important_tag"] + tables: + - name: test_table +""" + + +# Check backwards compatibility of setting configs as configs at top level and expect them to end up as properties +# expect fail since the properties don't get copied to the node.config yet +class TestConfigsAsProperties(SourceBackwardsCompatibility): + @pytest.fixture(scope="class") + def models(self): + return {"schema.yml": configs_as_properties__schema_yml} + + @pytest.mark.xfail + def test_source_configs_as_properties(self, project): + self.check_source_configs_and_properties(project) configs_properites__schema_yml = """version: 2 @@ -553,11 +587,10 @@ def test_source_configs_as_properties(self, project): loaded_at_field: some_column database: custom_database schema: custom_schema - # identifier: "seed" #this doesnt seems to be supported now? meta: {'languages': ['python']} tags: ["important_tag"] config: - enabled: True, + enabled: True quoting: database: False schema: False @@ -570,7 +603,6 @@ def test_source_configs_as_properties(self, project): loaded_at_field: some_column database: custom_database schema: custom_schema - # identifier: "seed" #this doesnt seems to be supported now? meta: {'languages': ['python']} tags: ["important_tag"] tables: @@ -589,6 +621,72 @@ def models(self): def test_error_source_configs_properties(self, project): # TODO: update below with correct exception/text/etc. This is placeholder. with pytest.raises(CompilationException) as exc: - run_dbt(["compile"]) + run_dbt(["parse"]) assert "???" in str(exc.value) + + +configs_properites__schema_yml = """version: 2 + +sources: + - name: test_source + tables: + - name: test_table + config: + enabled: True + identifier: "seed" +""" + + +class TestSourceIdentifierConfig: + @pytest.fixture(scope="class", autouse=True) + def setUp(self): + pytest.expected_config = SourceConfig( + enabled=True, + # TODO: uncomment all this once it's added to SourceConfig, throws error right now + # identifier = "seed" + ) + + +identifier_config_source_table__schema_yml = """version: 2 + +sources: + - name: test_source + tables: + - name: test_table + config: + identifier: "seed" + - name: other_test_table +""" + + +# Test identifier config at source table level in yml file. This config differs by +# table so should only be defined at this level. +# expect fail - not implemented +class TestSourceAllCTestSourceIdentifierConfig: + @pytest.fixture(scope="class") + def models(self): + return {"schema.yml": identifier_config_source_table__schema_yml} + + @pytest.mark.xfail + def test_source_identifier_config_source_table(self, project): + run_dbt(["parse"]) + manifest = get_manifest(project.project_root) + assert "source.test.test_source.test_table" in manifest.sources + assert "source.test.test_source.other_test_table" in manifest.sources + config_test_table = manifest.sources.get("source.test.test_source.test_table").config + config_other_test_table = manifest.sources.get( + "source.test.test_source.other_test_table" + ).config + + assert isinstance(config_test_table, SourceConfig) + assert isinstance(config_other_test_table, SourceConfig) + + identifier_expected_config = SourceConfig( + enabled=True, + # TODO: uncomment this once it's added to SourceConfig, throws error right now + # identifier = "seed" + ) + + assert config_test_table != config_other_test_table + assert config_test_table == identifier_expected_config From 1d5ca06cc1af018a981964b18c07d0257e90ebde Mon Sep 17 00:00:00 2001 From: Emily Rockman Date: Mon, 28 Mar 2022 11:37:31 -0500 Subject: [PATCH 05/21] clean up some test logic - add override tests --- .../test_simple_source_override.py | 242 ++++++++++++++++-- 1 file changed, 214 insertions(+), 28 deletions(-) diff --git a/tests/functional/source_overrides/test_simple_source_override.py b/tests/functional/source_overrides/test_simple_source_override.py index 7ff5ef09cab..c8b22b4fc28 100644 --- a/tests/functional/source_overrides/test_simple_source_override.py +++ b/tests/functional/source_overrides/test_simple_source_override.py @@ -1,5 +1,7 @@ from datetime import datetime, timedelta import pytest +from dbt.contracts.graph.model_config import SourceConfig + from dbt.tests.util import run_dbt, update_config_file, check_relations_equal from dbt.tests.fixtures.project import write_project_files @@ -16,6 +18,7 @@ class TestSourceOverride: @pytest.fixture(scope="class", autouse=True) def setUp(self, project_root, local_dependency): # noqa: F811 write_project_files(project_root, "local_dependency", local_dependency) + pytest._id = 101 @pytest.fixture(scope="class") def models(self): @@ -39,28 +42,7 @@ def packages(self): ] } - @pytest.fixture(scope="class") - def project_config_update(self): - return { - "seeds": { - "localdep": { - "enabled": False, - "keep": { - "enabled": True, - }, - }, - "quote_columns": False, - }, - "sources": { - "localdep": { - "my_other_source": { - "enabled": False, - } - } - }, - } - - def _set_updated_at_to(self, insert_id, delta, project): + def _set_updated_at_to(self, delta, project): insert_time = datetime.utcnow() + delta timestr = insert_time.strftime("%Y-%m-%d %H:%M:%S") # favorite_color,id,first_name,email,ip_address,updated_at @@ -69,6 +51,8 @@ def _set_updated_at_to(self, insert_id, delta, project): project.adapter.quote(c) for c in ("favorite_color", "id", "first_name", "email", "ip_address", "updated_at") ) + insert_id = pytest._id + pytest._id += 1 kwargs = { "schema": project.test_schema, @@ -88,11 +72,28 @@ def _set_updated_at_to(self, insert_id, delta, project): project.run_sql(raw_sql) - return insert_id + 1 + @pytest.fixture(scope="class") + def project_config_update(self): + return { + "seeds": { + "localdep": { + "enabled": False, + "keep": { + "enabled": True, + }, + }, + "quote_columns": False, + }, + "sources": { + "localdep": { + "my_other_source": { + "enabled": False, + } + } + }, + } def test_source_overrides(self, project): - insert_id = 101 - run_dbt(["deps"]) seed_results = run_dbt(["seed"]) @@ -108,7 +109,7 @@ def test_source_overrides(self, project): check_relations_equal(project.adapter, ["expected_result", "my_model"]) # set the updated_at field of this seed to last week - insert_id = self._set_updated_at_to(insert_id, timedelta(days=-7), project) + self._set_updated_at_to(timedelta(days=-7), project) # if snapshot-freshness fails, freshness just didn't happen! results = run_dbt(["source", "snapshot-freshness"], expect_pass=False) # we disabled my_other_source, so we only run the one freshness check @@ -116,14 +117,14 @@ def test_source_overrides(self, project): assert len(results) == 1 # If snapshot-freshness passes, that means error_after was # applied from the source override but not the source table override - insert_id = self._set_updated_at_to(insert_id, timedelta(days=-2), project) + self._set_updated_at_to(timedelta(days=-2), project) results = run_dbt( ["source", "snapshot-freshness"], expect_pass=False, ) assert len(results) == 1 - insert_id = self._set_updated_at_to(insert_id, timedelta(hours=-12), project) + self._set_updated_at_to(timedelta(hours=-12), project) results = run_dbt(["source", "snapshot-freshness"], expect_pass=True) assert len(results) == 1 @@ -143,3 +144,188 @@ def test_source_overrides(self, project): # not-fresh source results = run_dbt(["source", "snapshot-freshness"], expect_pass=False) assert len(results) == 2 + + +class SourceOverrideTest: + @pytest.fixture(scope="class", autouse=True) + def setUp(self, project_root, local_dependency): # noqa: F811 + write_project_files(project_root, "local_dependency", local_dependency) + + pytest.expected_config = SourceConfig( + enabled=True, + # TODO: uncomment all this once it's added to SourceConfig, throws error right now + # quoting = Quoting(database=True, schema=True, identifier=True, column=True) + # freshness = FreshnessThreshold( + # warn_after=Time(count=1, period=TimePeriod.minute), + # error_after=Time(count=5, period=TimePeriod.minute), + # filter=None + # ) + # loader = "override_a_loader" + # loaded_at_field = override_some_column + # database = override_custom_database + # schema = override_custom_schema + # meta = {'languages': ['java']} + # tags = ["override_important_tag"] + ) + + @pytest.fixture(scope="class") + def project_config_update(self): + return { + "sources": { + "localdep": { + "my_source": { + "quoting": { + "database": False, + "schema": False, + "identifier": False, + "column": False, + }, + "freshness": { + "error_after": {"count": 24, "period": "hour"}, + "warn_after": {"count": 12, "period": "hour"}, + }, + "loader": "a_loader", + "loaded_at_field": "some_column", + "database": "custom_database", + "schema": "custom_schema", + "meta": {"languages": ["python"]}, + "tags": ["important_tag"], + } + } + }, + } + + @pytest.fixture(scope="class") + def packages(self): + return { + "packages": [ + { + "local": "local_dependency", + }, + ] + } + + +overrides_source_level__schema_yml = """ +version: 2 +sources: + - name: my_source + overrides: localdep + config: + quoting: + database: True + schema: True + identifier: True + column: True + freshness: + error_after: {count: 1, period: minute} + warn_after: {count: 5, period: minute} + loader: "override_a_loader" + loaded_at_field: override_some_column + database: override_custom_database + schema: override_custom_schema + meta: {'languages': ['java']} + tags: ["override_important_tag"] + tables: + - name: my_table + - name: my_other_table + - name: snapshot_freshness +""" + + +# test overriding at the source level +# expect fail since these are no valid configs right now +class TestSourceLevelOverride(SourceOverrideTest): + @pytest.fixture(scope="class") + def models(self): + return {"schema.yml": overrides_source_level__schema_yml} + + @pytest.mark.xfail + def test_source_level_overrides(self, project): + run_dbt(["deps"]) + + # this currently fails because configs fail parsing under an override + run_dbt(["parse"]) + manifest = get_manifest(project.project_root) + + assert "source.localdep.my_source.my_table" in manifest.sources + assert "source.localdep.my_source.my_other_table" in manifest.sources + assert "source.localdep.my_source.snapshot_freshness" in manifest.sources + + config_my_table = manifest.sources.get("source.localdep.my_source.my_table").config + config_my_other_table = manifest.sources.get( + "source.localdep.my_source.my_other_table" + ).config + config_snapshot_freshness_table = manifest.sources.get( + "source.localdep.my_source.snapshot_freshness" + ).config + + assert isinstance(config_my_table, SourceConfig) + assert isinstance(config_my_other_table, SourceConfig) + + assert config_my_table == config_my_other_table + assert config_my_table == config_snapshot_freshness_table + assert config_my_table == pytest.expected_config + + +overrides_source_level__schema_yml = """ +version: 2 +sources: + - name: my_source + overrides: localdep + tables: + - name: my_table + - name: my_other_table + config: + quoting: + database: True + schema: True + identifier: True + column: True + freshness: + error_after: {count: 1, period: minute} + warn_after: {count: 5, period: minute} + loader: "override_a_loader" + loaded_at_field: override_some_column + database: override_custom_database + schema: override_custom_schema + meta: {'languages': ['java']} + tags: ["override_important_tag"] + - name: snapshot_freshness +""" + + +# test overriding at the source table level +# expect fail since these are no valid configs right now +class TestSourceTableOverride(SourceOverrideTest): + @pytest.fixture(scope="class") + def models(self): + return {"schema.yml": overrides_source_level__schema_yml} + + @pytest.mark.xfail + def test_source_table_overrides(self, project): + run_dbt(["deps"]) + + # this currently fails because configs fail parsing under an override + run_dbt(["parse"]) + manifest = get_manifest(project.project_root) + + assert "source.localdep.my_source.my_table" in manifest.sources + assert "source.localdep.my_source.my_other_table" in manifest.sources + assert "source.localdep.my_source.snapshot_freshness" in manifest.sources + + config_my_table = manifest.sources.get("source.localdep.my_source.my_table").config + config_my_other_table = manifest.sources.get( + "source.localdep.my_source.my_other_table" + ).config + config_snapshot_freshness_table = manifest.sources.get( + "source.localdep.my_source.snapshot_freshness" + ).config + + assert isinstance(config_my_table, SourceConfig) + assert isinstance(config_my_other_table, SourceConfig) + + assert config_my_table != config_my_other_table + assert config_my_table == config_snapshot_freshness_table + + assert config_my_other_table == pytest.expected_config From d6aa961c0ca690b02715a5d26666bc1e228e6538 Mon Sep 17 00:00:00 2001 From: Nathaniel May Date: Mon, 4 Apr 2022 13:30:50 -0400 Subject: [PATCH 06/21] add new fields to source config class --- core/dbt/contracts/graph/model_config.py | 10 +++++++ .../test_simple_source_override.py | 28 +++++++++---------- 2 files changed, 23 insertions(+), 15 deletions(-) diff --git a/core/dbt/contracts/graph/model_config.py b/core/dbt/contracts/graph/model_config.py index 50074c71359..35bb4d42e86 100644 --- a/core/dbt/contracts/graph/model_config.py +++ b/core/dbt/contracts/graph/model_config.py @@ -335,6 +335,16 @@ def replace(self, **kwargs): @dataclass class SourceConfig(BaseConfig): enabled: bool = True + quoting: Optional[Dict[str, Any]] = None + freshness: Optional[Dict[str, Any]] = None + loader: Optional[str] = None + loaded_at_field: Optional[ + Any + ] = None # TODO what type is this? docs say: "" + database: Optional[str] = None + schema: Optional[str] = None + meta: Optional[Dict[str, Any]] = None + tags: Optional[List[str]] = None @dataclass diff --git a/tests/functional/source_overrides/test_simple_source_override.py b/tests/functional/source_overrides/test_simple_source_override.py index c8b22b4fc28..80b360abaeb 100644 --- a/tests/functional/source_overrides/test_simple_source_override.py +++ b/tests/functional/source_overrides/test_simple_source_override.py @@ -153,19 +153,17 @@ def setUp(self, project_root, local_dependency): # noqa: F811 pytest.expected_config = SourceConfig( enabled=True, - # TODO: uncomment all this once it's added to SourceConfig, throws error right now - # quoting = Quoting(database=True, schema=True, identifier=True, column=True) - # freshness = FreshnessThreshold( - # warn_after=Time(count=1, period=TimePeriod.minute), - # error_after=Time(count=5, period=TimePeriod.minute), - # filter=None - # ) - # loader = "override_a_loader" - # loaded_at_field = override_some_column - # database = override_custom_database - # schema = override_custom_schema - # meta = {'languages': ['java']} - # tags = ["override_important_tag"] + quoting={"database": True, "schema": True, "identifier": True, "column": True}, + freshness={ + "warn_after": {"count": 1, "period": "minute"}, + "error_after": {"count": 5, "period": "minute"}, + }, + loader="override_a_loader", + loaded_at_field="override_some_column", + database="override_custom_database", + schema="override_custom_schema", + meta={"languages": ["java"]}, + tags=["override_important_tag"], ) @pytest.fixture(scope="class") @@ -240,7 +238,7 @@ class TestSourceLevelOverride(SourceOverrideTest): def models(self): return {"schema.yml": overrides_source_level__schema_yml} - @pytest.mark.xfail + # @pytest.mark.xfail def test_source_level_overrides(self, project): run_dbt(["deps"]) @@ -302,7 +300,7 @@ class TestSourceTableOverride(SourceOverrideTest): def models(self): return {"schema.yml": overrides_source_level__schema_yml} - @pytest.mark.xfail + # @pytest.mark.xfail def test_source_table_overrides(self, project): run_dbt(["deps"]) From 60561f9b251d5340946b67233b7a10bc894100f2 Mon Sep 17 00:00:00 2001 From: Nathaniel May Date: Mon, 4 Apr 2022 13:31:34 -0400 Subject: [PATCH 07/21] fix odd formatting --- core/dbt/contracts/graph/model_config.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/core/dbt/contracts/graph/model_config.py b/core/dbt/contracts/graph/model_config.py index 35bb4d42e86..889c0387fa8 100644 --- a/core/dbt/contracts/graph/model_config.py +++ b/core/dbt/contracts/graph/model_config.py @@ -338,9 +338,8 @@ class SourceConfig(BaseConfig): quoting: Optional[Dict[str, Any]] = None freshness: Optional[Dict[str, Any]] = None loader: Optional[str] = None - loaded_at_field: Optional[ - Any - ] = None # TODO what type is this? docs say: "" + # TODO what type is this? docs say: "" + loaded_at_field: Optional[Any] = None database: Optional[str] = None schema: Optional[str] = None meta: Optional[Dict[str, Any]] = None From 8bc2d40dc7420883ddb8d8bf8f15348f2b09bc41 Mon Sep 17 00:00:00 2001 From: Emily Rockman Date: Wed, 6 Apr 2022 15:12:14 -0500 Subject: [PATCH 08/21] got a test working --- core/dbt/contracts/graph/model_config.py | 40 +++++++++++++++---- core/dbt/contracts/graph/unparsed.py | 2 + core/dbt/parser/manifest.py | 2 + core/dbt/parser/sources.py | 15 ++++--- .../functional/permission/test_permissions.py | 15 +++++++ .../functional/sources/test_source_configs.py | 28 ++++++------- 6 files changed, 71 insertions(+), 31 deletions(-) diff --git a/core/dbt/contracts/graph/model_config.py b/core/dbt/contracts/graph/model_config.py index 889c0387fa8..93fb0639e2a 100644 --- a/core/dbt/contracts/graph/model_config.py +++ b/core/dbt/contracts/graph/model_config.py @@ -335,15 +335,39 @@ def replace(self, **kwargs): @dataclass class SourceConfig(BaseConfig): enabled: bool = True - quoting: Optional[Dict[str, Any]] = None - freshness: Optional[Dict[str, Any]] = None - loader: Optional[str] = None + quoting: Dict[str, Any] = field( + default_factory=dict, + metadata=MergeBehavior.Update.meta(), + ) + freshness: Optional[Dict[str, Any]] = field( + default=None, + metadata=CompareBehavior.Exclude.meta(), + ) + loader: Optional[str] = field( + default=None, + metadata=CompareBehavior.Exclude.meta(), + ) # TODO what type is this? docs say: "" - loaded_at_field: Optional[Any] = None - database: Optional[str] = None - schema: Optional[str] = None - meta: Optional[Dict[str, Any]] = None - tags: Optional[List[str]] = None + loaded_at_field: Optional[str] = field( + default=None, + metadata=CompareBehavior.Exclude.meta(), + ) + database: Optional[str] = field( + default=None, + metadata=CompareBehavior.Exclude.meta(), + ) + schema: Optional[str] = field( + default=None, + metadata=CompareBehavior.Exclude.meta(), + ) + meta: Dict[str, Any] = field( + default_factory=dict, + metadata=MergeBehavior.Update.meta(), + ) + tags: Union[List[str], str] = field( + default_factory=list_str, + metadata=metas(ShowBehavior.Hide, MergeBehavior.Append, CompareBehavior.Exclude), + ) @dataclass diff --git a/core/dbt/contracts/graph/unparsed.py b/core/dbt/contracts/graph/unparsed.py index d1c0b4a7a5a..80e956c06e0 100644 --- a/core/dbt/contracts/graph/unparsed.py +++ b/core/dbt/contracts/graph/unparsed.py @@ -242,6 +242,7 @@ class Quoting(dbtClassMixin, Mergeable): @dataclass class UnparsedSourceTableDefinition(HasColumnTests, HasTests): + config: Dict[str, Any] = field(default_factory=dict) loaded_at_field: Optional[str] = None identifier: Optional[str] = None quoting: Quoting = field(default_factory=Quoting) @@ -322,6 +323,7 @@ class SourcePatch(dbtClassMixin, Replaceable): path: Path = field( metadata=dict(description="The path to the patch-defining yml file"), ) + config: Dict[str, Any] = field(default_factory=dict) description: Optional[str] = None meta: Optional[Dict[str, Any]] = None database: Optional[str] = None diff --git a/core/dbt/parser/manifest.py b/core/dbt/parser/manifest.py index 0f4c7e97537..70f727c3634 100644 --- a/core/dbt/parser/manifest.py +++ b/core/dbt/parser/manifest.py @@ -465,6 +465,8 @@ def parse_project( else: dct = block.file.dict_from_yaml parser.parse_file(block, dct=dct) + # Came out of here with UnpatchedSourceDefinition containing configs at the source level + # and not configs at the table level (as expected) else: parser.parse_file(block) project_parsed_path_count += 1 diff --git a/core/dbt/parser/sources.py b/core/dbt/parser/sources.py index 44dcdb29b67..c83bea279db 100644 --- a/core/dbt/parser/sources.py +++ b/core/dbt/parser/sources.py @@ -1,6 +1,6 @@ import itertools from pathlib import Path -from typing import Iterable, Dict, Optional, Set, List, Any +from typing import Iterable, Dict, Optional, Set, Any from dbt.adapters.factory import get_adapter from dbt.config import RuntimeConfig from dbt.context.context_config import ( @@ -137,15 +137,13 @@ def parse_source(self, target: UnpatchedSourceDefinition) -> ParsedSourceDefinit tags = sorted(set(itertools.chain(source.tags, table.tags))) config = self._generate_source_config( - fqn=target.fqn, + target=target, rendered=True, - project_name=target.package_name, ) unrendered_config = self._generate_source_config( - fqn=target.fqn, + target=target, rendered=False, - project_name=target.package_name, ) if not isinstance(config, SourceConfig): @@ -261,7 +259,7 @@ def parse_source_test( ) return node - def _generate_source_config(self, fqn: List[str], rendered: bool, project_name: str): + def _generate_source_config(self, target: UnpatchedSourceDefinition, rendered: bool): generator: BaseContextConfigGenerator if rendered: generator = ContextConfigGenerator(self.root_project) @@ -270,10 +268,11 @@ def _generate_source_config(self, fqn: List[str], rendered: bool, project_name: return generator.calculate_node_config( config_call_dict={}, - fqn=fqn, + fqn=target.fqn, resource_type=NodeType.Source, - project_name=project_name, + project_name=target.package_name, base=False, + patch_config_dict=target.source.config, ) def _get_relation_name(self, node: ParsedSourceDefinition): diff --git a/tests/functional/permission/test_permissions.py b/tests/functional/permission/test_permissions.py index e2999f47548..9e89bcede50 100644 --- a/tests/functional/permission/test_permissions.py +++ b/tests/functional/permission/test_permissions.py @@ -48,3 +48,18 @@ def test_no_create_schema_permissions( project.run_sql('drop schema if exists "{}" cascade'.format(project.test_schema)) with pytest.raises(RuntimeError): run_dbt(["run", "--target", "noaccess"], expect_pass=False) + + def test_create_schema_permissions( + self, + project, + ): + # now it should work! + project.run_sql("grant create on database {} to noaccess".format(project.database)) + project.run_sql( + 'grant usage, create on schema "{}" to noaccess'.format(project.test_schema) + ) + project.run_sql( + 'grant select on all tables in schema "{}" to noaccess'.format(project.test_schema) + ) + results = run_dbt(["run", "--target", "noaccess"]) + assert len(results) == 1 diff --git a/tests/functional/sources/test_source_configs.py b/tests/functional/sources/test_source_configs.py index 1205dbfdf0a..9ab2bb1aa1f 100644 --- a/tests/functional/sources/test_source_configs.py +++ b/tests/functional/sources/test_source_configs.py @@ -11,19 +11,17 @@ class SourceConfigTests: def setUp(self): pytest.expected_config = SourceConfig( enabled=True, - # TODO: uncomment all this once it's added to SourceConfig, throws error right now - # quoting = Quoting(database=False, schema=False, identifier=False, column=False) - # freshness = FreshnessThreshold( - # warn_after=Time(count=12, period=TimePeriod.hour), - # error_after=Time(count=24, period=TimePeriod.hour), - # filter=None - # ) - # loader = "a_loader" - # loaded_at_field = some_column - # database = custom_database - # schema = custom_schema - # meta = {'languages': ['python']} - # tags = ["important_tag"] + quoting={"database": False, "schema": False, "identifier": False, "column": False}, + freshness={ + "warn_after": {"count": 12, "period": "hour"}, + "error_after": {"count": 24, "period": "hour"}, + }, + loader="a_loader", + loaded_at_field="some_column", + database="custom_database", + schema="custom_schema", + meta={"languages": ["python"]}, + tags=["important_tag"], ) @@ -105,7 +103,6 @@ def models(self): "schema.yml": disabled_source_level__schema_yml, } - @pytest.mark.xfail def test_source_config_yaml_source_level(self, project): run_dbt(["parse"]) manifest = get_manifest(project.project_root) @@ -531,7 +528,8 @@ class TestPropertiesAsConfigs(SourceBackwardsCompatibility): def models(self): return {"schema.yml": properties_as_configs__schema_yml} - def test_source_configs_as_properties(self, project): + @pytest.mark.xfail + def test_source_properties_as_configs(self, project): self.check_source_configs_and_properties(project) From 872dae2ef7f0dac2cbd75764bfb81e8f1176b945 Mon Sep 17 00:00:00 2001 From: Emily Rockman Date: Thu, 7 Apr 2022 11:30:19 -0500 Subject: [PATCH 09/21] removed unused tests --- .../functional/sources/test_source_configs.py | 434 +----------------- 1 file changed, 3 insertions(+), 431 deletions(-) diff --git a/tests/functional/sources/test_source_configs.py b/tests/functional/sources/test_source_configs.py index 9ab2bb1aa1f..4126fee597c 100644 --- a/tests/functional/sources/test_source_configs.py +++ b/tests/functional/sources/test_source_configs.py @@ -1,7 +1,5 @@ import pytest from dbt.contracts.graph.model_config import SourceConfig -from dbt.contracts.graph.unparsed import FreshnessThreshold, Time, TimePeriod, Quoting -from dbt.exceptions import CompilationException from dbt.tests.util import run_dbt, update_config_file, get_manifest @@ -11,17 +9,6 @@ class SourceConfigTests: def setUp(self): pytest.expected_config = SourceConfig( enabled=True, - quoting={"database": False, "schema": False, "identifier": False, "column": False}, - freshness={ - "warn_after": {"count": 12, "period": "hour"}, - "error_after": {"count": 24, "period": "hour"}, - }, - loader="a_loader", - loaded_at_field="some_column", - database="custom_database", - schema="custom_schema", - meta={"languages": ["python"]}, - tags=["important_tag"], ) @@ -95,7 +82,6 @@ def test_enabled_source_config_dbt_project(self, project): # Test enabled config at sources level in yml file -# expect fail - not implemented class TestConfigYamlSourceLevel(SourceConfigTests): @pytest.fixture(scope="class") def models(self): @@ -139,175 +125,16 @@ def test_source_config_yaml_source_table(self, project): assert "source.test.test_source.disabled_test_table" not in manifest.sources -# Test all configs other than enabled in dbt_project.yml -# expect fail - not implemented -class TestAllConfigsProjectLevel(SourceConfigTests): - @pytest.fixture(scope="class") - def models(self): - return {"schema.yml": models__schema_yml} - - @pytest.fixture(scope="class") - def project_config_update(self): - return { - "sources": { - "enabled": True, - "quoting": { - "database": False, - "schema": False, - "identifier": False, - "column": False, - }, - "freshness": { - "error_after": {"count": 24, "period": "hour"}, - "warn_after": {"count": 12, "period": "hour"}, - }, - "loader": "a_loader", - "loaded_at_field": "some_column", - "database": "custom_database", - "schema": "custom_schema", - "meta": {"languages": ["python"]}, - "tags": ["important_tag"], - } - } - - @pytest.mark.xfail - def test_source_all_configs_dbt_project(self, project): - run_dbt(["parse"]) - manifest = get_manifest(project.project_root) - assert "source.test.test_source.test_table" in manifest.sources - config = manifest.sources.get("source.test.test_source.test_table").config - - assert isinstance(config, SourceConfig) - - assert config == pytest.expected_config - - -configs_source_level__schema_yml = """version: 2 - -sources: - - name: test_source - config: - enabled: True - quoting: - database: False - schema: False - identifier: False - column: False - freshness: - error_after: {count: 24, period: hour} - warn_after: {count: 12, period: hour} - loader: "a_loader" - loaded_at_field: some_column - database: custom_database - schema: custom_schema - meta: {'languages': ['python']} - tags: ["important_tag"] - tables: - - name: test_table - - name: other_test_table -""" - - -# Test configs other than enabled at sources level in yaml file -# **currently passes since enabled is all that ends up in the -# node.config since it's the only thing implemented -class TestAllConfigsSourceLevel(SourceConfigTests): - @pytest.fixture(scope="class") - def models(self): - return {"schema.yml": configs_source_level__schema_yml} - - def test_source_all_configs_source_level(self, project): - run_dbt(["parse"]) - manifest = get_manifest(project.project_root) - assert "source.test.test_source.test_table" in manifest.sources - assert "source.test.test_source.other_test_table" in manifest.sources - config_test_table = manifest.sources.get("source.test.test_source.test_table").config - config_other_test_table = manifest.sources.get( - "source.test.test_source.other_test_table" - ).config - - assert isinstance(config_test_table, SourceConfig) - assert isinstance(config_other_test_table, SourceConfig) - - assert config_test_table == config_other_test_table - assert config_test_table == pytest.expected_config - - -configs_source_table__schema_yml = """version: 2 - -sources: - - name: test_source - tables: - - name: test_table - config: - enabled: True - quoting: - database: False - schema: False - identifier: False - column: False - freshness: - error_after: {count: 24, period: hour} - warn_after: {count: 12, period: hour} - loader: "a_loader" - loaded_at_field: some_column - database: custom_database - schema: custom_schema - meta: {'languages': ['python']} - tags: ["important_tag"] - - name: other_test_table -""" - - -# Test configs other than enabled at source table level in yml file -# expect fail - not implemented -class TestSourceAllConfigsSourceTable(SourceConfigTests): - @pytest.fixture(scope="class") - def models(self): - return {"schema.yml": configs_source_table__schema_yml} - - @pytest.mark.xfail - def test_source_all_configs_source_table(self, project): - run_dbt(["parse"]) - manifest = get_manifest(project.project_root) - assert "source.test.test_source.test_table" in manifest.sources - assert "source.test.test_source.other_test_table" in manifest.sources - config_test_table = manifest.sources.get("source.test.test_source.test_table").config - config_other_test_table = manifest.sources.get( - "source.test.test_source.other_test_table" - ).config - - assert isinstance(config_test_table, SourceConfig) - assert isinstance(config_other_test_table, SourceConfig) - - assert config_test_table != config_other_test_table - assert config_test_table == pytest.expected_config - - all_configs_everywhere__schema_yml = """version: 2 sources: - name: test_source config: - tags: ["source_level_important_tag"] + enabled: False tables: - name: test_table config: enabled: True - quoting: - database: False - schema: False - identifier: False - column: False - freshness: - error_after: {count: 24, period: hour} - warn_after: {count: 12, period: hour} - loader: "a_loader" - loaded_at_field: some_column - database: custom_database - schema: custom_schema - meta: {'languages': ['python']} - tags: ["important_tag"] - name: other_test_table """ @@ -342,7 +169,6 @@ def test_source_all_configs_source_table(self, project): expected_source_level_config = SourceConfig( enabled=True, - # "tags" = "source_level_important_tag" #TODO: update after SourceConfigs gets updated ) assert config_other_test_table == expected_source_level_config @@ -354,20 +180,6 @@ def test_source_all_configs_source_table(self, project): - name: test_source config: enabled: True - quoting: - database: False - schema: False - identifier: False - column: False - freshness: - error_after: {count: 24, period: hour} - warn_after: {count: 12, period: hour} - loader: "a_loader" - loaded_at_field: some_column - database: custom_database - schema: custom_schema - meta: {'languages': ['python']} - tags: ["important_tag"] tables: - name: test_table - name: other_test_table @@ -383,7 +195,7 @@ def models(self): @pytest.fixture(scope="class") def project_config_update(self): - return {"sources": {"tags": "project_level_important_tag"}} + return {"sources": {"enabled": False}} @pytest.mark.xfail def test_source_two_configs_source_level(self, project): @@ -411,20 +223,6 @@ def test_source_two_configs_source_level(self, project): - name: test_table config: enabled: True - quoting: - database: False - schema: False - identifier: False - column: False - freshness: - error_after: {count: 24, period: hour} - warn_after: {count: 12, period: hour} - loader: "a_loader" - loaded_at_field: some_column - database: custom_database - schema: custom_schema - meta: {'languages': ['python']} - tags: ["important_tag"] - name: other_test_table """ @@ -438,7 +236,7 @@ def models(self): @pytest.fixture(scope="class") def project_config_update(self): - return {"sources": {"tags": "project_level_important_tag"}} + return {"sources": {"enabled": False}} @pytest.mark.xfail def test_source_two_configs_source_table(self, project): @@ -459,232 +257,6 @@ def test_source_two_configs_source_table(self, project): expected_project_level_config = SourceConfig( enabled=True, - # tags = "project_level_important_tag", # TODO: uncomment these once SourceConfig is updated ) assert config_other_test_table == expected_project_level_config - - -class SourceBackwardsCompatibility(SourceConfigTests): - @pytest.mark.xfail - def check_source_configs_and_properties(self, project): - run_dbt(["parse"]) - manifest = get_manifest(project.project_root) - assert "source.test.test_source.test_table" in manifest.sources - config_test_table = manifest.sources.get("source.test.test_source.test_table").config - - # this is new functionality - but it currently passes since SourceConfig is not updated - # and is commented out in the setup becuse it is not updated with new configs - # even when properties are defined at the top level they should end up on the node.config - assert isinstance(config_test_table, SourceConfig) - assert config_test_table == pytest.expected_config - - # this is existing functionality - should always pass - properties_test_table = manifest.sources.get("source.test.test_source.test_table") - assert properties_test_table.quoting == Quoting( - database=False, schema=False, identifier=False, column=False - ) - assert properties_test_table.freshness == FreshnessThreshold( - warn_after=Time(count=12, period=TimePeriod.hour), - error_after=Time(count=24, period=TimePeriod.hour), - filter=None, - ) - assert properties_test_table.loader == "a_loader" - assert properties_test_table.loaded_at_field == "some_column" - assert properties_test_table.database == "custom_database" - assert properties_test_table.schema == "custom_schema" - assert properties_test_table.meta == {} # TODO: why is this empty - assert properties_test_table.tags == ["important_tag"] - - -properties_as_configs__schema_yml = """version: 2 - -sources: - - name: test_source - quoting: - database: False - schema: False - identifier: False - column: False - freshness: - error_after: {count: 24, period: hour} - warn_after: {count: 12, period: hour} - loader: "a_loader" - loaded_at_field: some_column - database: custom_database - schema: custom_schema - meta: {'languages': ['python']} - tags: ["important_tag"] - tables: - - name: test_table -""" - - -# Check backwards compatibility of setting configs as properties at top level -# expect pass since the properties don't get copied to the node.config yet so -# the values match since we haven't build the full SourceConfig yet -class TestPropertiesAsConfigs(SourceBackwardsCompatibility): - @pytest.fixture(scope="class") - def models(self): - return {"schema.yml": properties_as_configs__schema_yml} - - @pytest.mark.xfail - def test_source_properties_as_configs(self, project): - self.check_source_configs_and_properties(project) - - -configs_as_properties__schema_yml = """version: 2 - -sources: - - name: test_source - config: - quoting: - database: False - schema: False - identifier: False - column: False - freshness: - error_after: {count: 24, period: hour} - warn_after: {count: 12, period: hour} - loader: "a_loader" - loaded_at_field: some_column - database: custom_database - schema: custom_schema - meta: {'languages': ['python']} - tags: ["important_tag"] - tables: - - name: test_table -""" - - -# Check backwards compatibility of setting configs as configs at top level and expect them to end up as properties -# expect fail since the properties don't get copied to the node.config yet -class TestConfigsAsProperties(SourceBackwardsCompatibility): - @pytest.fixture(scope="class") - def models(self): - return {"schema.yml": configs_as_properties__schema_yml} - - @pytest.mark.xfail - def test_source_configs_as_properties(self, project): - self.check_source_configs_and_properties(project) - - -configs_properites__schema_yml = """version: 2 - -sources: - - name: test_source - quoting: - database: False - schema: False - identifier: False - column: False - freshness: - error_after: {count: 24, period: hour} - warn_after: {count: 12, period: hour} - loader: "a_loader" - loaded_at_field: some_column - database: custom_database - schema: custom_schema - meta: {'languages': ['python']} - tags: ["important_tag"] - config: - enabled: True - quoting: - database: False - schema: False - identifier: False - column: False - freshness: - error_after: {count: 24, period: hour} - warn_after: {count: 12, period: hour} - loader: "a_loader" - loaded_at_field: some_column - database: custom_database - schema: custom_schema - meta: {'languages': ['python']} - tags: ["important_tag"] - tables: - - name: test_table - - name: other_test_table -""" - - -# Raise an error when properties are set at top level and also as configs -class TestErrorSourceConfigProperty(SourceConfigTests): - @pytest.fixture(scope="class") - def models(self): - return {"schema.yml": configs_properites__schema_yml} - - @pytest.mark.xfail - def test_error_source_configs_properties(self, project): - # TODO: update below with correct exception/text/etc. This is placeholder. - with pytest.raises(CompilationException) as exc: - run_dbt(["parse"]) - - assert "???" in str(exc.value) - - -configs_properites__schema_yml = """version: 2 - -sources: - - name: test_source - tables: - - name: test_table - config: - enabled: True - identifier: "seed" -""" - - -class TestSourceIdentifierConfig: - @pytest.fixture(scope="class", autouse=True) - def setUp(self): - pytest.expected_config = SourceConfig( - enabled=True, - # TODO: uncomment all this once it's added to SourceConfig, throws error right now - # identifier = "seed" - ) - - -identifier_config_source_table__schema_yml = """version: 2 - -sources: - - name: test_source - tables: - - name: test_table - config: - identifier: "seed" - - name: other_test_table -""" - - -# Test identifier config at source table level in yml file. This config differs by -# table so should only be defined at this level. -# expect fail - not implemented -class TestSourceAllCTestSourceIdentifierConfig: - @pytest.fixture(scope="class") - def models(self): - return {"schema.yml": identifier_config_source_table__schema_yml} - - @pytest.mark.xfail - def test_source_identifier_config_source_table(self, project): - run_dbt(["parse"]) - manifest = get_manifest(project.project_root) - assert "source.test.test_source.test_table" in manifest.sources - assert "source.test.test_source.other_test_table" in manifest.sources - config_test_table = manifest.sources.get("source.test.test_source.test_table").config - config_other_test_table = manifest.sources.get( - "source.test.test_source.other_test_table" - ).config - - assert isinstance(config_test_table, SourceConfig) - assert isinstance(config_other_test_table, SourceConfig) - - identifier_expected_config = SourceConfig( - enabled=True, - # TODO: uncomment this once it's added to SourceConfig, throws error right now - # identifier = "seed" - ) - - assert config_test_table != config_other_test_table - assert config_test_table == identifier_expected_config From dda443daf639d1d0cc4af376627c0549840fdc7f Mon Sep 17 00:00:00 2001 From: Emily Rockman Date: Thu, 7 Apr 2022 11:58:41 -0500 Subject: [PATCH 10/21] removed extra fields from SourceConfig class --- core/dbt/contracts/graph/model_config.py | 67 ++++++++++++------------ 1 file changed, 34 insertions(+), 33 deletions(-) diff --git a/core/dbt/contracts/graph/model_config.py b/core/dbt/contracts/graph/model_config.py index 93fb0639e2a..3352303a47f 100644 --- a/core/dbt/contracts/graph/model_config.py +++ b/core/dbt/contracts/graph/model_config.py @@ -335,39 +335,40 @@ def replace(self, **kwargs): @dataclass class SourceConfig(BaseConfig): enabled: bool = True - quoting: Dict[str, Any] = field( - default_factory=dict, - metadata=MergeBehavior.Update.meta(), - ) - freshness: Optional[Dict[str, Any]] = field( - default=None, - metadata=CompareBehavior.Exclude.meta(), - ) - loader: Optional[str] = field( - default=None, - metadata=CompareBehavior.Exclude.meta(), - ) - # TODO what type is this? docs say: "" - loaded_at_field: Optional[str] = field( - default=None, - metadata=CompareBehavior.Exclude.meta(), - ) - database: Optional[str] = field( - default=None, - metadata=CompareBehavior.Exclude.meta(), - ) - schema: Optional[str] = field( - default=None, - metadata=CompareBehavior.Exclude.meta(), - ) - meta: Dict[str, Any] = field( - default_factory=dict, - metadata=MergeBehavior.Update.meta(), - ) - tags: Union[List[str], str] = field( - default_factory=list_str, - metadata=metas(ShowBehavior.Hide, MergeBehavior.Append, CompareBehavior.Exclude), - ) + # to be implmented to complete CT-201 + # quoting: Dict[str, Any] = field( + # default_factory=dict, + # metadata=MergeBehavior.Update.meta(), + # ) + # freshness: Optional[Dict[str, Any]] = field( + # default=None, + # metadata=CompareBehavior.Exclude.meta(), + # ) + # loader: Optional[str] = field( + # default=None, + # metadata=CompareBehavior.Exclude.meta(), + # ) + # # TODO what type is this? docs say: "" + # loaded_at_field: Optional[str] = field( + # default=None, + # metadata=CompareBehavior.Exclude.meta(), + # ) + # database: Optional[str] = field( + # default=None, + # metadata=CompareBehavior.Exclude.meta(), + # ) + # schema: Optional[str] = field( + # default=None, + # metadata=CompareBehavior.Exclude.meta(), + # ) + # meta: Dict[str, Any] = field( + # default_factory=dict, + # metadata=MergeBehavior.Update.meta(), + # ) + # tags: Union[List[str], str] = field( + # default_factory=list_str, + # metadata=metas(ShowBehavior.Hide, MergeBehavior.Append, CompareBehavior.Exclude), + # ) @dataclass From 7dfd83687011ec58f7da7de954f8149cf208a13b Mon Sep 17 00:00:00 2001 From: Emily Rockman Date: Thu, 7 Apr 2022 13:13:34 -0500 Subject: [PATCH 11/21] fixed next failing unit test --- test/unit/test_contracts_graph_unparsed.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/test/unit/test_contracts_graph_unparsed.py b/test/unit/test_contracts_graph_unparsed.py index f82e617f55c..c4df44f7100 100644 --- a/test/unit/test_contracts_graph_unparsed.py +++ b/test/unit/test_contracts_graph_unparsed.py @@ -308,6 +308,7 @@ def test_table_defaults(self): to_dict = { 'name': 'foo', 'description': '', + 'config': {}, 'loader': '', 'freshness': {'error_after': {}, 'warn_after': {}}, 'quoting': {}, @@ -316,6 +317,7 @@ def test_table_defaults(self): { 'name': 'table1', 'description': '', + 'config': {}, 'docs': {'show': True}, 'tests': [], 'columns': [], @@ -327,6 +329,7 @@ def test_table_defaults(self): { 'name': 'table2', 'description': 'table 2', + 'config': {}, 'docs': {'show': True}, 'tests': [], 'columns': [], From 05448abcb79aacf7e7c7145f96ebf9ea453d740f Mon Sep 17 00:00:00 2001 From: Emily Rockman Date: Thu, 7 Apr 2022 13:36:45 -0500 Subject: [PATCH 12/21] adding back missing import --- .../functional/source_overrides/test_simple_source_override.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/functional/source_overrides/test_simple_source_override.py b/tests/functional/source_overrides/test_simple_source_override.py index 80b360abaeb..cd4b3d87204 100644 --- a/tests/functional/source_overrides/test_simple_source_override.py +++ b/tests/functional/source_overrides/test_simple_source_override.py @@ -3,7 +3,7 @@ from dbt.contracts.graph.model_config import SourceConfig -from dbt.tests.util import run_dbt, update_config_file, check_relations_equal +from dbt.tests.util import run_dbt, update_config_file, check_relations_equal, get_manifest from dbt.tests.fixtures.project import write_project_files from tests.functional.source_overrides.fixtures import ( # noqa: F401 local_dependency, From 3507cf07967083a0b30bdd3ce12f2021b7258af1 Mon Sep 17 00:00:00 2001 From: Nathaniel May Date: Thu, 7 Apr 2022 15:27:35 -0400 Subject: [PATCH 13/21] first pass at adding source table configs --- core/dbt/parser/sources.py | 9 ++++++++- tests/functional/sources/test_source_configs.py | 2 -- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/core/dbt/parser/sources.py b/core/dbt/parser/sources.py index c83bea279db..155e223ad8b 100644 --- a/core/dbt/parser/sources.py +++ b/core/dbt/parser/sources.py @@ -266,13 +266,20 @@ def _generate_source_config(self, target: UnpatchedSourceDefinition, rendered: b else: generator = UnrenderedConfigGenerator(self.root_project) + # configs with precendence set + precedence_configs = dict() + # first apply source configs + precedence_configs.update(target.source.config) + # then overrite anything that is defined on source tables + precedence_configs.update(target.table.config) + return generator.calculate_node_config( config_call_dict={}, fqn=target.fqn, resource_type=NodeType.Source, project_name=target.package_name, base=False, - patch_config_dict=target.source.config, + patch_config_dict=precedence_configs, ) def _get_relation_name(self, node: ParsedSourceDefinition): diff --git a/tests/functional/sources/test_source_configs.py b/tests/functional/sources/test_source_configs.py index 4126fee597c..1409566a15d 100644 --- a/tests/functional/sources/test_source_configs.py +++ b/tests/functional/sources/test_source_configs.py @@ -117,7 +117,6 @@ def models(self): "schema.yml": disabled_source_table__schema_yml, } - @pytest.mark.xfail def test_source_config_yaml_source_table(self, project): run_dbt(["parse"]) manifest = get_manifest(project.project_root) @@ -197,7 +196,6 @@ def models(self): def project_config_update(self): return {"sources": {"enabled": False}} - @pytest.mark.xfail def test_source_two_configs_source_level(self, project): run_dbt(["parse"]) manifest = get_manifest(project.project_root) From bef9eb28ea1e773254d9b7352ede1153709ec193 Mon Sep 17 00:00:00 2001 From: Nathaniel May Date: Thu, 7 Apr 2022 15:45:20 -0400 Subject: [PATCH 14/21] updated remaining tests to pass --- .../functional/sources/test_source_configs.py | 32 ++----------------- 1 file changed, 3 insertions(+), 29 deletions(-) diff --git a/tests/functional/sources/test_source_configs.py b/tests/functional/sources/test_source_configs.py index 1409566a15d..dcdba066ff5 100644 --- a/tests/functional/sources/test_source_configs.py +++ b/tests/functional/sources/test_source_configs.py @@ -147,31 +147,18 @@ def models(self): @pytest.fixture(scope="class") def project_config_update(self): - return {"sources": {"tags": "project_level_important_tag"}} + return {"sources": {"enabled": True}} - @pytest.mark.xfail def test_source_all_configs_source_table(self, project): run_dbt(["parse"]) manifest = get_manifest(project.project_root) assert "source.test.test_source.test_table" in manifest.sources - assert "source.test.test_source.other_test_table" in manifest.sources + assert "source.test.test_source.other_test_table" not in manifest.sources config_test_table = manifest.sources.get("source.test.test_source.test_table").config - config_other_test_table = manifest.sources.get( - "source.test.test_source.other_test_table" - ).config assert isinstance(config_test_table, SourceConfig) - assert isinstance(config_other_test_table, SourceConfig) - - assert config_test_table != config_other_test_table assert config_test_table == pytest.expected_config - expected_source_level_config = SourceConfig( - enabled=True, - ) - - assert config_other_test_table == expected_source_level_config - all_configs_not_table_schema_yml = """version: 2 @@ -236,25 +223,12 @@ def models(self): def project_config_update(self): return {"sources": {"enabled": False}} - @pytest.mark.xfail def test_source_two_configs_source_table(self, project): run_dbt(["parse"]) manifest = get_manifest(project.project_root) assert "source.test.test_source.test_table" in manifest.sources - assert "source.test.test_source.other_test_table" in manifest.sources + assert "source.test.test_source.other_test_table" not in manifest.sources config_test_table = manifest.sources.get("source.test.test_source.test_table").config - config_other_test_table = manifest.sources.get( - "source.test.test_source.other_test_table" - ).config assert isinstance(config_test_table, SourceConfig) - assert isinstance(config_other_test_table, SourceConfig) - - assert config_test_table != config_other_test_table assert config_test_table == pytest.expected_config - - expected_project_level_config = SourceConfig( - enabled=True, - ) - - assert config_other_test_table == expected_project_level_config From b172fefe1c3a70b939d3ba31c9201867a6b0ff5f Mon Sep 17 00:00:00 2001 From: Nathaniel May Date: Thu, 7 Apr 2022 16:54:41 -0400 Subject: [PATCH 15/21] remove source override tests --- .../test_simple_source_override.py | 329 ------------------ 1 file changed, 329 deletions(-) delete mode 100644 tests/functional/source_overrides/test_simple_source_override.py diff --git a/tests/functional/source_overrides/test_simple_source_override.py b/tests/functional/source_overrides/test_simple_source_override.py deleted file mode 100644 index cd4b3d87204..00000000000 --- a/tests/functional/source_overrides/test_simple_source_override.py +++ /dev/null @@ -1,329 +0,0 @@ -from datetime import datetime, timedelta -import pytest -from dbt.contracts.graph.model_config import SourceConfig - - -from dbt.tests.util import run_dbt, update_config_file, check_relations_equal, get_manifest -from dbt.tests.fixtures.project import write_project_files -from tests.functional.source_overrides.fixtures import ( # noqa: F401 - local_dependency, - models__schema_yml, - seeds__expected_result_csv, - seeds__my_real_other_seed_csv, - seeds__my_real_seed_csv, -) - - -class TestSourceOverride: - @pytest.fixture(scope="class", autouse=True) - def setUp(self, project_root, local_dependency): # noqa: F811 - write_project_files(project_root, "local_dependency", local_dependency) - pytest._id = 101 - - @pytest.fixture(scope="class") - def models(self): - return {"schema.yml": models__schema_yml} - - @pytest.fixture(scope="class") - def seeds(self): - return { - "expected_result.csv": seeds__expected_result_csv, - "my_real_other_seed.csv": seeds__my_real_other_seed_csv, - "my_real_seed.csv": seeds__my_real_seed_csv, - } - - @pytest.fixture(scope="class") - def packages(self): - return { - "packages": [ - { - "local": "local_dependency", - }, - ] - } - - def _set_updated_at_to(self, delta, project): - insert_time = datetime.utcnow() + delta - timestr = insert_time.strftime("%Y-%m-%d %H:%M:%S") - # favorite_color,id,first_name,email,ip_address,updated_at - - quoted_columns = ",".join( - project.adapter.quote(c) - for c in ("favorite_color", "id", "first_name", "email", "ip_address", "updated_at") - ) - insert_id = pytest._id - pytest._id += 1 - - kwargs = { - "schema": project.test_schema, - "time": timestr, - "id": insert_id, - "source": project.adapter.quote("snapshot_freshness_base"), - "quoted_columns": quoted_columns, - } - - raw_sql = """INSERT INTO {schema}.{source} - ({quoted_columns}) - VALUES ( - 'blue',{id},'Jake','abc@example.com','192.168.1.1','{time}' - )""".format( - **kwargs - ) - - project.run_sql(raw_sql) - - @pytest.fixture(scope="class") - def project_config_update(self): - return { - "seeds": { - "localdep": { - "enabled": False, - "keep": { - "enabled": True, - }, - }, - "quote_columns": False, - }, - "sources": { - "localdep": { - "my_other_source": { - "enabled": False, - } - } - }, - } - - def test_source_overrides(self, project): - run_dbt(["deps"]) - - seed_results = run_dbt(["seed"]) - assert len(seed_results) == 5 - - # There should be 7, as we disabled 1 test of the original 8 - test_results = run_dbt(["test"]) - assert len(test_results) == 7 - - results = run_dbt(["run"]) - assert len(results) == 1 - - check_relations_equal(project.adapter, ["expected_result", "my_model"]) - - # set the updated_at field of this seed to last week - self._set_updated_at_to(timedelta(days=-7), project) - # if snapshot-freshness fails, freshness just didn't happen! - results = run_dbt(["source", "snapshot-freshness"], expect_pass=False) - # we disabled my_other_source, so we only run the one freshness check - # in - assert len(results) == 1 - # If snapshot-freshness passes, that means error_after was - # applied from the source override but not the source table override - self._set_updated_at_to(timedelta(days=-2), project) - results = run_dbt( - ["source", "snapshot-freshness"], - expect_pass=False, - ) - assert len(results) == 1 - - self._set_updated_at_to(timedelta(hours=-12), project) - results = run_dbt(["source", "snapshot-freshness"], expect_pass=True) - assert len(results) == 1 - - # update source to be enabled - new_source_config = { - "sources": { - "localdep": { - "my_other_source": { - "enabled": True, - } - } - } - } - update_config_file(new_source_config, project.project_root, "dbt_project.yml") - - # enable my_other_source, snapshot freshness should fail due to the new - # not-fresh source - results = run_dbt(["source", "snapshot-freshness"], expect_pass=False) - assert len(results) == 2 - - -class SourceOverrideTest: - @pytest.fixture(scope="class", autouse=True) - def setUp(self, project_root, local_dependency): # noqa: F811 - write_project_files(project_root, "local_dependency", local_dependency) - - pytest.expected_config = SourceConfig( - enabled=True, - quoting={"database": True, "schema": True, "identifier": True, "column": True}, - freshness={ - "warn_after": {"count": 1, "period": "minute"}, - "error_after": {"count": 5, "period": "minute"}, - }, - loader="override_a_loader", - loaded_at_field="override_some_column", - database="override_custom_database", - schema="override_custom_schema", - meta={"languages": ["java"]}, - tags=["override_important_tag"], - ) - - @pytest.fixture(scope="class") - def project_config_update(self): - return { - "sources": { - "localdep": { - "my_source": { - "quoting": { - "database": False, - "schema": False, - "identifier": False, - "column": False, - }, - "freshness": { - "error_after": {"count": 24, "period": "hour"}, - "warn_after": {"count": 12, "period": "hour"}, - }, - "loader": "a_loader", - "loaded_at_field": "some_column", - "database": "custom_database", - "schema": "custom_schema", - "meta": {"languages": ["python"]}, - "tags": ["important_tag"], - } - } - }, - } - - @pytest.fixture(scope="class") - def packages(self): - return { - "packages": [ - { - "local": "local_dependency", - }, - ] - } - - -overrides_source_level__schema_yml = """ -version: 2 -sources: - - name: my_source - overrides: localdep - config: - quoting: - database: True - schema: True - identifier: True - column: True - freshness: - error_after: {count: 1, period: minute} - warn_after: {count: 5, period: minute} - loader: "override_a_loader" - loaded_at_field: override_some_column - database: override_custom_database - schema: override_custom_schema - meta: {'languages': ['java']} - tags: ["override_important_tag"] - tables: - - name: my_table - - name: my_other_table - - name: snapshot_freshness -""" - - -# test overriding at the source level -# expect fail since these are no valid configs right now -class TestSourceLevelOverride(SourceOverrideTest): - @pytest.fixture(scope="class") - def models(self): - return {"schema.yml": overrides_source_level__schema_yml} - - # @pytest.mark.xfail - def test_source_level_overrides(self, project): - run_dbt(["deps"]) - - # this currently fails because configs fail parsing under an override - run_dbt(["parse"]) - manifest = get_manifest(project.project_root) - - assert "source.localdep.my_source.my_table" in manifest.sources - assert "source.localdep.my_source.my_other_table" in manifest.sources - assert "source.localdep.my_source.snapshot_freshness" in manifest.sources - - config_my_table = manifest.sources.get("source.localdep.my_source.my_table").config - config_my_other_table = manifest.sources.get( - "source.localdep.my_source.my_other_table" - ).config - config_snapshot_freshness_table = manifest.sources.get( - "source.localdep.my_source.snapshot_freshness" - ).config - - assert isinstance(config_my_table, SourceConfig) - assert isinstance(config_my_other_table, SourceConfig) - - assert config_my_table == config_my_other_table - assert config_my_table == config_snapshot_freshness_table - assert config_my_table == pytest.expected_config - - -overrides_source_level__schema_yml = """ -version: 2 -sources: - - name: my_source - overrides: localdep - tables: - - name: my_table - - name: my_other_table - config: - quoting: - database: True - schema: True - identifier: True - column: True - freshness: - error_after: {count: 1, period: minute} - warn_after: {count: 5, period: minute} - loader: "override_a_loader" - loaded_at_field: override_some_column - database: override_custom_database - schema: override_custom_schema - meta: {'languages': ['java']} - tags: ["override_important_tag"] - - name: snapshot_freshness -""" - - -# test overriding at the source table level -# expect fail since these are no valid configs right now -class TestSourceTableOverride(SourceOverrideTest): - @pytest.fixture(scope="class") - def models(self): - return {"schema.yml": overrides_source_level__schema_yml} - - # @pytest.mark.xfail - def test_source_table_overrides(self, project): - run_dbt(["deps"]) - - # this currently fails because configs fail parsing under an override - run_dbt(["parse"]) - manifest = get_manifest(project.project_root) - - assert "source.localdep.my_source.my_table" in manifest.sources - assert "source.localdep.my_source.my_other_table" in manifest.sources - assert "source.localdep.my_source.snapshot_freshness" in manifest.sources - - config_my_table = manifest.sources.get("source.localdep.my_source.my_table").config - config_my_other_table = manifest.sources.get( - "source.localdep.my_source.my_other_table" - ).config - config_snapshot_freshness_table = manifest.sources.get( - "source.localdep.my_source.snapshot_freshness" - ).config - - assert isinstance(config_my_table, SourceConfig) - assert isinstance(config_my_other_table, SourceConfig) - - assert config_my_table != config_my_other_table - assert config_my_table == config_snapshot_freshness_table - - assert config_my_other_table == pytest.expected_config From 08d70cf38dd699b909f091e71c83aa1053207ce3 Mon Sep 17 00:00:00 2001 From: Nathaniel May Date: Fri, 8 Apr 2022 12:05:30 -0400 Subject: [PATCH 16/21] add comment for config merging --- core/dbt/parser/sources.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/core/dbt/parser/sources.py b/core/dbt/parser/sources.py index 155e223ad8b..28c0428cbbe 100644 --- a/core/dbt/parser/sources.py +++ b/core/dbt/parser/sources.py @@ -271,6 +271,8 @@ def _generate_source_config(self, target: UnpatchedSourceDefinition, rendered: b # first apply source configs precedence_configs.update(target.source.config) # then overrite anything that is defined on source tables + # this is not quite complex enough for configs that can be set as top-level node keys, but + # it works while source configs can only include `enabled`. precedence_configs.update(target.table.config) return generator.calculate_node_config( From 5c515d9e6a1c6e940601f326f51d043ce90d9344 Mon Sep 17 00:00:00 2001 From: Nathaniel May Date: Fri, 8 Apr 2022 13:27:35 -0400 Subject: [PATCH 17/21] changelog --- .changes/unreleased/Features-20220408-132725.yaml | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 .changes/unreleased/Features-20220408-132725.yaml diff --git a/.changes/unreleased/Features-20220408-132725.yaml b/.changes/unreleased/Features-20220408-132725.yaml new file mode 100644 index 00000000000..778c54257d2 --- /dev/null +++ b/.changes/unreleased/Features-20220408-132725.yaml @@ -0,0 +1,7 @@ +kind: Features +body: add enabled as a source config +time: 2022-04-08T13:27:25.292454-04:00 +custom: + Author: nathaniel-may + Issue: "3662" + PR: "5008" From 612299de3714159e5f48fcb6b605c26611de4e08 Mon Sep 17 00:00:00 2001 From: Nathaniel May Date: Fri, 8 Apr 2022 14:49:47 -0400 Subject: [PATCH 18/21] remove old comments --- tests/functional/sources/test_source_configs.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/tests/functional/sources/test_source_configs.py b/tests/functional/sources/test_source_configs.py index dcdba066ff5..23d0d6fd5de 100644 --- a/tests/functional/sources/test_source_configs.py +++ b/tests/functional/sources/test_source_configs.py @@ -109,7 +109,6 @@ def test_source_config_yaml_source_level(self, project): # Test enabled config at source table level in yaml file -# expect fail - not implemented class TestConfigYamlSourceTable(SourceConfigTests): @pytest.fixture(scope="class") def models(self): @@ -139,7 +138,6 @@ def test_source_config_yaml_source_table(self, project): # Test inheritence - set configs at project, source, and source-table level - expect source-table level to win -# expect fail - not implemented class TestSourceConfigsInheritence1(SourceConfigTests): @pytest.fixture(scope="class") def models(self): @@ -173,7 +171,6 @@ def test_source_all_configs_source_table(self, project): # Test inheritence - set configs at project and source level - expect source level to win -# expect fail - not implemented class TestSourceConfigsInheritence2(SourceConfigTests): @pytest.fixture(scope="class") def models(self): @@ -213,7 +210,6 @@ def test_source_two_configs_source_level(self, project): # Test inheritence - set configs at project and source-table level - expect source-table level to win -# expect fail - not implemented class TestSourceConfigsInheritence3(SourceConfigTests): @pytest.fixture(scope="class") def models(self): From f3ddf3b72fab460afb99bb691bb3537659c2596e Mon Sep 17 00:00:00 2001 From: Chenyu Li Date: Fri, 8 Apr 2022 14:02:20 -0600 Subject: [PATCH 19/21] hacky fix for permission test --- tests/functional/permission/test_permissions.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/tests/functional/permission/test_permissions.py b/tests/functional/permission/test_permissions.py index 9e89bcede50..64d4e5e7be1 100644 --- a/tests/functional/permission/test_permissions.py +++ b/tests/functional/permission/test_permissions.py @@ -48,18 +48,19 @@ def test_no_create_schema_permissions( project.run_sql('drop schema if exists "{}" cascade'.format(project.test_schema)) with pytest.raises(RuntimeError): run_dbt(["run", "--target", "noaccess"], expect_pass=False) + # this command will update the adapter to use default profile again + run_dbt(["run", "--target", "default"], expect_pass=False) def test_create_schema_permissions( self, project, ): # now it should work! - project.run_sql("grant create on database {} to noaccess".format(project.database)) project.run_sql( 'grant usage, create on schema "{}" to noaccess'.format(project.test_schema) ) - project.run_sql( - 'grant select on all tables in schema "{}" to noaccess'.format(project.test_schema) - ) results = run_dbt(["run", "--target", "noaccess"]) assert len(results) == 1 + # this command will update the adapter to use default profile again + # otherwise we are going to have issue tear down + run_dbt(["run", "--target", "default"]) From 20422628fae08a674a60fbaf725b438ee7652389 Mon Sep 17 00:00:00 2001 From: Nathaniel May Date: Fri, 8 Apr 2022 16:34:32 -0400 Subject: [PATCH 20/21] remove unhelpful test --- .../functional/permission/test_permissions.py | 66 ------------------- 1 file changed, 66 deletions(-) delete mode 100644 tests/functional/permission/test_permissions.py diff --git a/tests/functional/permission/test_permissions.py b/tests/functional/permission/test_permissions.py deleted file mode 100644 index 64d4e5e7be1..00000000000 --- a/tests/functional/permission/test_permissions.py +++ /dev/null @@ -1,66 +0,0 @@ -import pytest -import os - -from dbt.tests.util import run_dbt -from tests.functional.permission.fixtures import models, project_files # noqa: F401 - - -class TestPermissions: - @pytest.fixture(autouse=True) - def setUp(self, project): - path = os.path.join(project.test_data_dir, "seed.sql") - project.run_sql_file(path) - - @pytest.fixture(scope="class") - def profiles_config_update(self, unique_schema): - return { - "test": { - "outputs": { - "default": { - "type": "postgres", - "threads": 4, - "host": "localhost", - "port": int(os.getenv("POSTGRES_TEST_PORT", 5432)), - "user": os.getenv("POSTGRES_TEST_USER", "root"), - "pass": os.getenv("POSTGRES_TEST_PASS", "password"), - "dbname": os.getenv("POSTGRES_TEST_DATABASE", "dbt"), - "schema": unique_schema, - }, - "noaccess": { - "type": "postgres", - "threads": 4, - "host": "localhost", - "port": int(os.getenv("POSTGRES_TEST_PORT", 5432)), - "user": "noaccess", - "pass": "password", - "dbname": os.getenv("POSTGRES_TEST_DATABASE", "dbt"), - "schema": unique_schema, - }, - }, - } - } - - def test_no_create_schema_permissions( - self, - project, - ): - # the noaccess user does not have permissions to create a schema -- this should fail - project.run_sql('drop schema if exists "{}" cascade'.format(project.test_schema)) - with pytest.raises(RuntimeError): - run_dbt(["run", "--target", "noaccess"], expect_pass=False) - # this command will update the adapter to use default profile again - run_dbt(["run", "--target", "default"], expect_pass=False) - - def test_create_schema_permissions( - self, - project, - ): - # now it should work! - project.run_sql( - 'grant usage, create on schema "{}" to noaccess'.format(project.test_schema) - ) - results = run_dbt(["run", "--target", "noaccess"]) - assert len(results) == 1 - # this command will update the adapter to use default profile again - # otherwise we are going to have issue tear down - run_dbt(["run", "--target", "default"]) From 582be80c99001fdc0d4287a7faaaa6e2b468548c Mon Sep 17 00:00:00 2001 From: Nathaniel May Date: Fri, 8 Apr 2022 16:37:05 -0400 Subject: [PATCH 21/21] adding back test file that was accidentally deleted --- .../test_simple_source_override.py | 145 ++++++++++++++++++ 1 file changed, 145 insertions(+) create mode 100644 tests/functional/source_overrides/test_simple_source_override.py diff --git a/tests/functional/source_overrides/test_simple_source_override.py b/tests/functional/source_overrides/test_simple_source_override.py new file mode 100644 index 00000000000..7ff5ef09cab --- /dev/null +++ b/tests/functional/source_overrides/test_simple_source_override.py @@ -0,0 +1,145 @@ +from datetime import datetime, timedelta +import pytest + +from dbt.tests.util import run_dbt, update_config_file, check_relations_equal +from dbt.tests.fixtures.project import write_project_files +from tests.functional.source_overrides.fixtures import ( # noqa: F401 + local_dependency, + models__schema_yml, + seeds__expected_result_csv, + seeds__my_real_other_seed_csv, + seeds__my_real_seed_csv, +) + + +class TestSourceOverride: + @pytest.fixture(scope="class", autouse=True) + def setUp(self, project_root, local_dependency): # noqa: F811 + write_project_files(project_root, "local_dependency", local_dependency) + + @pytest.fixture(scope="class") + def models(self): + return {"schema.yml": models__schema_yml} + + @pytest.fixture(scope="class") + def seeds(self): + return { + "expected_result.csv": seeds__expected_result_csv, + "my_real_other_seed.csv": seeds__my_real_other_seed_csv, + "my_real_seed.csv": seeds__my_real_seed_csv, + } + + @pytest.fixture(scope="class") + def packages(self): + return { + "packages": [ + { + "local": "local_dependency", + }, + ] + } + + @pytest.fixture(scope="class") + def project_config_update(self): + return { + "seeds": { + "localdep": { + "enabled": False, + "keep": { + "enabled": True, + }, + }, + "quote_columns": False, + }, + "sources": { + "localdep": { + "my_other_source": { + "enabled": False, + } + } + }, + } + + def _set_updated_at_to(self, insert_id, delta, project): + insert_time = datetime.utcnow() + delta + timestr = insert_time.strftime("%Y-%m-%d %H:%M:%S") + # favorite_color,id,first_name,email,ip_address,updated_at + + quoted_columns = ",".join( + project.adapter.quote(c) + for c in ("favorite_color", "id", "first_name", "email", "ip_address", "updated_at") + ) + + kwargs = { + "schema": project.test_schema, + "time": timestr, + "id": insert_id, + "source": project.adapter.quote("snapshot_freshness_base"), + "quoted_columns": quoted_columns, + } + + raw_sql = """INSERT INTO {schema}.{source} + ({quoted_columns}) + VALUES ( + 'blue',{id},'Jake','abc@example.com','192.168.1.1','{time}' + )""".format( + **kwargs + ) + + project.run_sql(raw_sql) + + return insert_id + 1 + + def test_source_overrides(self, project): + insert_id = 101 + + run_dbt(["deps"]) + + seed_results = run_dbt(["seed"]) + assert len(seed_results) == 5 + + # There should be 7, as we disabled 1 test of the original 8 + test_results = run_dbt(["test"]) + assert len(test_results) == 7 + + results = run_dbt(["run"]) + assert len(results) == 1 + + check_relations_equal(project.adapter, ["expected_result", "my_model"]) + + # set the updated_at field of this seed to last week + insert_id = self._set_updated_at_to(insert_id, timedelta(days=-7), project) + # if snapshot-freshness fails, freshness just didn't happen! + results = run_dbt(["source", "snapshot-freshness"], expect_pass=False) + # we disabled my_other_source, so we only run the one freshness check + # in + assert len(results) == 1 + # If snapshot-freshness passes, that means error_after was + # applied from the source override but not the source table override + insert_id = self._set_updated_at_to(insert_id, timedelta(days=-2), project) + results = run_dbt( + ["source", "snapshot-freshness"], + expect_pass=False, + ) + assert len(results) == 1 + + insert_id = self._set_updated_at_to(insert_id, timedelta(hours=-12), project) + results = run_dbt(["source", "snapshot-freshness"], expect_pass=True) + assert len(results) == 1 + + # update source to be enabled + new_source_config = { + "sources": { + "localdep": { + "my_other_source": { + "enabled": True, + } + } + } + } + update_config_file(new_source_config, project.project_root, "dbt_project.yml") + + # enable my_other_source, snapshot freshness should fail due to the new + # not-fresh source + results = run_dbt(["source", "snapshot-freshness"], expect_pass=False) + assert len(results) == 2