From 0fa9d0ab1269fe84ab96a3f576a12f0187a0f3cc Mon Sep 17 00:00:00 2001 From: Jacob Beck Date: Tue, 12 May 2020 07:09:00 -0600 Subject: [PATCH 1/2] make full refresh a config item Added integration tests, fixed existing tests --- CHANGELOG.md | 5 +- core/dbt/contracts/graph/model_config.py | 1 + .../macros/etc/is_incremental.sql | 2 +- .../macros/materializations/helpers.sql | 9 +++ .../incremental/incremental.sql | 3 +- .../macros/materializations/seed/seed.sql | 2 +- .../view/create_or_replace_view.sql | 2 +- .../macros/materializations/incremental.sql | 22 +++---- .../macros/materializations/incremental.sql | 2 +- .../models-downstream-seed/model.sql | 1 + .../005_simple_seed_test/test_simple_seed.py | 66 ++++++++++++++----- .../models/already_exists.sql | 2 +- .../test_runtime_materialization.py | 20 +++++- .../test_docs_generate.py | 24 +++++++ test/integration/047_dbt_ls_test/test_ls.py | 12 +++- 15 files changed, 133 insertions(+), 40 deletions(-) create mode 100644 test/integration/005_simple_seed_test/models-downstream-seed/model.sql diff --git a/CHANGELOG.md b/CHANGELOG.md index 768e1d098a8..b78e82be3aa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,8 @@ ## dbt 0.18.0 (Release TBD) +### Features +- Added a `full_refresh` config item that overrides the behavior of the `--full-refresh` flag ([#1009](https://github.com/fishtown-analytics/dbt/issues/1009), [#2348](https://github.com/fishtown-analytics/dbt/pull/2348)) + ## dbt 0.17.0 (Release TBD) ## dbt 0.17.0rc1 (May 12, 2020) @@ -28,7 +31,7 @@ - Track distinct project hashes in anonymous usage metrics for package downloads ([#2351](https://github.com/fishtown-analytics/dbt/issues/2351), [#2429](https://github.com/fishtown-analytics/dbt/pull/2429)) Contributors: - - [@azhard](https://github.com/azhard) ([#2413](https://github.com/fishtown-analytics/dbt/pull/2413), [#2422](https://github.com/fishtown-analytics/dbt/pull/2422)) + - [@azhard](https://github.com/azhard) ([#2413](https://github.com/fishtown-analytics/dbt/pull/2413), [#2422](https://github.com/fishtown-analytics/dbt/pull/2422)) - [@mikaelene](https://github.com/mikaelene) [#2414](https://github.com/fishtown-analytics/dbt/pull/2414) - [@raalsky](https://github.com/Raalsky) ([#2343](https://github.com/fishtown-analytics/dbt/pull/2343)) - [@alf-mindshift](https://github.com/alf-mindshift) ([docs#90](https://github.com/fishtown-analytics/dbt-docs/pull/90)) diff --git a/core/dbt/contracts/graph/model_config.py b/core/dbt/contracts/graph/model_config.py index 468695b521e..b3b8fa745f8 100644 --- a/core/dbt/contracts/graph/model_config.py +++ b/core/dbt/contracts/graph/model_config.py @@ -321,6 +321,7 @@ class NodeConfig(BaseConfig): # TODO: hide this one? metadata=MergeBehavior.Append.meta(), ) + full_refresh: Optional[bool] = None @classmethod def from_dict(cls, data, validate=True): diff --git a/core/dbt/include/global_project/macros/etc/is_incremental.sql b/core/dbt/include/global_project/macros/etc/is_incremental.sql index 2d02e8e3c90..10f45e0238a 100644 --- a/core/dbt/include/global_project/macros/etc/is_incremental.sql +++ b/core/dbt/include/global_project/macros/etc/is_incremental.sql @@ -8,6 +8,6 @@ {{ return(relation is not none and relation.type == 'table' and model.config.materialized == 'incremental' - and not flags.FULL_REFRESH) }} + and not should_full_refresh()) }} {% endif %} {% endmacro %} diff --git a/core/dbt/include/global_project/macros/materializations/helpers.sql b/core/dbt/include/global_project/macros/materializations/helpers.sql index 871bfd1c408..caf730c4d5f 100644 --- a/core/dbt/include/global_project/macros/materializations/helpers.sql +++ b/core/dbt/include/global_project/macros/materializations/helpers.sql @@ -63,3 +63,12 @@ identifier=relation.identifier )) -%} {% endmacro %} + + +{% macro should_full_refresh() %} + {% set config_full_refresh = config.get('full_refresh', false) %} + {% if config_full_refresh is none %} + {% set config_full_refresh = flags.FULL_REFRESH %} + {% endif %} + {% do return(config_full_refresh) %} +{% endmacro %} diff --git a/core/dbt/include/global_project/macros/materializations/incremental/incremental.sql b/core/dbt/include/global_project/macros/materializations/incremental/incremental.sql index ef304ad1796..98e85865bc2 100644 --- a/core/dbt/include/global_project/macros/materializations/incremental/incremental.sql +++ b/core/dbt/include/global_project/macros/materializations/incremental/incremental.sql @@ -2,7 +2,6 @@ {% materialization incremental, default -%} {% set unique_key = config.get('unique_key') %} - {% set full_refresh_mode = flags.FULL_REFRESH %} {% set target_relation = this.incorporate(type='table') %} {% set existing_relation = load_relation(this) %} @@ -16,7 +15,7 @@ {% set to_drop = [] %} {% if existing_relation is none %} {% set build_sql = create_table_as(False, target_relation, sql) %} - {% elif existing_relation.is_view or full_refresh_mode %} + {% elif existing_relation.is_view or should_full_refresh() %} {#-- Make sure the backup doesn't exist so we don't encounter issues with the rename below #} {% set backup_identifier = existing_relation.identifier ~ "__dbt_backup" %} {% set backup_relation = existing_relation.incorporate(path={"identifier": backup_identifier}) %} diff --git a/core/dbt/include/global_project/macros/materializations/seed/seed.sql b/core/dbt/include/global_project/macros/materializations/seed/seed.sql index 490ca1457d1..ad2221c359c 100644 --- a/core/dbt/include/global_project/macros/materializations/seed/seed.sql +++ b/core/dbt/include/global_project/macros/materializations/seed/seed.sql @@ -104,7 +104,7 @@ {% materialization seed, default %} {%- set identifier = model['alias'] -%} - {%- set full_refresh_mode = (flags.FULL_REFRESH == True) -%} + {%- set full_refresh_mode = (should_full_refresh()) -%} {%- set old_relation = adapter.get_relation(database=database, schema=schema, identifier=identifier) -%} diff --git a/core/dbt/include/global_project/macros/materializations/view/create_or_replace_view.sql b/core/dbt/include/global_project/macros/materializations/view/create_or_replace_view.sql index afcfd50ca59..1d59f4de54f 100644 --- a/core/dbt/include/global_project/macros/materializations/view/create_or_replace_view.sql +++ b/core/dbt/include/global_project/macros/materializations/view/create_or_replace_view.sql @@ -42,7 +42,7 @@ -- that's an error. If we were told to full refresh, drop it. This behavior differs -- for Snowflake and BigQuery, so multiple dispatch is used. {%- if old_relation is not none and old_relation.is_table -%} - {{ handle_existing_table(flags.FULL_REFRESH, old_relation) }} + {{ handle_existing_table(should_full_refresh(), old_relation) }} {%- endif -%} -- build model diff --git a/plugins/bigquery/dbt/include/bigquery/macros/materializations/incremental.sql b/plugins/bigquery/dbt/include/bigquery/macros/materializations/incremental.sql index 28ceedebc77..fb6cccb2568 100644 --- a/plugins/bigquery/dbt/include/bigquery/macros/materializations/incremental.sql +++ b/plugins/bigquery/dbt/include/bigquery/macros/materializations/incremental.sql @@ -17,11 +17,11 @@ {% macro bq_insert_overwrite(tmp_relation, target_relation, sql, unique_key, partition_by, partitions, dest_columns) %} {%- set partition_type = - 'date' if partition_by.data_type in ('timestamp, datetime') + 'date' if partition_by.data_type in ('timestamp, datetime') else partition_by.data_type -%} {% if partitions is not none and partitions != [] %} {# static #} - + {% set predicate -%} {{ partition_by.render(alias='DBT_INTERNAL_DEST') }} in ( {{ partitions | join (', ') }} @@ -33,11 +33,11 @@ {{sql}} ) {%- endset -%} - + {{ get_insert_overwrite_merge_sql(target_relation, source_sql, dest_columns, [predicate], include_sql_header=true) }} - + {% else %} {# dynamic #} - + {% set predicate -%} {{ partition_by.render(alias='DBT_INTERNAL_DEST') }} in unnest(dbt_partitions_for_replacement) {%- endset %} @@ -65,7 +65,7 @@ array_agg(distinct {{ partition_by.render() }}) from {{ tmp_relation }} ); - + {# TODO: include_sql_header is a hack; consider a better approach that includes the sql_header at the materialization-level instead @@ -75,7 +75,7 @@ -- 4. clean up the temp table drop table if exists {{ tmp_relation }} - + {% endif %} {% endmacro %} @@ -84,12 +84,12 @@ {% materialization incremental, adapter='bigquery' -%} {%- set unique_key = config.get('unique_key') -%} - {%- set full_refresh_mode = (flags.FULL_REFRESH == True) -%} + {%- set full_refresh_mode = (should_full_refresh()) -%} {%- set target_relation = this %} {%- set existing_relation = load_relation(this) %} {%- set tmp_relation = make_temp_relation(this) %} - + {#-- Validate early so we don't run SQL if the strategy is invalid --#} {% set strategy = dbt_bigquery_validate_get_incremental_strategy(config) -%} @@ -118,14 +118,14 @@ {#-- if partitioned, use BQ scripting to get the range of partition values to be updated --#} {% if strategy == 'insert_overwrite' %} - + {% set missing_partition_msg -%} The 'insert_overwrite' strategy requires the `partition_by` config. {%- endset %} {% if partition_by is none %} {% do exceptions.raise_compiler_error(missing_partition_msg) %} {% endif %} - + {% set build_sql = bq_insert_overwrite( tmp_relation, target_relation, diff --git a/plugins/snowflake/dbt/include/snowflake/macros/materializations/incremental.sql b/plugins/snowflake/dbt/include/snowflake/macros/materializations/incremental.sql index cdb660b0990..1c8bfbde3c1 100644 --- a/plugins/snowflake/dbt/include/snowflake/macros/materializations/incremental.sql +++ b/plugins/snowflake/dbt/include/snowflake/macros/materializations/incremental.sql @@ -27,7 +27,7 @@ {% materialization incremental, adapter='snowflake' -%} {%- set unique_key = config.get('unique_key') -%} - {%- set full_refresh_mode = (flags.FULL_REFRESH == True) -%} + {%- set full_refresh_mode = (should_full_refresh()) -%} {% set target_relation = this %} {% set existing_relation = load_relation(this) %} diff --git a/test/integration/005_simple_seed_test/models-downstream-seed/model.sql b/test/integration/005_simple_seed_test/models-downstream-seed/model.sql new file mode 100644 index 00000000000..6d56d7de539 --- /dev/null +++ b/test/integration/005_simple_seed_test/models-downstream-seed/model.sql @@ -0,0 +1 @@ +select * from {{ ref('seed_actual') }} diff --git a/test/integration/005_simple_seed_test/test_simple_seed.py b/test/integration/005_simple_seed_test/test_simple_seed.py index f1c3cd67817..cd5b46c7c9b 100644 --- a/test/integration/005_simple_seed_test/test_simple_seed.py +++ b/test/integration/005_simple_seed_test/test_simple_seed.py @@ -7,7 +7,6 @@ class TestSimpleSeed(DBTIntegrationTest): def setUp(self): DBTIntegrationTest.setUp(self) - self.run_sql_file("seed.sql") @property @@ -16,7 +15,7 @@ def schema(self): @property def models(self): - return "models" + return "models-downstream-seed" @property def project_config(self): @@ -28,27 +27,61 @@ def project_config(self): } } + def use_full_refresh_project(self, full_refresh: bool): + overrides = { + 'seeds': { + 'quote_columns': False, + 'full_refresh': full_refresh, + } + } + self.use_default_project(overrides) + + def _seed_and_run(self): + assert len(self.run_dbt(['seed'])) == 1 + self.assertTablesEqual('seed_actual', 'seed_expected') + + assert len(self.run_dbt(['run'])) == 1 + self.assertTablesEqual('model', 'seed_expected') + + def _after_seed_model_state(self, cmd, exists: bool): + assert len(self.run_dbt(cmd)) == 1 + self.assertTablesEqual('seed_actual', 'seed_expected') + if exists: + self.assertTableDoesExist('model') + else: + self.assertTableDoesNotExist('model') + @use_profile('postgres') def test_postgres_simple_seed(self): - results = self.run_dbt(["seed"]) - self.assertEqual(len(results), 1) - self.assertTablesEqual("seed_actual","seed_expected") + self._seed_and_run() # this should truncate the seed_actual table, then re-insert. - results = self.run_dbt(["seed"]) - self.assertEqual(len(results), 1) - self.assertTablesEqual("seed_actual","seed_expected") + self._after_seed_model_state(['seed'], exists=True) @use_profile('postgres') - def test_postgres_simple_seed_with_drop(self): - results = self.run_dbt(["seed"]) - self.assertEqual(len(results), 1) - self.assertTablesEqual("seed_actual","seed_expected") + def test_postgres_simple_seed_full_refresh_flag(self): + self._seed_and_run() - # this should drop the seed table, then re-create - results = self.run_dbt(["seed", "--full-refresh"]) - self.assertEqual(len(results), 1) - self.assertTablesEqual("seed_actual","seed_expected") + # this should drop the seed_actual table, then re-create it, so the + # model won't exist. + self._after_seed_model_state(['seed', '--full-refresh'], exists=False) + + @use_profile('postgres') + def test_postgres_simple_seed_full_refresh_config(self): + self._seed_and_run() + + # set the full_refresh config to False + self.use_full_refresh_project(False) + + self._after_seed_model_state(['seed'], exists=True) + # make sure we ignore the full-refresh flag (the config is higher + # priority than the flag) + self._after_seed_model_state(['seed', '--full-refresh'], exists=True) + + # this should drop the seed_actual table, then re-create it, so the + # model won't exist. + self.use_full_refresh_project(True) + self._after_seed_model_state(['seed'], exists=False) class TestSimpleSeedCustomSchema(DBTIntegrationTest): @@ -89,7 +122,6 @@ def test_postgres_simple_seed_with_schema(self): self.assertEqual(len(results), 1) self.assertTablesEqual("seed_actual","seed_expected", table_a_schema=schema_name) - @use_profile('postgres') def test_postgres_simple_seed_with_drop_and_schema(self): schema_name = "{}_{}".format(self.unique_schema(), 'custom_schema') diff --git a/test/integration/012_deprecation_tests/models/already_exists.sql b/test/integration/012_deprecation_tests/models/already_exists.sql index 0290d5e1c10..c215a179a26 100644 --- a/test/integration/012_deprecation_tests/models/already_exists.sql +++ b/test/integration/012_deprecation_tests/models/already_exists.sql @@ -1,5 +1,5 @@ select 1 as id -{% if adapter.already_exists(this.schema, this.identifier) and not flags.FULL_REFRESH %} +{% if adapter.already_exists(this.schema, this.identifier) and not should_full_refresh() %} where id > (select max(id) from {{this}}) {% endif %} diff --git a/test/integration/017_runtime_materialization_tests/test_runtime_materialization.py b/test/integration/017_runtime_materialization_tests/test_runtime_materialization.py index 2e3ba9c1af4..59be6280ac6 100644 --- a/test/integration/017_runtime_materialization_tests/test_runtime_materialization.py +++ b/test/integration/017_runtime_materialization_tests/test_runtime_materialization.py @@ -25,10 +25,13 @@ def schema(self): def models(self): return "models" + def run_dbt_full_refresh(self): + return self.run_dbt(['run', '--full-refresh']) + @use_profile('postgres') def test_postgres_full_refresh(self): # initial full-refresh should have no effect - results = self.run_dbt(['run', '--full-refresh']) + results = self.run_dbt_full_refresh() self.assertEqual(len(results), 3) self.assertTablesEqual("seed", "view") @@ -37,13 +40,13 @@ def test_postgres_full_refresh(self): # adds one record to the incremental model. full-refresh should truncate then re-run self.run_sql_file("invalidate_incremental.sql") - results = self.run_dbt(['run', '--full-refresh']) + results = self.run_dbt_full_refresh() self.assertEqual(len(results), 3) self.assertTablesEqual("seed", "incremental") self.run_sql_file("update.sql") - results = self.run_dbt(['run', '--full-refresh']) + results = self.run_dbt_full_refresh() self.assertEqual(len(results), 3) self.assertTablesEqual("seed", "view") @@ -59,3 +62,14 @@ def test_postgres_delete__dbt_tmp_relation(self): self.assertTableDoesNotExist('view__dbt_tmp') self.assertTablesEqual("seed", "view") + + +class TestRuntimeMaterializationWithConfig(TestRuntimeMaterialization): + @property + def project_config(self): + result = super().project_config + result.update({'models': {'full_refresh': True}}) + return result + + def run_dbt_full_refresh(self): + return self.run_dbt(['run']) diff --git a/test/integration/029_docs_generate_tests/test_docs_generate.py b/test/integration/029_docs_generate_tests/test_docs_generate.py index 153eefc5d6b..310aa64afca 100644 --- a/test/integration/029_docs_generate_tests/test_docs_generate.py +++ b/test/integration/029_docs_generate_tests/test_docs_generate.py @@ -898,6 +898,7 @@ def expected_seeded_manifest(self, model_database=None): 'quoting': {}, 'tags': [], 'persist_docs': {}, + 'full_refresh': None, } return { @@ -983,6 +984,7 @@ def expected_seeded_manifest(self, model_database=None): 'quoting': {}, 'tags': [], 'quote_columns': True, + 'full_refresh': None, }, 'patch_path': seed_schema_yml_path, 'path': 'seed.csv', @@ -1064,6 +1066,7 @@ def expected_seeded_manifest(self, model_database=None): 'vars': {}, 'tags': [], 'severity': 'ERROR', + 'full_refresh': None, }, 'sources': [], 'depends_on': { @@ -1117,6 +1120,7 @@ def expected_seeded_manifest(self, model_database=None): 'vars': {}, 'tags': [], 'severity': 'ERROR', + 'full_refresh': None, }, 'sources': [], 'depends_on': { @@ -1169,6 +1173,7 @@ def expected_seeded_manifest(self, model_database=None): 'vars': {}, 'tags': [], 'severity': 'ERROR', + 'full_refresh': None, }, 'sources': [], 'depends_on': { @@ -1286,6 +1291,7 @@ def expected_postgres_references_manifest(self, model_database=None): 'quoting': {}, 'vars': {}, 'tags': [], + 'full_refresh': None, }, 'sources': [['my_source', 'my_table']], 'depends_on': { @@ -1347,6 +1353,7 @@ def expected_postgres_references_manifest(self, model_database=None): 'quoting': {}, 'vars': {}, 'tags': [], + 'full_refresh': None, }, 'sources': [], 'depends_on': { @@ -1410,6 +1417,7 @@ def expected_postgres_references_manifest(self, model_database=None): 'quoting': {}, 'vars': {}, 'tags': [], + 'full_refresh': None, }, 'database': self.default_database, 'depends_on': { @@ -1494,6 +1502,7 @@ def expected_postgres_references_manifest(self, model_database=None): 'vars': {}, 'tags': [], 'quote_columns': True, + 'full_refresh': None, }, 'sources': [], 'depends_on': {'macros': [], 'nodes': []}, @@ -1725,6 +1734,7 @@ def expected_bigquery_complex_manifest(self): 'quoting': {}, 'vars': {}, 'tags': [], + 'full_refresh': None, }, 'sources': [], 'depends_on': {'macros': [], 'nodes': ['seed.test.seed']}, @@ -1804,6 +1814,7 @@ def expected_bigquery_complex_manifest(self): 'quoting': {}, 'tags': [], 'vars': {}, + 'full_refresh': None, }, 'sources': [], 'depends_on': {'macros': [], 'nodes': ['seed.test.seed']}, @@ -1880,6 +1891,7 @@ def expected_bigquery_complex_manifest(self): 'quoting': {}, 'vars': {}, 'tags': [], + 'full_refresh': None, }, 'sources': [], 'depends_on': { @@ -1959,6 +1971,7 @@ def expected_bigquery_complex_manifest(self): 'quoting': {}, 'vars': {}, 'tags': [], + 'full_refresh': None, }, 'sources': [], 'depends_on': { @@ -2020,6 +2033,7 @@ def expected_bigquery_complex_manifest(self): 'quoting': {}, 'tags': [], 'quote_columns': True, + 'full_refresh': None, }, 'schema': my_schema_name, 'database': self.default_database, @@ -2160,6 +2174,7 @@ def expected_redshift_incremental_view_manifest(self): 'quoting': {}, 'tags': [], 'vars': {}, + 'full_refresh': None, }, 'schema': my_schema_name, 'database': self.default_database, @@ -2241,6 +2256,7 @@ def expected_redshift_incremental_view_manifest(self): 'tags': [], 'vars': {}, 'quote_columns': True, + 'full_refresh': None, }, 'schema': my_schema_name, 'database': self.default_database, @@ -2364,6 +2380,7 @@ def expected_run_results(self, quote_schema=True, quote_model=False, 'column_types': {}, 'quoting': {}, 'tags': [], + 'full_refresh': None, } schema = self.unique_schema() @@ -2523,6 +2540,7 @@ def expected_run_results(self, quote_schema=True, quote_model=False, 'vars': {}, 'tags': [], 'quote_columns': True, + 'full_refresh': None, }, 'sources': [], 'depends_on': {'macros': [], 'nodes': []}, @@ -2575,6 +2593,7 @@ def expected_run_results(self, quote_schema=True, quote_model=False, 'vars': {}, 'tags': [], 'severity': 'ERROR', + 'full_refresh': None, }, 'sources': [], 'depends_on': { @@ -2638,6 +2657,7 @@ def expected_run_results(self, quote_schema=True, quote_model=False, 'vars': {}, 'tags': [], 'severity': 'ERROR', + 'full_refresh': None, }, 'database': self.default_database, 'depends_on': { @@ -2700,6 +2720,7 @@ def expected_run_results(self, quote_schema=True, quote_model=False, 'vars': {}, 'tags': [], 'severity': 'ERROR', + 'full_refresh': None, }, 'database': self.default_database, 'depends_on': { @@ -2804,6 +2825,7 @@ def expected_postgres_references_run_results(self): 'column_types': {}, 'quoting': {}, 'tags': [], + 'full_refresh': None, }, 'sources': [], 'depends_on': { @@ -2884,6 +2906,7 @@ def expected_postgres_references_run_results(self): 'column_types': {}, 'quoting': {}, 'tags': [], + 'full_refresh': None, }, 'sources': [], 'depends_on': { @@ -2981,6 +3004,7 @@ def expected_postgres_references_run_results(self): 'vars': {}, 'tags': [], 'quote_columns': True, + 'full_refresh': None, }, 'sources': [], 'depends_on': {'macros': [], 'nodes': []}, diff --git a/test/integration/047_dbt_ls_test/test_ls.py b/test/integration/047_dbt_ls_test/test_ls.py index 9fbacdc5e41..00683001f59 100644 --- a/test/integration/047_dbt_ls_test/test_ls.py +++ b/test/integration/047_dbt_ls_test/test_ls.py @@ -83,7 +83,8 @@ def expect_snapshot_output(self): 'target_schema': self.unique_schema(), 'unique_key': 'id', 'strategy': 'timestamp', - 'updated_at': 'updated_at' + 'updated_at': 'updated_at', + 'full_refresh': None, }, 'alias': 'my_snapshot', 'resource_type': 'snapshot', @@ -111,6 +112,7 @@ def expect_analyses_output(self): 'vars': {}, 'column_types': {}, 'persist_docs': {}, + 'full_refresh': None, }, 'alias': 'a', 'resource_type': 'analysis', @@ -139,6 +141,7 @@ def expect_model_output(self): 'vars': {}, 'column_types': {}, 'persist_docs': {}, + 'full_refresh': None, }, 'alias': 'ephemeral', 'resource_type': 'model', @@ -158,6 +161,7 @@ def expect_model_output(self): 'vars': {}, 'column_types': {}, 'persist_docs': {}, + 'full_refresh': None, }, 'alias': 'inner', 'resource_type': 'model', @@ -177,6 +181,7 @@ def expect_model_output(self): 'vars': {}, 'column_types': {}, 'persist_docs': {}, + 'full_refresh': None, }, 'alias': 'outer', 'resource_type': 'model', @@ -207,6 +212,7 @@ def expect_model_ephemeral_output(self): 'vars': {}, 'column_types': {}, 'persist_docs': {}, + 'full_refresh': None, }, 'alias': 'outer', 'resource_type': 'model', @@ -256,6 +262,7 @@ def expect_seed_output(self): 'column_types': {}, 'persist_docs': {}, 'quote_columns': False, + 'full_refresh': None, }, 'alias': 'seed', 'resource_type': 'seed', @@ -285,6 +292,7 @@ def expect_test_output(self): 'vars': {}, 'column_types': {}, 'persist_docs': {}, + 'full_refresh': None, }, 'alias': 'not_null_outer_id', 'resource_type': 'test', @@ -306,6 +314,7 @@ def expect_test_output(self): 'vars': {}, 'column_types': {}, 'persist_docs': {}, + 'full_refresh': None, }, 'alias': 't', 'resource_type': 'test', @@ -326,6 +335,7 @@ def expect_test_output(self): 'vars': {}, 'column_types': {}, 'persist_docs': {}, + 'full_refresh': None, }, 'alias': 'unique_outer_id', 'resource_type': 'test', From d0f5664358483a2597030b428b31055ee5ccf939 Mon Sep 17 00:00:00 2001 From: Jacob Beck Date: Tue, 12 May 2020 14:49:01 -0600 Subject: [PATCH 2/2] PR feedback - config is still not a dict --- .../include/global_project/macros/materializations/helpers.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/dbt/include/global_project/macros/materializations/helpers.sql b/core/dbt/include/global_project/macros/materializations/helpers.sql index caf730c4d5f..c83ce4c8d1f 100644 --- a/core/dbt/include/global_project/macros/materializations/helpers.sql +++ b/core/dbt/include/global_project/macros/materializations/helpers.sql @@ -66,7 +66,7 @@ {% macro should_full_refresh() %} - {% set config_full_refresh = config.get('full_refresh', false) %} + {% set config_full_refresh = config.get('full_refresh') %} {% if config_full_refresh is none %} {% set config_full_refresh = flags.FULL_REFRESH %} {% endif %}