diff --git a/core/dbt/clients/jinja.py b/core/dbt/clients/jinja.py index c82180a798f..f83076a1709 100644 --- a/core/dbt/clients/jinja.py +++ b/core/dbt/clients/jinja.py @@ -52,8 +52,13 @@ def _compile(self, source, filename): ) # encode, though I don't think this matters filename = jinja2._compat.encode_filename(filename) - # put ourselves in the cache using the 'lazycache' method - linecache.cache[filename] = (lambda: source,) + # put ourselves in the cache + linecache.cache[filename] = ( + len(source), + None, + [line+'\n' for line in source.splitlines()], + filename + ) return super(MacroFuzzEnvironment, self)._compile(source, filename) diff --git a/core/dbt/compat.py b/core/dbt/compat.py index 9443eb7fcdb..a3fe87d273f 100644 --- a/core/dbt/compat.py +++ b/core/dbt/compat.py @@ -62,6 +62,21 @@ def to_string(s): return str(s) +def to_native_string(s): + if WHICH_PYTHON == 2: + if isinstance(s, unicode): + return str(s) + elif isinstance(s, basestring): + return s + else: + return str(s) + else: + if isinstance(s, basestring): + return s + else: + return str(s) + + def write_file(path, s): if WHICH_PYTHON == 2: with codecs.open(path, 'w', encoding='utf-8') as f: diff --git a/core/dbt/config/renderer.py b/core/dbt/config/renderer.py index 35c66ae880a..ae7d7895106 100644 --- a/core/dbt/config/renderer.py +++ b/core/dbt/config/renderer.py @@ -60,8 +60,9 @@ def render_value(self, value, keypath=None): # if it wasn't read as a string, ignore it if not isinstance(value, compat.basestring): return value - - return get_rendered(value, self.context) + # force the result of rendering into this python version's native + # string type + return compat.to_native_string(get_rendered(value, self.context)) def _render_profile_data(self, value, keypath): result = self.render_value(value) @@ -73,6 +74,14 @@ def _render_profile_data(self, value, keypath): pass return result + def _render_schema_source_data(self, value, keypath): + # things to not render: + # - descriptions + if len(keypath) > 0 and keypath[-1] == 'description': + return value + + return self.render_value(value) + def render_project(self, as_parsed): """Render the parsed data, returning a new dict (or whatever was read). """ @@ -93,3 +102,12 @@ def render_profile_data(self, as_parsed): 'Cycle detected: Profile input has a reference to itself', project=as_parsed ) + + def render_schema_source(self, as_parsed): + try: + return deep_map(self._render_schema_source_data, as_parsed) + except RecursionException: + raise DbtProfileError( + 'Cycle detected: schema.yml input has a reference to itself', + project=as_parsed + ) diff --git a/core/dbt/context/common.py b/core/dbt/context/common.py index 598de08c53b..6b91ed4f310 100644 --- a/core/dbt/context/common.py +++ b/core/dbt/context/common.py @@ -387,6 +387,7 @@ def generate_base(model, model_dict, config, manifest, source_config, "schema": config.credentials.schema, "sql": None, "sql_now": adapter.date_function(), + "source": provider.source(db_wrapper, model, config, manifest), "fromjson": fromjson, "tojson": tojson, "target": target, diff --git a/core/dbt/context/parser.py b/core/dbt/context/parser.py index 9d35546ab6f..2a9d2a87881 100644 --- a/core/dbt/context/parser.py +++ b/core/dbt/context/parser.py @@ -45,6 +45,14 @@ def do_docs(*args): return do_docs +def source(db_wrapper, model, config, manifest): + def do_source(source_name, table_name): + model.sources.append([source_name, table_name]) + return '' + + return do_source + + class Config(object): def __init__(self, model, source_config): self.model = model diff --git a/core/dbt/context/runtime.py b/core/dbt/context/runtime.py index 0f26726c6b8..8780c7e49e1 100644 --- a/core/dbt/context/runtime.py +++ b/core/dbt/context/runtime.py @@ -60,6 +60,30 @@ def do_ref(*args): return do_ref +def source(db_wrapper, model, config, manifest): + current_project = config.project_name + + def do_source(source_name, table_name): + target_source = ParserUtils.resolve_source( + manifest, + source_name, + table_name, + current_project, + model.get('package_name') + ) + + if target_source is None: + dbt.exceptions.source_target_not_found( + model, + source_name, + table_name) + + model.sources.append([source_name, table_name]) + return target_source.sql_table_name + + return do_source + + class Config: def __init__(self, model, source_config=None): self.model = model diff --git a/core/dbt/contracts/graph/compiled.py b/core/dbt/contracts/graph/compiled.py index 240d5dad5d7..7030bdf9e5f 100644 --- a/core/dbt/contracts/graph/compiled.py +++ b/core/dbt/contracts/graph/compiled.py @@ -76,6 +76,7 @@ } ) + COMPILED_NODES_CONTRACT = { 'type': 'object', 'additionalProperties': False, @@ -87,8 +88,10 @@ }, } + COMPILED_MACRO_CONTRACT = PARSED_MACRO_CONTRACT + COMPILED_MACROS_CONTRACT = { 'type': 'object', 'additionalProperties': False, @@ -100,6 +103,7 @@ }, } + COMPILED_GRAPH_CONTRACT = { 'type': 'object', 'additionalProperties': False, diff --git a/core/dbt/contracts/graph/manifest.py b/core/dbt/contracts/graph/manifest.py index c56cf3bf27c..689bbc7d40c 100644 --- a/core/dbt/contracts/graph/manifest.py +++ b/core/dbt/contracts/graph/manifest.py @@ -1,7 +1,8 @@ from dbt.api import APIObject from dbt.contracts.graph.unparsed import UNPARSED_NODE_CONTRACT from dbt.contracts.graph.parsed import PARSED_NODE_CONTRACT, \ - PARSED_MACRO_CONTRACT, PARSED_DOCUMENTATION_CONTRACT, ParsedNode + PARSED_MACRO_CONTRACT, PARSED_DOCUMENTATION_CONTRACT, ParsedNode, \ + PARSED_SOURCE_DEFINITION_CONTRACT from dbt.contracts.graph.compiled import COMPILED_NODE_CONTRACT, CompiledNode from dbt.exceptions import ValidationException from dbt.node_types import NodeType @@ -9,10 +10,15 @@ from dbt import tracking import dbt.utils -# We allow either parsed or compiled nodes, as some 'compile()' calls in the -# runner actually just return the original parsed node they were given. +# We allow either parsed or compiled nodes, or parsed sources, as some +# 'compile()' calls in the runner actually just return the original parsed +# node they were given. COMPILE_RESULT_NODE_CONTRACT = { - 'anyOf': [PARSED_NODE_CONTRACT, COMPILED_NODE_CONTRACT] + 'anyOf': [ + PARSED_NODE_CONTRACT, + COMPILED_NODE_CONTRACT, + PARSED_SOURCE_DEFINITION_CONTRACT, + ] } @@ -268,6 +274,13 @@ def find_refable_by_name(self, name, package): """ return self._find_by_name(name, package, 'nodes', NodeType.refable()) + def find_source_by_name(self, source_name, table_name, package): + """Find any valid target for "source()" in the graph by its name and + package name, or None for any package. + """ + name = '{}.{}'.format(source_name, table_name) + return self._find_by_name(name, package, 'nodes', [NodeType.Source]) + def get_materialization_macro(self, materialization_name, adapter_type=None): macro_name = dbt.utils.get_materialization_macro_name( @@ -293,6 +306,8 @@ def get_materialization_macro(self, materialization_name, def get_resource_fqns(self): resource_fqns = {} for unique_id, node in self.nodes.items(): + if node.resource_type == NodeType.Source: + continue # sources have no FQNs and can't be configured resource_type_plural = node.resource_type + 's' if resource_type_plural not in resource_fqns: resource_fqns[resource_type_plural] = set() @@ -314,6 +329,8 @@ def _filter_subgraph(self, subgraph, predicate): return to_return def _model_matches_schema_and_table(self, schema, table, model): + if model.resource_type == NodeType.Source: + return False return (model.schema.lower() == schema.lower() and model.alias.lower() == table.lower()) @@ -387,11 +404,17 @@ def __getattr__(self, name): type(self).__name__, name) ) + def parsed_nodes(self): + for node in self.nodes.values(): + if node.resource_type == NodeType.Source: + continue + yield node + def get_used_schemas(self): return frozenset({ (node.database, node.schema) - for node in self.nodes.values() + for node in self.parsed_nodes() }) def get_used_databases(self): - return frozenset(node.database for node in self.nodes.values()) + return frozenset(node.database for node in self.parsed_nodes()) diff --git a/core/dbt/contracts/graph/parsed.py b/core/dbt/contracts/graph/parsed.py index 0009a5134be..f8e9d8c195f 100644 --- a/core/dbt/contracts/graph/parsed.py +++ b/core/dbt/contracts/graph/parsed.py @@ -7,7 +7,8 @@ import dbt.clients.jinja from dbt.contracts.graph.unparsed import UNPARSED_NODE_CONTRACT, \ - UNPARSED_MACRO_CONTRACT, UNPARSED_DOCUMENTATION_FILE_CONTRACT + UNPARSED_MACRO_CONTRACT, UNPARSED_DOCUMENTATION_FILE_CONTRACT, \ + UNPARSED_BASE_CONTRACT, FRESHNESS_CONTRACT from dbt.logger import GLOBAL_LOGGER as logger # noqa @@ -137,20 +138,153 @@ } -PARSED_NODE_CONTRACT = deep_merge( - UNPARSED_NODE_CONTRACT, - { - 'properties': { - 'unique_id': { +HAS_FQN_CONTRACT = { + 'properties': { + 'fqn': { + 'type': 'array', + 'items': { 'type': 'string', - 'minLength': 1, + } + }, + }, + 'required': ['fqn'], +} + + +HAS_UNIQUE_ID_CONTRACT = { + 'properties': { + 'unique_id': { + 'type': 'string', + 'minLength': 1, + }, + }, + 'required': ['unique_id'], +} + +CAN_REF_CONTRACT = { + 'properties': { + 'refs': { + 'type': 'array', + 'items': { + 'type': 'array', + 'description': ( + 'The list of arguments passed to a single ref call.' + ), }, - 'fqn': { + 'description': ( + 'The list of call arguments, one list of arguments per ' + 'call.' + ) + }, + 'sources': { + 'type': 'array', + 'items': { 'type': 'array', - 'items': { - 'type': 'string', - } + 'description': ( + 'The list of arguments passed to a single source call.' + ), + }, + 'description': ( + 'The list of call arguments, one list of arguments per ' + 'call.' + ) + }, + 'depends_on': { + 'type': 'object', + 'additionalProperties': False, + 'properties': { + 'nodes': { + 'type': 'array', + 'items': { + 'type': 'string', + 'minLength': 1, + 'description': ( + 'A node unique ID that this depends on.' + ) + } + }, + 'macros': { + 'type': 'array', + 'items': { + 'type': 'string', + 'minLength': 1, + 'description': ( + 'A macro unique ID that this depends on.' + ) + } + }, }, + 'description': ( + 'A list of unique IDs for nodes and macros that this ' + 'node depends upon.' + ), + 'required': ['nodes', 'macros'], + }, + }, + 'required': ['refs', 'sources', 'depends_on'], +} + + +HAS_DOCREFS_CONTRACT = { + 'properties': { + 'docrefs': { + 'type': 'array', + 'items': DOCREF_CONTRACT, + }, + }, +} + + +HAS_DESCRIPTION_CONTRACT = { + 'properties': { + 'description': { + 'type': 'string', + 'description': 'A user-supplied description of the model', + }, + 'columns': { + 'type': 'object', + 'properties': { + '.*': COLUMN_INFO_CONTRACT, + }, + }, + }, + 'required': ['description', 'columns'], +} + +# does this belong inside another contract? +HAS_CONFIG_CONTRACT = { + 'properties': { + 'config': CONFIG_CONTRACT, + }, + 'required': ['config'], +} + + +COLUMN_TEST_CONTRACT = { + 'properties': { + 'column_name': { + 'type': 'string', + 'description': ( + 'In tests parsed from a v2 schema, the column the test is ' + 'associated with (if there is one)' + ) + }, + } +} + + +PARSED_NODE_CONTRACT = deep_merge( + UNPARSED_NODE_CONTRACT, + HAS_UNIQUE_ID_CONTRACT, + HAS_FQN_CONTRACT, + CAN_REF_CONTRACT, + HAS_DOCREFS_CONTRACT, + HAS_DESCRIPTION_CONTRACT, + HAS_CONFIG_CONTRACT, + COLUMN_TEST_CONTRACT, + { + 'properties': { + # these next 3 make sense as a contract 'database': { 'type': 'string', 'description': ( @@ -169,100 +303,32 @@ 'The name of the relation that this will build into' ) }, - 'refs': { - 'type': 'array', - 'items': { - 'type': 'array', - 'description': ( - 'The list of arguments passed to a single ref call.' - ), - }, - 'description': ( - 'The list of call arguments, one list of arguments per ' - 'call.' - ) - }, - 'depends_on': { - 'type': 'object', - 'additionalProperties': False, - 'properties': { - 'nodes': { - 'type': 'array', - 'items': { - 'type': 'string', - 'minLength': 1, - 'description': ( - 'A node unique ID that this depends on.' - ) - } - }, - 'macros': { - 'type': 'array', - 'items': { - 'type': 'string', - 'minLength': 1, - 'description': ( - 'A macro unique ID that this depends on.' - ) - } - }, - }, - 'description': ( - 'A list of unique IDs for nodes and macros that this ' - 'node depends upon.' - ), - 'required': ['nodes', 'macros'], - }, # TODO: move this into a class property. 'empty': { 'type': 'boolean', 'description': 'True if the SQL is empty', }, - 'config': CONFIG_CONTRACT, 'tags': { 'type': 'array', 'items': { 'type': 'string', } }, - 'description': { - 'type': 'string', - 'description': 'A user-supplied description of the model', - }, - 'columns': { - 'type': 'object', - 'properties': { - '.*': COLUMN_INFO_CONTRACT, - } - }, + # this is really nodes-only 'patch_path': { 'type': 'string', 'description': ( 'The path to the patch source if the node was patched' ), }, - 'docrefs': { - 'type': 'array', - 'items': DOCREF_CONTRACT, - }, 'build_path': { 'type': 'string', 'description': ( 'In seeds, the path to the source file used during build.' ), }, - 'column_name': { - 'type': 'string', - 'description': ( - 'In tests parsed from a v2 schema, the column the test is ' - 'associated with (if there is one)' - ) - }, }, - 'required': UNPARSED_NODE_CONTRACT['required'] + [ - 'unique_id', 'fqn', 'database', 'schema', 'refs', 'depends_on', - 'empty', 'config', 'tags', 'alias', 'columns', 'description' - ] + 'required': ['database', 'schema', 'empty', 'tags', 'alias'], } ) @@ -392,8 +458,9 @@ def config(self, value): 'items': DOCREF_CONTRACT, } }, - 'required': ['name', 'original_file_path', 'description', 'columns', - 'docrefs'], + 'required': [ + 'name', 'original_file_path', 'description', 'columns', 'docrefs' + ], } @@ -453,7 +520,7 @@ class ParsedNodePatch(APIObject): 'required': ['macros'], }, }, - 'required': UNPARSED_MACRO_CONTRACT['required'] + [ + 'required': [ 'resource_type', 'unique_id', 'tags', 'depends_on', 'name', ] } @@ -472,7 +539,6 @@ def generator(self): # available in this class. should we just generate this here? return dbt.clients.jinja.macro_generator(self._contents) - # This is just the file + its ID PARSED_DOCUMENTATION_CONTRACT = deep_merge( UNPARSED_DOCUMENTATION_FILE_CONTRACT, @@ -497,9 +563,7 @@ def generator(self): 'description': 'The contents of just the docs block', }, }, - 'required': UNPARSED_DOCUMENTATION_FILE_CONTRACT['required'] + [ - 'name', 'unique_id', 'block_contents', - ], + 'required': ['name', 'unique_id', 'block_contents'], } ) @@ -526,3 +590,77 @@ class ParsedDocumentation(APIObject): class Hook(APIObject): SCHEMA = HOOK_CONTRACT + + +PARSED_SOURCE_DEFINITION_CONTRACT = deep_merge( + UNPARSED_BASE_CONTRACT, + FRESHNESS_CONTRACT, + HAS_DESCRIPTION_CONTRACT, + HAS_UNIQUE_ID_CONTRACT, + HAS_DOCREFS_CONTRACT, + { + 'description': ( + 'A source table definition, as parsed from the one provided in the' + '"tables" subsection of the "sources" section of schema.yml' + ), + 'properties': { + 'name': { + 'type': 'string', + 'description': ( + 'The name of this node, which is the name of the model it' + 'refers to' + ), + 'minLength': 1, + }, + 'source_name': { + 'type': 'string', + 'description': 'The reference name of the source definition', + 'minLength': 1, + }, + 'source_description': { + 'type': 'string', + 'description': 'The user-supplied description of the source', + }, + 'loader': { + 'type': 'string', + 'description': 'The user-defined loader for this source', + }, + 'sql_table_name': { + 'type': 'string', + 'description': 'The exact identifier for the source table', + 'minLength': 1, + }, + # the manifest search stuff really requires this, sadly + 'resource_type': { + 'enum': [NodeType.Source], + } + }, + # note that while required, loaded_at_field and freshness may be null + # (and either of freshness's members may be null as well!) + 'required': [ + 'source_name', 'source_description', 'loaded_at_field', 'loader', + 'freshness', 'description', 'columns', 'docrefs', 'sql_table_name', + ], + } +) + + +class ParsedSourceDefinition(APIObject): + SCHEMA = PARSED_SOURCE_DEFINITION_CONTRACT + + def to_shallow_dict(self): + return self._contents.copy() + + # provide some emtpy/meaningless properties so these look more like + # ParsedNodes + @property + def depends_on_nodes(self): + return [] + + @property + def refs(self): + return [] + + @property + def sources(self): + return [] diff --git a/core/dbt/contracts/graph/unparsed.py b/core/dbt/contracts/graph/unparsed.py index 788f3359af9..50a98945619 100644 --- a/core/dbt/contracts/graph/unparsed.py +++ b/core/dbt/contracts/graph/unparsed.py @@ -27,8 +27,14 @@ 'type': 'string', 'description': ( 'Relative path to the originating file from the project root.' - ), - }, + ), + } + }, + 'required': ['package_name', 'root_path', 'path', 'original_file_path'] +} + +UNPARSED_HAS_SQL_CONTRACT = { + 'properties': { 'raw_sql': { 'type': 'string', 'description': ( @@ -40,14 +46,17 @@ 'type': 'integer', } }, - 'required': ['package_name', 'root_path', 'path', 'original_file_path', - 'raw_sql'] + 'required': ['raw_sql'] } -UNPARSED_MACRO_CONTRACT = UNPARSED_BASE_CONTRACT +UNPARSED_MACRO_CONTRACT = deep_merge( + UNPARSED_BASE_CONTRACT, + UNPARSED_HAS_SQL_CONTRACT +) UNPARSED_NODE_CONTRACT = deep_merge( UNPARSED_BASE_CONTRACT, + UNPARSED_HAS_SQL_CONTRACT, { 'properties': { 'name': { @@ -69,8 +78,7 @@ ] }, }, - 'required': UNPARSED_BASE_CONTRACT['required'] + [ - 'resource_type', 'name'] + 'required': ['resource_type', 'name'] } ) @@ -112,13 +120,17 @@ class UnparsedNode(APIObject): } -UNPARSED_NODE_UPDATE_CONTRACT = { - 'type': 'object', - 'additionalProperties': False, - 'description': ( - 'A collection of the unparsed node updates, as provided in the ' - '"models" section of schema.yml' - ), +UNPARSED_COLUMN_DESCRIPTION_CONTRACT = { + 'properties': { + 'columns': { + 'type': 'array', + 'items': COLUMN_TEST_CONTRACT, + }, + }, +} + + +UNPARSED_NODE_DESCRIPTION_CONTRACT = { 'properties': { 'name': { 'type': 'string', @@ -134,10 +146,6 @@ class UnparsedNode(APIObject): 'The raw string description of the node after parsing the yaml' ), }, - 'columns': { - 'type': 'array', - 'items': COLUMN_TEST_CONTRACT, - }, 'tests': { 'type': 'array', 'items': { @@ -152,13 +160,149 @@ class UnparsedNode(APIObject): } +UNPARSED_NODE_UPDATE_CONTRACT = deep_merge( + UNPARSED_NODE_DESCRIPTION_CONTRACT, + UNPARSED_COLUMN_DESCRIPTION_CONTRACT, + { + 'type': 'object', + 'additionalProperties': False, + 'description': ( + 'A collection of the unparsed node updates, as provided in the ' + '"models" section of schema.yml' + ), + } +) + + class UnparsedNodeUpdate(APIObject): """An unparsed node update is the blueprint for tests to be added and nodes - to be updated, referencing a certain node (specifically, a Model). + to be updated, referencing a certain node (specifically, a Model or + Source). """ SCHEMA = UNPARSED_NODE_UPDATE_CONTRACT +_TIME_CONTRACT = { + 'type': 'object', + 'additionalProperties': False, + 'properties': { + 'count': { + 'type': 'integer', + }, + 'period': { + 'enum': ['minute', 'hour', 'day'], + }, + }, + 'required': ['count', 'period'], +} + + +_FRESHNESS_CONTRACT = { + 'type': 'object', + 'additionalProperties': False, + 'properties': { + 'warn_after': { + 'anyOf': [ + {'type': 'null'}, + _TIME_CONTRACT, + ] + }, + 'error_after': { + 'anyOf': [ + {'type': 'null'}, + _TIME_CONTRACT, + + ] + }, + }, +} + + +FRESHNESS_CONTRACT = { + 'properties': { + 'loaded_at_field': { + 'type': ['null', 'string'], + 'description': 'The field to use as the "loaded at" timestamp', + }, + 'freshness': { + 'anyOf': [ + {'type': 'null'}, + _FRESHNESS_CONTRACT, + ], + }, + }, +} + + +UNPARSED_SOURCE_TABLE_DEFINITION_CONTRACT = deep_merge( + UNPARSED_NODE_DESCRIPTION_CONTRACT, + UNPARSED_COLUMN_DESCRIPTION_CONTRACT, + FRESHNESS_CONTRACT, + { + 'description': ( + 'A source table definition, as provided in the "tables" ' + 'subsection of the "sources" section of schema.yml' + ), + 'properties': { + 'sql_table_name': { + 'type': 'string', + 'description': 'The exact identifier for the source table', + 'minLength': 1, + }, + }, + 'required': ['sql_table_name'], + } +) + + +UNPARSED_SOURCE_DEFINITION_CONTRACT = deep_merge( + FRESHNESS_CONTRACT, + { + 'type': 'object', + 'additionalProperties': False, + 'description': ( + 'A collection of the unparsed sources, as provided in the ' + '"sources" section of schema.yml' + ), + 'properties': { + 'name': { + 'type': 'string', + 'description': 'The reference name of the source definition', + 'minLength': 1, + }, + 'loader': { + 'type': 'string', + 'description': 'The user-defined loader for this source', + 'minLength': 1, + }, + 'description': { + 'type': 'string', + 'description': 'The user-supplied description of the source', + }, + 'tables': { + 'type': 'array', + 'items': UNPARSED_SOURCE_TABLE_DEFINITION_CONTRACT, + 'description': 'The tables for this source', + 'minLength': 1, + }, + }, + 'required': ['name'], + } +) + + +class UnparsedTableDefinition(APIObject): + SCHEMA = UNPARSED_SOURCE_TABLE_DEFINITION_CONTRACT + + +class UnparsedSourceDefinition(APIObject): + SCHEMA = UNPARSED_SOURCE_DEFINITION_CONTRACT + + @property + def tables(self): + return [UnparsedTableDefinition(**t) for t in self.get('tables', [])] + + UNPARSED_DOCUMENTATION_FILE_CONTRACT = { 'type': 'object', 'additionalProperties': False, diff --git a/core/dbt/exceptions.py b/core/dbt/exceptions.py index 0a39044fcc3..497c52d70a8 100644 --- a/core/dbt/exceptions.py +++ b/core/dbt/exceptions.py @@ -333,6 +333,20 @@ def ref_target_not_found(model, target_model_name, target_model_package): raise_compiler_error(msg, model) +def source_disabled_message(model, target_name, target_table_name): + return ("{} '{}' ({}) depends on source '{}.{}' which was not found" + .format(model.get('resource_type').title(), + model.get('unique_id'), + model.get('original_file_path'), + target_name, + target_table_name)) + + +def source_target_not_found(model, target_name, target_table_name): + msg = source_disabled_message(model, target_name, target_table_name) + raise_compiler_error(msg, model) + + def ref_disabled_dependency(model, target_model): raise_compiler_error( "Model '{}' depends on model '{}' which is disabled in " @@ -546,15 +560,12 @@ def raise_duplicate_patch_name(name, patch_1, patch_2): ) -def raise_incorrect_version(path): +def raise_invalid_schema_yml_version(path, issue): raise_compiler_error( - 'The schema file at {} does not contain a valid version specifier. ' - 'dbt assumes that schema.yml files without version specifiers are ' - 'version 1 schemas, but this file looks like a version 2 schema. If ' - 'this is the case, you can fix this error by adding `version: 2` to ' - 'the top of the file.\n\nOtherwise, please consult the documentation ' - 'for more information on schema.yml syntax:\n\n' - 'https://docs.getdbt.com/v0.11/docs/schemayml-files'.format(path) + 'The schema file at {} is invalid because {}. Please consult the ' + 'documentation for more information on schema.yml syntax:\n\n' + 'https://docs.getdbt.com/docs/schemayml-filesf' + .format(path, issue) ) diff --git a/core/dbt/graph/selector.py b/core/dbt/graph/selector.py index 19c699eaea0..3093b572992 100644 --- a/core/dbt/graph/selector.py +++ b/core/dbt/graph/selector.py @@ -143,14 +143,20 @@ def get_nodes_by_qualified_name(self, graph, qualified_name_selector): qualified_name = qualified_name_selector.split(".") package_names = get_package_names(graph) for node in graph.nodes(): - fqn_ish = self.manifest.nodes[node].fqn + real_node = self.manifest.nodes[node] + if real_node.resource_type == NodeType.Source: + continue + fqn_ish = real_node.fqn if _node_is_match(qualified_name, package_names, fqn_ish): yield node def get_nodes_by_tag(self, graph, tag_name): """ yields nodes from graph that have the specified tag """ for node in graph.nodes(): - tags = self.manifest.nodes[node].tags + real_node = self.manifest.nodes[node] + if real_node.resource_type == NodeType.Source: + continue + tags = real_node.tags if tag_name in tags: yield node diff --git a/core/dbt/loader.py b/core/dbt/loader.py index 5f696ff14e4..d8a6d58603e 100644 --- a/core/dbt/loader.py +++ b/core/dbt/loader.py @@ -103,7 +103,7 @@ def _load_schema_tests(self): parser = SchemaParser(self.root_project, self.all_projects, self.macro_manifest) for project_name, project in self.all_projects.items(): - tests, patches = parser.load_and_parse( + tests, patches, sources = parser.load_and_parse( package_name=project_name, root_dir=project.project_root, relative_dirs=project.source_paths @@ -116,6 +116,13 @@ def _load_schema_tests(self): ) self.tests[unique_id] = test + for unique_id, source in sources.items(): + if unique_id in self.nodes: + dbt.exceptions.raise_duplicate_resource_name( + source, self.nodes[unique_id], + ) + self.nodes[unique_id] = source + for name, patch in patches.items(): if name in self.patches: dbt.exceptions.raise_duplicate_patch_name( @@ -143,6 +150,7 @@ def create_manifest(self): ) manifest.add_nodes(self.tests) manifest.patch_nodes(self.patches) + manifest = ParserUtils.process_sources(manifest, self.root_project) manifest = ParserUtils.process_refs(manifest, self.root_project.project_name) manifest = ParserUtils.process_docs(manifest, self.root_project) diff --git a/core/dbt/node_runners.py b/core/dbt/node_runners.py index fa99dfbd0d3..66ffe9ede96 100644 --- a/core/dbt/node_runners.py +++ b/core/dbt/node_runners.py @@ -116,6 +116,7 @@ def safe_run(self, manifest): error=str(e).strip()) logger.error(error) + logger.debug('', exc_info=True) result.error = dbt.compat.to_string(e) result.status = 'ERROR' diff --git a/core/dbt/node_types.py b/core/dbt/node_types.py index c1dfc8f8cb4..da941ad05db 100644 --- a/core/dbt/node_types.py +++ b/core/dbt/node_types.py @@ -9,6 +9,7 @@ class NodeType(object): Operation = 'operation' Seed = 'seed' Documentation = 'documentation' + Source = 'source' @classmethod def executable(cls): @@ -20,6 +21,7 @@ def executable(cls): cls.Operation, cls.Seed, cls.Documentation, + cls.Source, ] @classmethod diff --git a/core/dbt/parser/base.py b/core/dbt/parser/base.py index 4cdee6ee8c0..d21b48cb1ea 100644 --- a/core/dbt/parser/base.py +++ b/core/dbt/parser/base.py @@ -69,7 +69,6 @@ def get_schema_func(self): - if neither of those exist (unit tests?), a function that returns the 'default schema' as set in the root project's 'credentials' is used - """ if self._get_schema_func is not None: return self._get_schema_func @@ -96,49 +95,17 @@ def get_schema(_): self._get_schema_func = get_schema return self._get_schema_func - def parse_node(self, node, node_path, package_project_config, tags=None, - fqn_extra=None, fqn=None, agate_table=None, - archive_config=None, column_name=None): - """Parse a node, given an UnparsedNode and any other required information. - - agate_table should be set if the node came from a seed file. - archive_config should be set if the node is an Archive node. - column_name should be set if the node is a Test node associated with a - particular column. + def _build_intermediate_node_dict(self, config, node_dict, node_path, + package_project_config, tags, fqn, + agate_table, archive_config, + column_name): + """Update the unparsed node dictionary and build the basis for an + intermediate ParsedNode that will be passed into the renderer """ - logger.debug("Parsing {}".format(node_path)) - - node = node.serialize() - + # because this takes and returns dicts, subclasses can safely override + # this and mutate its results using super() both before and after. if agate_table is not None: - node['agate_table'] = agate_table - tags = coalesce(tags, []) - fqn_extra = coalesce(fqn_extra, []) - - node.update({ - 'refs': [], - 'depends_on': { - 'nodes': [], - 'macros': [], - } - }) - - if fqn is None: - fqn = self.get_fqn(node.get('path'), package_project_config, - fqn_extra) - - config = SourceConfig( - self.root_project_config, - package_project_config, - fqn, - node['resource_type']) - - node['unique_id'] = node_path - node['empty'] = ( - 'raw_sql' in node and len(node['raw_sql'].strip()) == 0 - ) - node['fqn'] = fqn - node['tags'] = tags + node_dict['agate_table'] = agate_table # Set this temporarily. Not the full config yet (as config() hasn't # been called from jinja yet). But the Var() call below needs info @@ -146,19 +113,42 @@ def parse_node(self, node, node_path, package_project_config, tags=None, # TODO: Restructure this? config_dict = coalesce(archive_config, {}) config_dict.update(config.config) - node['config'] = config_dict - # Set this temporarily so get_rendered() has access to a schema & alias - node['schema'] = self.default_schema - node['database'] = self.default_database - default_alias = node.get('name') - node['alias'] = default_alias + empty = ( + 'raw_sql' in node_dict and len(node_dict['raw_sql'].strip()) == 0 + ) + + node_dict.update({ + 'refs': [], + 'sources': [], + 'depends_on': { + 'nodes': [], + 'macros': [], + }, + 'unique_id': node_path, + 'empty': empty, + 'fqn': fqn, + 'tags': tags, + 'config': config_dict, + # Set these temporarily so get_rendered() has access to a schema, + # database, and alias. + 'schema': self.default_schema, + 'database': self.default_database, + 'alias': node_dict.get('name'), + }) # if there's a column, it should end up part of the ParsedNode if column_name is not None: - node['column_name'] = column_name + node_dict['column_name'] = column_name + + return node_dict - parsed_node = ParsedNode(**node) + def _render_with_context(self, parsed_node, config): + """Given the parsed node and a SourceConfig to use during parsing, + render the node's sql wtih macro capture enabled. + + Note: this mutates the config object when config() calls are rendered. + """ context = dbt.context.parser.generate( parsed_node, self.root_project_config, @@ -171,17 +161,19 @@ def parse_node(self, node, node_path, package_project_config, tags=None, # Clean up any open conns opened by adapter functions that hit the db db_wrapper = context['adapter'] - adapter = db_wrapper.adapter - runtime_config = db_wrapper.config - adapter.release_connection(parsed_node.name) + db_wrapper.adapter.release_connection(parsed_node.name) + def _update_parsed_node_info(self, parsed_node, config): + """Given the SourceConfig used for parsing and the parsed node, + generate and set the true values to use, overriding the temporary parse + values set in _build_intermediate_parsed_node. + """ # Special macro defined in the global project. Use the root project's # definition, not the current package schema_override = config.config.get('schema') get_schema = self.get_schema_func() parsed_node.schema = get_schema(schema_override).strip() - parsed_node.alias = config.config.get('alias', default_alias) - # no fancy macro for database (for now?) + parsed_node.alias = config.config.get('alias', parsed_node.get('name')) parsed_node.database = config.config.get( 'database', self.default_database ).strip() @@ -199,6 +191,39 @@ def parse_node(self, node, node_path, package_project_config, tags=None, parsed_node.config[hook_type] = dbt.hooks.get_hooks(parsed_node, hook_type) + def parse_node(self, node, node_path, package_project_config, tags=None, + fqn_extra=None, fqn=None, agate_table=None, + archive_config=None, column_name=None): + """Parse a node, given an UnparsedNode and any other required information. + + agate_table should be set if the node came from a seed file. + archive_config should be set if the node is an Archive node. + column_name should be set if the node is a Test node associated with a + particular column. + """ + logger.debug("Parsing {}".format(node_path)) + + tags = coalesce(tags, []) + fqn_extra = coalesce(fqn_extra, []) + + if fqn is None: + fqn = self.get_fqn(node.path, package_project_config, fqn_extra) + + config = SourceConfig( + self.root_project_config, + package_project_config, + fqn, + node.resource_type) + + parsed_dict = self._build_intermediate_node_dict( + config, node.serialize(), node_path, config, tags, fqn, + agate_table, archive_config, column_name + ) + parsed_node = ParsedNode(**parsed_dict) + + self._render_with_context(parsed_node, config) + self._update_parsed_node_info(parsed_node, config) + parsed_node.validate() return parsed_node diff --git a/core/dbt/parser/schemas.py b/core/dbt/parser/schemas.py index dc2fc566f24..4ad5897617b 100644 --- a/core/dbt/parser/schemas.py +++ b/core/dbt/parser/schemas.py @@ -1,3 +1,5 @@ +from __future__ import unicode_literals +import itertools import os import re import hashlib @@ -10,13 +12,16 @@ import dbt.context.parser import dbt.contracts.project +from dbt.clients.jinja import get_rendered from dbt.node_types import NodeType -from dbt.compat import basestring, to_string +from dbt.compat import basestring, to_string, to_native_string from dbt.logger import GLOBAL_LOGGER as logger from dbt.utils import get_pseudo_test_path -from dbt.contracts.graph.unparsed import UnparsedNode, UnparsedNodeUpdate -from dbt.contracts.graph.parsed import ParsedNodePatch +from dbt.contracts.graph.unparsed import UnparsedNode, UnparsedNodeUpdate, \ + UnparsedSourceDefinition +from dbt.contracts.graph.parsed import ParsedNodePatch, ParsedSourceDefinition from dbt.parser.base import MacrosKnownParser +from dbt.config.renderer import ConfigRenderer def get_nice_schema_test_name(test_type, test_name, args): @@ -50,7 +55,8 @@ def get_nice_schema_test_name(test_type, test_name, args): def as_kwarg(key, value): test_value = to_string(value) - is_function = re.match(r'^\s*(ref|var)\s*\(.+\)\s*$', test_value) + is_function = re.match(r'^\s*(env_var|ref|var|source|doc)\s*\(.+\)\s*$', + test_value) # if the value is a function, don't wrap it in quotes! if is_function: @@ -61,11 +67,11 @@ def as_kwarg(key, value): return "{key}={value}".format(key=key, value=formatted_value) -def build_test_raw_sql(test_namespace, model_name, test_type, test_args): +def build_test_raw_sql(test_namespace, model, test_type, test_args): """Build the raw SQL from a test definition. :param test_namespace: The test's namespace, if one exists - :param model_name: The model name under test + :param model: The model under test :param test_type: The type of the test (unique_id, etc) :param test_args: The arguments passed to the test as a list of `key=value` strings @@ -81,7 +87,7 @@ def build_test_raw_sql(test_namespace, model_name, test_type, test_args): raw_sql = "{{{{ {macro}(model=ref('{model}'), {kwargs}) }}}}".format( **{ - 'model': model_name, + 'model': model['name'], 'macro': macro_name, 'kwargs': ", ".join(kwargs) } @@ -89,110 +95,179 @@ def build_test_raw_sql(test_namespace, model_name, test_type, test_args): return raw_sql -class SchemaParser(MacrosKnownParser): - """This is the original schema parser but with everything in one huge CF of - a method so I can refactor it more nicely. +def build_source_test_raw_sql(test_namespace, source, table, test_type, + test_args): + """Build the raw SQL from a source test definition. + + :param test_namespace: The test's namespace, if one exists + :param source: The source under test. + :param table: The table under test + :param test_type: The type of the test (unique_id, etc) + :param test_args: The arguments passed to the test as a list of `key=value` + strings + :return: A string of raw sql for the test node. """ - @staticmethod - def check_v2_missing_version(path, test_yml): - """Given the loaded yaml from a file, return True if it looks like the - file is probably a v2 schema.yml with a missing `version: 2`. - """ - # in v1, it's Dict[str, dict] instead of Dict[str, list] - if 'models' in test_yml and isinstance(test_yml['models'], list): - dbt.exceptions.raise_incorrect_version(path) + # sort the dict so the keys are rendered deterministically (for tests) + kwargs = [as_kwarg(key, test_args[key]) for key in sorted(test_args)] - @classmethod - def _build_v1_test_args(cls, config): - if isinstance(config, (basestring, int, float, bool)): - return {'arg': config} - else: - return config + if test_namespace is None: + macro_name = "test_{}".format(test_type) + else: + macro_name = "{}.test_{}".format(test_namespace, test_type) - @classmethod - def _build_v2_test_args(cls, test, name): - if isinstance(test, basestring): - test_name = test - test_args = {} - elif isinstance(test, dict): - test = list(test.items()) - if len(test) != 1: - dbt.exceptions.raise_compiler_error( - 'test definition dictionary must have exactly one key, got' - ' {} instead ({} keys)'.format(test, len(test)) - ) - test_name, test_args = test[0] - else: + raw_sql = ( + "{{{{ {macro}(model=source('{source}', '{table}'), {kwargs}) }}}}" + .format( + source=source['name'], + table=table['name'], + macro=macro_name, + kwargs=", ".join(kwargs)) + ) + return raw_sql + + +def calculate_test_namespace(test_type, package_name): + test_namespace = None + split = test_type.split('.') + if len(split) > 1: + test_type = split[1] + package_name = split[0] + test_namespace = package_name + + return test_namespace, test_type, package_name + + +def _build_test_args(test, name): + if isinstance(test, basestring): + test_name = test + test_args = {} + elif isinstance(test, dict): + test = list(test.items()) + if len(test) != 1: dbt.exceptions.raise_compiler_error( - 'test must be dict or str, got {} (value {})'.format( - type(test), test - ) + 'test definition dictionary must have exactly one key, got' + ' {} instead ({} keys)'.format(test, len(test)) ) - if name is not None: - test_args['column_name'] = name - return test_name, test_args + test_name, test_args = test[0] + else: + dbt.exceptions.raise_compiler_error( + 'test must be dict or str, got {} (value {})'.format( + type(test), test + ) + ) + if name is not None: + test_args['column_name'] = name + return test_name, test_args - @classmethod - def calculate_namespace(cls, test_type, package_name): - test_namespace = None - split = test_type.split('.') - if len(split) > 1: - test_type = split[1] - package_name = split[0] - test_namespace = package_name - return test_namespace, test_type, package_name +def warn_invalid(filepath, key, value, explain): + msg = ( + "Invalid test config given in {} @ {}: {} {}" + ).format(filepath, key, value, explain) + if dbt.flags.STRICT_MODE: + dbt.exceptions.raise_compiler_error(msg, value) + dbt.utils.compiler_warning(value, msg) - @classmethod - def build_unparsed_node(cls, model_name, package_name, test_type, - test_args, test_namespace, root_dir, - original_file_path): - """Given a model name (for the model under test), a pacakge name, - a test type (identifying the test macro to use), arguments dictionary, - the root directory of the search, and the original file path to the - schema.yml file that specified the test, build an UnparsedNode - representing the test. + +def _filter_validate(filepath, location, values, validate): + """Generator for validate() results called against all given values. On + errors, fields are warned about and ignored, unless strict mode is set in + which case a compiler error is raised. + """ + for value in values: + if not isinstance(value, dict): + warn_invalid(filepath, location, value, '(expected a dict)') + continue + try: + yield validate(**value) + except dbt.exceptions.JSONValidationException as exc: + # we don't want to fail the full run, but we do want to fail + # parsing this file + warn_invalid(filepath, location, value, '- '+exc.msg) + continue + + +class ParserRef(object): + """A helper object to hold parse-time references.""" + def __init__(self): + self.column_info = {} + self.docrefs = [] + + def add(self, column_name, description): + self.column_info[column_name] = { + 'name': column_name, + 'description': description, + } + + +class SchemaBaseTestParser(MacrosKnownParser): + def _parse_column(self, target, column, package_name, root_dir, path, + refs): + # this should yield ParsedNodes where resource_type == NodeType.Test + column_name = column['name'] + description = column.get('description', '') + + refs.add(column_name, description) + context = { + 'doc': dbt.context.parser.docs(target, refs.docrefs, column_name) + } + get_rendered(description, context) + + for test in column.get('tests', []): + yield self.build_test_node( + target, package_name, test, root_dir, + path, column_name + ) + + def _build_raw_sql(self, test_namespace, target, test_type, test_args): + raise NotImplementedError + + def _generate_test_name(self, target, test_type, test_args): + """Returns a hashed_name, full_name pair.""" + raise NotImplementedError + + def build_test_node(self, test_target, package_name, test, root_dir, path, + column_name=None): + """Build a test node against the given target (a model or a source). + + :param test_target: An unparsed form of the target. """ - test_path = os.path.basename(original_file_path) + test_type, test_args = _build_test_args(test, column_name) - raw_sql = build_test_raw_sql(test_namespace, model_name, test_type, - test_args) + test_namespace, test_type, package_name = calculate_test_namespace( + test_type, package_name + ) - hashed_name, full_name = get_nice_schema_test_name(test_type, - model_name, - test_args) + source_package = self.all_projects.get(package_name) + if source_package is None: + desc = '"{}" test on model "{}"'.format(test_type, + model_name) + dbt.exceptions.raise_dep_not_found(None, desc, test_namespace) + + test_path = os.path.basename(path) + + hashed_name, full_name = self._generate_test_name(test_target, + test_type, + test_args) hashed_path = get_pseudo_test_path(hashed_name, test_path, 'schema_test') - full_path = get_pseudo_test_path(full_name, test_path, - 'schema_test') - return UnparsedNode( + + full_path = get_pseudo_test_path(full_name, test_path, 'schema_test') + raw_sql = self._build_raw_sql(test_namespace, test_target, test_type, + test_args) + unparsed = UnparsedNode( name=full_name, resource_type=NodeType.Test, package_name=package_name, root_path=root_dir, path=hashed_path, - original_file_path=original_file_path, + original_file_path=path, raw_sql=raw_sql ) - def build_parsed_node(self, unparsed, model_name, test_namespace, - test_type, column_name): - """Given an UnparsedNode with a node type of Test and some extra - information, build a ParsedNode representing the test. - """ - - test_path = os.path.basename(unparsed.original_file_path) - - source_package = self.all_projects.get(unparsed.package_name) - if source_package is None: - desc = '"{}" test on model "{}"'.format(test_type, - model_name) - dbt.exceptions.raise_dep_not_found(None, desc, test_namespace) - # supply our own fqn which overrides the hashed version from the path - full_path = get_pseudo_test_path(unparsed.name, test_path, - 'schema_test') + # TODO: is this necessary even a little bit for tests? fqn_override = self.get_fqn(full_path, source_package) node_path = self.get_path(NodeType.Test, unparsed.package_name, @@ -206,28 +281,176 @@ def build_parsed_node(self, unparsed, model_name, test_namespace, fqn=fqn_override, column_name=column_name) - def build_node(self, model_name, package_name, test_type, test_args, - root_dir, original_file_path, column_name=None): - """From the various components that are common to both v1 and v2 schema, - build a ParsedNode representing a test case. + +class SchemaModelParser(SchemaBaseTestParser): + def _build_raw_sql(self, test_namespace, target, test_type, test_args): + return build_test_raw_sql(test_namespace, target, test_type, test_args) + + def _generate_test_name(self, target, test_type, test_args): + return get_nice_schema_test_name(test_type, target['name'], test_args) + + def parse_models_entry(self, model_dict, path, package_name, root_dir): + model_name = model_dict['name'] + refs = ParserRef() + for column in model_dict.get('columns', []): + column_tests = self._parse_column(model_dict, column, package_name, + root_dir, path, refs) + for node in column_tests: + yield 'test', node + + for test in model_dict.get('tests', []): + node = self.build_test_node(model_dict, package_name, test, + root_dir, path) + yield 'test', node + + context = {'doc': dbt.context.parser.docs(model_dict, refs.docrefs)} + description = model_dict.get('description', '') + get_rendered(description, context) + + patch = ParsedNodePatch( + name=model_name, + original_file_path=path, + description=description, + columns=refs.column_info, + docrefs=refs.docrefs + ) + yield 'patch', patch + + def parse_all(self, models, path, package_name, root_dir): + """Parse all the model dictionaries in models. + + :param List[dict] models: The `models` section of the schema.yml, as a + list of dicts. + :param str path: The path to the schema.yml file + :param str package_name: The name of the current package + :param str root_dir: The root directory of the search """ - original_test_type = test_type - test_namespace, test_type, package_name = self.calculate_namespace( - test_type, package_name + filtered = _filter_validate(path, 'models', models, UnparsedNodeUpdate) + nodes = itertools.chain.from_iterable( + self.parse_models_entry(model, path, package_name, root_dir) + for model in filtered ) + for node_type, node in nodes: + yield node_type, node - test_namespace, test_type, package_name = self.calculate_namespace( - test_type, package_name + +class SchemaSourceParser(SchemaBaseTestParser): + def __init__(self, root_project_config, all_projects, macro_manifest): + super(SchemaSourceParser, self).__init__( + root_project_config=root_project_config, + all_projects=all_projects, + macro_manifest=macro_manifest + ) + self._renderer = ConfigRenderer(self.root_project_config.cli_vars) + + def _build_raw_sql(self, test_namespace, target, test_type, test_args): + return build_source_test_raw_sql(test_namespace, target['source'], + target['table'], test_type, + test_args) + + def _generate_test_name(self, target, test_type, test_args): + return get_nice_schema_test_name( + 'source_'+test_type, + '{}_{}'.format(target['source']['name'], target['table']['name']), + test_args ) - unparsed = self.build_unparsed_node(model_name, package_name, - test_type, test_args, - test_namespace, root_dir, - original_file_path) + def get_path(self, *parts): + return '.'.join(str(s) for s in parts) + + def generate_source_node(self, source, table, path, package_name, root_dir, + refs): + unique_id = self.get_path(NodeType.Source, package_name, + source.name, table.name) + + context = {'doc': dbt.context.parser.docs(source, refs.docrefs)} + description = table.get('description', '') + source_description = source.get('description', '') + get_rendered(description, context) + get_rendered(source_description, context) + + # we'll fill columns in later. + freshness = dbt.utils.deep_merge(source.get('freshness', {}), + table.get('freshness', {})) + + loaded_at_field = table.get('loaded_at_field', + source.get('loaded_at_field')) + return ParsedSourceDefinition( + package_name=package_name, + root_path=root_dir, + path=path, + original_file_path=path, + columns=refs.column_info, + unique_id=unique_id, + name=table.name, + description=description, + source_name=source.name, + source_description=source_description, + loader=source.get('loader', ''), + sql_table_name=table.sql_table_name, + docrefs=refs.docrefs, + loaded_at_field=loaded_at_field, + freshness=freshness, + resource_type=NodeType.Source + ) + + def parse_source_table(self, source, table, path, package_name, root_dir): + refs = ParserRef() + test_target = {'source': source, 'table': table} + for column in table.get('columns', []): + column_tests = self._parse_column(test_target, column, + package_name, root_dir, path, + refs) + for node in column_tests: + yield 'test', node + + for test in table.get('tests', []): + node = self.build_test_node(test_target, package_name, test, + root_dir, path) + yield 'test', node + + node = self.generate_source_node(source, table, path, package_name, + root_dir, refs) + yield 'source', node + + def parse_source_entry(self, source, path, package_name, root_dir): + nodes = itertools.chain.from_iterable( + self.parse_source_table(source, table, path, package_name, + root_dir) + for table in source.tables + ) + for node_type, node in nodes: + yield node_type, node + + def _sources_validate(self, **kwargs): + kwargs = self._renderer.render_schema_source(kwargs) + return UnparsedSourceDefinition(**kwargs) + + def parse_all(self, sources, path, package_name, root_dir): + """Parse all the model dictionaries in sources. + + :param List[dict] sources: The `sources` section of the schema.yml, as + a list of dicts. + :param str path: The path to the schema.yml file + :param str package_name: The name of the current package + :param str root_dir: The root directory of the search + """ + filtered = _filter_validate(path, 'sources', sources, + self._sources_validate) + nodes = itertools.chain.from_iterable( + self.parse_source_entry(source, path, package_name, root_dir) + for source in filtered + ) + + for node_type, node in nodes: + yield node_type, node + - parsed = self.build_parsed_node(unparsed, model_name, test_namespace, - original_test_type, column_name) - return parsed +class SchemaParser(object): + def __init__(self, root_project_config, all_projects, macro_manifest): + self.root_project_config = root_project_config + self.all_projects = all_projects + self.macro_manifest = macro_manifest @classmethod def find_schema_yml(cls, package_name, root_dir, relative_dirs): @@ -264,211 +487,63 @@ def find_schema_yml(cls, package_name, root_dir, relative_dirs): yield original_file_path, test_yml - def parse_v1_test_yml(self, original_file_path, test_yml, package_name, - root_dir): - """Parse v1 yml contents, yielding parsed nodes. - - A v1 yml file is laid out like this ('variables' written - bash-curly-brace style): - - ${model_name}: - constraints: - ${constraint_type}: - - ${column_1} - - ${column_2} - ${other_constraint_type}: - - ... - ${other_model_name}: - constraints: - ... - """ - for model_name, test_spec in test_yml.items(): - # in v1 we can really only have constraints, so not having any is - # a concern - no_tests_warning = ( - "* WARNING: No constraints found for model '{}' in file {}\n" - ) - if not isinstance(test_spec, dict): - msg = ( - "Invalid test config given in {} near {} (expected a dict)" - ).format(original_file_path, test_spec) - if dbt.flags.STRICT_MODE: - dbt.exceptions.raise_compiler_error(msg) - dbt.utils.compiler_warning(model_name, msg, - resource_type='test') - continue - - if test_spec is None or test_spec.get('constraints') is None: - logger.warning(no_tests_warning.format(model_name, - original_file_path)) - continue - constraints = test_spec.get('constraints', {}) - for test_type, configs in constraints.items(): - if configs is None: - continue - - if not isinstance(configs, (list, tuple)): - dbt.utils.compiler_warning( - model_name, - "Invalid test config given in {}".format( - original_file_path) - ) - continue - - for config in configs: - test_args = self._build_v1_test_args(config) - to_add = self.build_node( - model_name, package_name, test_type, test_args, - root_dir, original_file_path) - if to_add is not None: - yield to_add - - def parse_v2_yml(self, original_file_path, test_yml, package_name, - root_dir): - """Parse v2 yml contents, yielding both parsed nodes and node patches. - - A v2 yml file is laid out like this ('variables' written - bash-curly-brace style): - - models: - - name: ${model_name} - description: ${node_description} - columns: - - name: ${column_1} - description: ${column_1_description} - tests: - - ${constraint_type} - - ${other_constraint_type} - - name: ${column_2} - description: ${column_2_description} - tests: - - ${constraint_type}: {$keyword_args_dict} - ... - - name: ${other_model_name} - ... - """ - if 'models' not in test_yml: - # You could legitimately not have any models in your schema.yml, if - # sources were supported - return - - for model in test_yml['models']: - if not isinstance(model, dict): - msg = ( - "Invalid test config given in {} near {} (expected a dict)" - ).format(original_file_path, model) - if dbt.flags.STRICT_MODE: - dbt.exceptions.raise_compiler_error(msg, model) - dbt.utils.compiler_warning(model, msg) - continue - try: - model = UnparsedNodeUpdate(**model) - except dbt.exceptions.JSONValidationException as exc: - # we don't want to fail the full run, but we do want to fail - # parsing this file - msg = "Invalid test config given in {}: {}".format( - original_file_path, exc.errors_message - ) - if dbt.flags.STRICT_MODE: - dbt.exceptions.raise_compiler_error(msg, model) - - dbt.utils.compiler_warning(model.get('name'), msg) - continue - - iterator = self.parse_model(model, package_name, root_dir, - original_file_path) - - for node_type, node in iterator: - yield node_type, node - - def parse_model(self, model, package_name, root_dir, path): - """Given an UnparsedNodeUpdate, return column info about the model - - - column info (name and maybe description) as a dict - - a list of ParsedNodes repreenting tests - - This is only used in parsing the v2 schema. - """ - model_name = model['name'] - docrefs = [] - column_info = {} - for column in model.get('columns', []): - column_name = column['name'] - description = column.get('description', '') - column_info[column_name] = { - 'name': column_name, - 'description': description, - } - context = { - 'doc': dbt.context.parser.docs(model, docrefs, column_name) - } - dbt.clients.jinja.get_rendered(description, context) - for test in column.get('tests', []): - test_type, test_args = self._build_v2_test_args( - test, column_name - ) - node = self.build_node( - model_name, package_name, test_type, test_args, root_dir, - path, column_name - ) - yield 'test', node - - for test in model.get('tests', []): - # table tests don't inject any extra values, model name is - # available via `model.name` - test_type, test_args = self._build_v2_test_args(test, None) - node = self.build_node(model_name, package_name, test_type, - test_args, root_dir, path) - yield 'test', node + def parse_schema(self, path, test_yml, package_name, root_dir): + model_parser = SchemaModelParser(self.root_project_config, + self.all_projects, + self.macro_manifest) + source_parser = SchemaSourceParser(self.root_project_config, + self.all_projects, + self.macro_manifest) + models = test_yml.get('models', []) + sources = test_yml.get('sources', []) + return itertools.chain( + model_parser.parse_all(models, path, package_name, root_dir), + source_parser.parse_all(sources, path, package_name, root_dir), + ) - context = {'doc': dbt.context.parser.docs(model, docrefs)} - description = model.get('description', '') - dbt.clients.jinja.get_rendered(description, context) + def _parse_format_version(self, path, test_yml): + if 'version' not in test_yml: + dbt.exceptions.raise_invalid_schema_yml_version( + path, 'no version is specified' + ) - patch = ParsedNodePatch( - name=model_name, - original_file_path=path, - description=description, - columns=column_info, - docrefs=docrefs - ) - yield 'patch', patch + version = test_yml['version'] + # if it's not an integer, the version is malformed, or not + # set. Either way, only 'version: 2' is supported. + if not isinstance(version, int): + dbt.exceptions.raise_invalid_schema_yml_version( + path, 'the version is not an integer' + ) + return version def load_and_parse(self, package_name, root_dir, relative_dirs): if dbt.flags.STRICT_MODE: dbt.contracts.project.ProjectList(**self.all_projects) new_tests = {} # test unique ID -> ParsedNode node_patches = {} # model name -> dict + new_sources = {} # source unique ID -> ParsedSourceDefinition iterator = self.find_schema_yml(package_name, root_dir, relative_dirs) - for original_file_path, test_yml in iterator: - version = test_yml.get('version', 1) - # the version will not be an int if it's a v1 model that has a - # model named 'version'. - if version == 1 or not isinstance(version, int): - self.check_v2_missing_version(original_file_path, test_yml) - new_tests.update( - (t.get('unique_id'), t) - for t in self.parse_v1_test_yml( - original_file_path, test_yml, package_name, root_dir) - ) - elif version == 2: - v2_results = self.parse_v2_yml( - original_file_path, test_yml, package_name, root_dir) - for result_type, node in v2_results: - if result_type == 'patch': - node_patches[node.name] = node - elif result_type == 'test': - new_tests[node.unique_id] = node - else: - raise dbt.exceptions.InternalException( - 'Got invalid result type {} '.format(result_type) - ) - else: - dbt.exceptions.raise_compiler_error(( - 'Got an invalid schema.yml version {} in {}, only 1 and 2 ' - 'are supported').format(version, original_file_path) + for path, test_yml in iterator: + version = self._parse_format_version(path, test_yml) + if version != 2: + dbt.exceptions.raise_invalid_schema_yml_version( + path, + 'version {} is not supported'.format(version) ) - return new_tests, node_patches + results = self.parse_schema(path, test_yml, package_name, root_dir) + for result_type, node in results: + if result_type == 'patch': + node_patches[node.name] = node + elif result_type == 'test': + new_tests[node.unique_id] = node + elif result_type == 'source': + new_sources[node.unique_id] = node + else: + raise dbt.exceptions.InternalException( + 'Got invalid result type {} '.format(result_type) + ) + + return new_tests, node_patches, new_sources diff --git a/core/dbt/parser/util.py b/core/dbt/parser/util.py index 52d3aaf0bd9..c4c2245746d 100644 --- a/core/dbt/parser/util.py +++ b/core/dbt/parser/util.py @@ -37,6 +37,22 @@ def do_docs(*args): class ParserUtils(object): DISABLED = object() + @classmethod + def resolve_source(cls, manifest, target_source_name, + target_table_name, current_project, node_package): + candidate_targets = [current_project, node_package, None] + target_source = None + for candidate in candidate_targets: + target_source = manifest.find_source_by_name( + target_source_name, + target_table_name, + candidate + ) + if target_source is not None: + return target_source + + return None + @classmethod def resolve_ref(cls, manifest, target_model_name, target_model_package, current_project, node_package): @@ -176,3 +192,28 @@ def process_refs(cls, manifest, current_project): manifest.nodes[node['unique_id']] = node return manifest + + @classmethod + def process_sources(cls, manifest, current_project): + for _, node in manifest.nodes.items(): + target_source = None + for source_name, table_name in node.sources: + target_source = cls.resolve_source( + manifest, + source_name, + table_name, + current_project, + node.get('package_name')) + + if target_source is None: + # this folows the same pattern as refs + node.config['enabled'] = False + dbt.utils.invalid_source_fail_unless_test( + node, + source_name, + table_name) + continue + target_source_id = target_source.unique_id + node.depends_on['nodes'].append(target_source_id) + manifest.nodes[node['unique_id']] = node + return manifest diff --git a/core/dbt/utils.py b/core/dbt/utils.py index db6be8cb7c7..cf0b35fa463 100644 --- a/core/dbt/utils.py +++ b/core/dbt/utils.py @@ -73,9 +73,11 @@ def get_model_name_or_none(model): elif isinstance(model, basestring): name = model elif isinstance(model, dict): - name = model['alias'] - else: + name = model.get('alias', model.get('name')) + elif hasattr(model, 'nice_name'): name = model.nice_name + else: + name = str(model) return name @@ -94,13 +96,22 @@ def id_matches(unique_id, target_name, target_package, nodetypes, model): nodetypes should be a container of NodeTypes that implements the 'in' operator. """ - node_parts = unique_id.split('.') + node_type = model.get('resource_type', 'node') + node_parts = unique_id.split('.', 2) if len(node_parts) != 3: - node_type = model.get('resource_type', 'node') - msg = "{} names cannot contain '.' characters".format(node_type) + msg = "unique_id {} is malformed".format(unique_id) dbt.exceptions.raise_compiler_error(msg, model) resource_type, package_name, node_name = node_parts + if node_type == NodeType.Source: + if node_name.count('.') != 1: + msg = "{} names must contain exactly 1 '.' character"\ + .format(node_type) + dbt.exceptions.raise_compiler_error(msg, model) + else: + if '.' in node_name: + msg = "{} names cannot contain '.' characters".format(node_type) + dbt.exceptions.raise_compiler_error(msg, model) if resource_type not in nodetypes: return False @@ -414,6 +425,16 @@ def invalid_ref_fail_unless_test(node, target_model_name, target_model_package) +def invalid_source_fail_unless_test(node, target_name, target_table_name): + if node.get('resource_type') == NodeType.Test: + msg = dbt.exceptions.source_disabled_message(node, target_name, + target_table_name) + logger.warning('WARNING: {}'.format(msg)) + else: + dbt.exceptions.source_target_not_found(node, target_name, + target_table_name) + + def parse_cli_vars(var_string): try: cli_vars = yaml_helper.load_yaml_text(var_string) diff --git a/test/integration/001_simple_copy_test/models/schema.yml b/test/integration/001_simple_copy_test/models/schema.yml index f8eb708f45f..9a813eed538 100644 --- a/test/integration/001_simple_copy_test/models/schema.yml +++ b/test/integration/001_simple_copy_test/models/schema.yml @@ -1,8 +1,7 @@ - - -# Confirm that this does not throw an exception for -# a missing ref to the disabled model -disabled: - constraints: - unique: - - id +version: 2 +models: +- name: disabled + columns: + - name: id + tests: + - unique diff --git a/test/integration/005_simple_seed_test/macros/schema_test.sql b/test/integration/005_simple_seed_test/macros/schema_test.sql index 564a4fd334a..cdfa116a860 100644 --- a/test/integration/005_simple_seed_test/macros/schema_test.sql +++ b/test/integration/005_simple_seed_test/macros/schema_test.sql @@ -1,5 +1,5 @@ -{% macro test_column_type(model, field, type) %} +{% macro test_column_type(model, column_name, type) %} {% set cols = adapter.get_columns_in_relation(model) %} @@ -8,7 +8,7 @@ {% set _ = col_types.update({col.name: col.data_type}) %} {% endfor %} - {% set val = 0 if col_types[field] == type else 1 %} + {% set val = 0 if col_types[column_name] == type else 1 %} select {{ val }} as pass_fail diff --git a/test/integration/005_simple_seed_test/models-bq/schema.yml b/test/integration/005_simple_seed_test/models-bq/schema.yml index b0023f65fc4..118c3e02e02 100644 --- a/test/integration/005_simple_seed_test/models-bq/schema.yml +++ b/test/integration/005_simple_seed_test/models-bq/schema.yml @@ -1,6 +1,12 @@ - -seed_enabled: - constraints: - column_type: - - {field: id, type: 'FLOAT64' } - - {field: birthday, type: 'STRING' } +version: 2 +models: +- name: seed_enabled + columns: + - name: birthday + tests: + - column_type: + type: STRING + - name: id + tests: + - column_type: + type: FLOAT64 diff --git a/test/integration/005_simple_seed_test/models-pg/schema.yml b/test/integration/005_simple_seed_test/models-pg/schema.yml index 2b068c10488..871fd51854d 100644 --- a/test/integration/005_simple_seed_test/models-pg/schema.yml +++ b/test/integration/005_simple_seed_test/models-pg/schema.yml @@ -1,6 +1,12 @@ - -seed_enabled: - constraints: - column_type: - - {field: id, type: 'character varying(255)' } - - {field: birthday, type: 'date' } +version: 2 +models: +- name: seed_enabled + columns: + - name: birthday + tests: + - column_type: + type: date + - name: id + tests: + - column_type: + type: character varying(255) diff --git a/test/integration/005_simple_seed_test/models-snowflake/schema.yml b/test/integration/005_simple_seed_test/models-snowflake/schema.yml index 846478fb426..d12cc4c2d6c 100644 --- a/test/integration/005_simple_seed_test/models-snowflake/schema.yml +++ b/test/integration/005_simple_seed_test/models-snowflake/schema.yml @@ -1,6 +1,12 @@ - -seed_enabled: - constraints: - column_type: - - {field: ID, type: 'FLOAT' } - - {field: BIRTHDAY, type: 'character varying(16777216)' } +version: 2 +models: +- name: seed_enabled + columns: + - name: BIRTHDAY + tests: + - column_type: + type: character varying(16777216) + - name: ID + tests: + - column_type: + type: FLOAT diff --git a/test/integration/007_graph_selection_tests/models/schema.yml b/test/integration/007_graph_selection_tests/models/schema.yml index aad82fd785e..6e06720672e 100644 --- a/test/integration/007_graph_selection_tests/models/schema.yml +++ b/test/integration/007_graph_selection_tests/models/schema.yml @@ -1,15 +1,17 @@ - -emails: - constraints: - unique: - - email - -users: - constraints: - unique: - - id - -users_rollup: - constraints: - unique: - - gender +version: 2 +models: +- name: emails + columns: + - name: email + tests: + - unique +- name: users + columns: + - name: id + tests: + - unique +- name: users_rollup + columns: + - name: gender + tests: + - unique diff --git a/test/integration/008_schema_tests_test/macros-v1/tests.sql b/test/integration/008_schema_tests_test/macros-v1/tests.sql deleted file mode 100644 index e962f7896a8..00000000000 --- a/test/integration/008_schema_tests_test/macros-v1/tests.sql +++ /dev/null @@ -1,26 +0,0 @@ - - -{% macro test_every_value_is_blue(model, arg) %} - - select - count(*) - - from {{ model }} - where {{ arg }} != 'blue' - -{% endmacro %} - - -{% macro test_rejected_values(model, field, values) %} - - select - count(*) - - from {{ model }} - where {{ field }} in ( - {% for value in values %} - '{{ value }}' {% if not loop.last %} , {% endif %} - {% endfor %} - ) - -{% endmacro %} diff --git a/test/integration/008_schema_tests_test/models-v1/custom/schema.yml b/test/integration/008_schema_tests_test/models-v1/custom/schema.yml deleted file mode 100644 index 15c9b95552a..00000000000 --- a/test/integration/008_schema_tests_test/models-v1/custom/schema.yml +++ /dev/null @@ -1,22 +0,0 @@ -table_copy: - constraints: - - # fail - not_null: - - email - - # pass - unique: - - id - - # fails - every_value_is_blue: - - favorite_color - - # passes - rejected_values: - - { field: 'favorite_color', values: ['orange', 'purple'] } - - # passes - dbt_utils.equality: - - ref('table_copy') diff --git a/test/integration/008_schema_tests_test/models-v1/custom/table_copy.sql b/test/integration/008_schema_tests_test/models-v1/custom/table_copy.sql deleted file mode 100644 index 56e90a6d93c..00000000000 --- a/test/integration/008_schema_tests_test/models-v1/custom/table_copy.sql +++ /dev/null @@ -1,8 +0,0 @@ - -{{ - config( - materialized='table' - ) -}} - -select * from {{ this.schema }}.seed diff --git a/test/integration/008_schema_tests_test/models-v1/malformed/schema.yml b/test/integration/008_schema_tests_test/models-v1/malformed/schema.yml deleted file mode 100644 index 4cc04bc3540..00000000000 --- a/test/integration/008_schema_tests_test/models-v1/malformed/schema.yml +++ /dev/null @@ -1,11 +0,0 @@ - -table_copy: - constraints: - not_null: - - id - unique: - - id - - accepted_values: - # this is missing a "-" and is malformed - { field: favorite_color, values: ['blue', 'green'] } diff --git a/test/integration/008_schema_tests_test/models-v1/malformed/table_copy.sql b/test/integration/008_schema_tests_test/models-v1/malformed/table_copy.sql deleted file mode 100644 index 56e90a6d93c..00000000000 --- a/test/integration/008_schema_tests_test/models-v1/malformed/table_copy.sql +++ /dev/null @@ -1,8 +0,0 @@ - -{{ - config( - materialized='table' - ) -}} - -select * from {{ this.schema }}.seed diff --git a/test/integration/008_schema_tests_test/models-v1/models/.gitkeep b/test/integration/008_schema_tests_test/models-v1/models/.gitkeep deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/test/integration/008_schema_tests_test/models-v1/models/schema.yml b/test/integration/008_schema_tests_test/models-v1/models/schema.yml deleted file mode 100644 index 12e1b4df9f9..00000000000 --- a/test/integration/008_schema_tests_test/models-v1/models/schema.yml +++ /dev/null @@ -1,73 +0,0 @@ - -table_copy: - constraints: - not_null: - - id - - first_name - - ip_address - - updated_at - - unique: - - id - - email - - accepted_values: - - field: favorite_color - values: - - 'blue' - - 'green' - - # Include extra (long) options here to ensure that dbt does - # not fail on parsing of really long model names or unique ids - - 'other_long_option_long_option_long_option_1' - - 'other_long_option_long_option_long_option_2' - - 'other_long_option_long_option_long_option_3' - - 'other_long_option_long_option_long_option_4' - - 'other_long_option_long_option_long_option_5' - - 'other_long_option_long_option_long_option_6' - - -table_summary: - constraints: - not_null: - - favorite_color_copy - - count - - unique: - - favorite_color_copy - - accepted_values: - - { field: favorite_color_copy, values: ['blue', 'green'] } - - relationships: - - { from: favorite_color_copy, to: ref('table_copy'), field: favorite_color } - - -# all of these constraints will fail -table_failure_copy: - constraints: - not_null: - - id - - unique: - - id - - accepted_values: - - { field: favorite_color, values: ['blue', 'green'] } - - -# all of these constraints will fail -table_failure_summary: - constraints: - accepted_values: - - { field: favorite_color, values: ['red'] } - - relationships: - - { from: favorite_color, to: ref('table_copy'), field: favorite_color } - - -# all of these constraints will fail -table_failure_null_relation: - constraints: - relationships: - - { from: id, to: ref('table_failure_copy'), field: id } diff --git a/test/integration/008_schema_tests_test/models-v1/models/table_copy.sql b/test/integration/008_schema_tests_test/models-v1/models/table_copy.sql deleted file mode 100644 index 56e90a6d93c..00000000000 --- a/test/integration/008_schema_tests_test/models-v1/models/table_copy.sql +++ /dev/null @@ -1,8 +0,0 @@ - -{{ - config( - materialized='table' - ) -}} - -select * from {{ this.schema }}.seed diff --git a/test/integration/008_schema_tests_test/models-v1/models/table_failure_copy.sql b/test/integration/008_schema_tests_test/models-v1/models/table_failure_copy.sql deleted file mode 100644 index b14f583b466..00000000000 --- a/test/integration/008_schema_tests_test/models-v1/models/table_failure_copy.sql +++ /dev/null @@ -1,8 +0,0 @@ - -{{ - config( - materialized='table' - ) -}} - -select * from {{ this.schema }}.seed_failure diff --git a/test/integration/008_schema_tests_test/models-v1/models/table_failure_null_relation.sql b/test/integration/008_schema_tests_test/models-v1/models/table_failure_null_relation.sql deleted file mode 100644 index fccfb751385..00000000000 --- a/test/integration/008_schema_tests_test/models-v1/models/table_failure_null_relation.sql +++ /dev/null @@ -1,10 +0,0 @@ -{{ - config( - materialized='table' - ) -}} - --- force a foreign key constraint failure here -select 105 as id, count(*) as count -from {{ ref('table_failure_copy') }} -group by 1 diff --git a/test/integration/008_schema_tests_test/models-v1/models/table_failure_summary.sql b/test/integration/008_schema_tests_test/models-v1/models/table_failure_summary.sql deleted file mode 100644 index c1db2e20637..00000000000 --- a/test/integration/008_schema_tests_test/models-v1/models/table_failure_summary.sql +++ /dev/null @@ -1,10 +0,0 @@ -{{ - config( - materialized='table' - ) -}} - --- force a foreign key constraint failure here -select 'purple' as favorite_color, count(*) as count -from {{ ref('table_failure_copy') }} -group by 1 diff --git a/test/integration/008_schema_tests_test/models-v1/models/table_summary.sql b/test/integration/008_schema_tests_test/models-v1/models/table_summary.sql deleted file mode 100644 index dfe200b331e..00000000000 --- a/test/integration/008_schema_tests_test/models-v1/models/table_summary.sql +++ /dev/null @@ -1,9 +0,0 @@ -{{ - config( - materialized='table' - ) -}} - -select favorite_color as favorite_color_copy, count(*) as count -from {{ ref('table_copy') }} -group by 1 diff --git a/test/integration/008_schema_tests_test/test_schema_tests.py b/test/integration/008_schema_tests_test/test_schema_tests.py deleted file mode 100644 index fb8dd93809f..00000000000 --- a/test/integration/008_schema_tests_test/test_schema_tests.py +++ /dev/null @@ -1,145 +0,0 @@ -from nose.plugins.attrib import attr -from test.integration.base import DBTIntegrationTest, FakeArgs - -from dbt.task.test import TestTask - - -class TestSchemaTests(DBTIntegrationTest): - - def setUp(self): - DBTIntegrationTest.setUp(self) - self.run_sql_file("test/integration/008_schema_tests_test/seed.sql") - self.run_sql_file("test/integration/008_schema_tests_test/seed_failure.sql") - - @property - def schema(self): - return "schema_tests_008" - - @property - def models(self): - return "test/integration/008_schema_tests_test/models-v1/models" - - def run_schema_validations(self): - args = FakeArgs() - - test_task = TestTask(args, self.config) - return test_task.run() - - @attr(type='postgres') - def test_schema_tests(self): - results = self.run_dbt() - self.assertEqual(len(results), 5) - test_results = self.run_schema_validations() - self.assertEqual(len(test_results), 18) - - for result in test_results: - # assert that all deliberately failing tests actually fail - if 'failure' in result.node.get('name'): - self.assertFalse(result.errored) - self.assertFalse(result.skipped) - self.assertTrue( - result.status > 0, - 'test {} did not fail'.format(result.node.get('name')) - ) - - # assert that actual tests pass - else: - self.assertFalse(result.errored) - self.assertFalse(result.skipped) - # status = # of failing rows - self.assertEqual( - result.status, 0, - 'test {} failed'.format(result.node.get('name')) - ) - - self.assertEqual(sum(x.status for x in test_results), 6) - - -class TestMalformedSchemaTests(DBTIntegrationTest): - - def setUp(self): - DBTIntegrationTest.setUp(self) - self.run_sql_file("test/integration/008_schema_tests_test/seed.sql") - - @property - def schema(self): - return "schema_tests_008" - - @property - def models(self): - return "test/integration/008_schema_tests_test/models-v1/malformed" - - def run_schema_validations(self): - args = FakeArgs() - - test_task = TestTask(args, self.config) - return test_task.run() - - @attr(type='postgres') - def test_malformed_schema_test_wont_brick_run(self): - # dbt run should work (Despite broken schema test) - results = self.run_dbt() - self.assertEqual(len(results), 1) - - ran_tests = self.run_schema_validations() - self.assertEqual(len(ran_tests), 2) - self.assertEqual(sum(x.status for x in ran_tests), 0) - - -class TestCustomSchemaTests(DBTIntegrationTest): - - def setUp(self): - DBTIntegrationTest.setUp(self) - self.run_sql_file("test/integration/008_schema_tests_test/seed.sql") - - @property - def schema(self): - return "schema_tests_008" - - @property - def packages_config(self): - return { - "packages": [ - { - 'git': 'https://github.com/fishtown-analytics/dbt-utils', - 'revision': '0.13-support', - }, - {'git': 'https://github.com/fishtown-analytics/dbt-integration-project'}, - ] - } - - - @property - def project_config(self): - # dbt-utils containts a schema test (equality) - # dbt-integration-project contains a schema.yml file - # both should work! - return { - "macro-paths": ["test/integration/008_schema_tests_test/macros-v1"], - } - - @property - def models(self): - return "test/integration/008_schema_tests_test/models-v1/custom" - - def run_schema_validations(self): - args = FakeArgs() - - test_task = TestTask(args, self.config) - return test_task.run() - - @attr(type='postgres') - def test_schema_tests(self): - self.run_dbt(["deps"]) - results = self.run_dbt() - self.assertEqual(len(results), 4) - - test_results = self.run_schema_validations() - self.assertEqual(len(test_results), 6) - - expected_failures = ['unique', 'every_value_is_blue'] - - for result in test_results: - if result.errored: - self.assertTrue(result.node['name'] in expected_failures) - self.assertEqual(sum(x.status for x in test_results), 52) diff --git a/test/integration/014_hook_tests/seed-models-bq/schema.yml b/test/integration/014_hook_tests/seed-models-bq/schema.yml index c112d2d3ebf..187a0db9bfe 100644 --- a/test/integration/014_hook_tests/seed-models-bq/schema.yml +++ b/test/integration/014_hook_tests/seed-models-bq/schema.yml @@ -1,5 +1,7 @@ - -example_seed: - constraints: - not_null: - - a +version: 2 +models: +- name: example_seed + columns: + - name: a + tests: + - not_null diff --git a/test/integration/014_hook_tests/seed-models/schema.yml b/test/integration/014_hook_tests/seed-models/schema.yml index d8d9adbfbd5..5017027d8f5 100644 --- a/test/integration/014_hook_tests/seed-models/schema.yml +++ b/test/integration/014_hook_tests/seed-models/schema.yml @@ -1,5 +1,7 @@ - -example_seed: - constraints: - not_null: - - new_col +version: 2 +models: +- name: example_seed + columns: + - name: new_col + tests: + - not_null diff --git a/test/integration/015_cli_invocation_tests/models/subdir1/subdir2/schema.yml b/test/integration/015_cli_invocation_tests/models/subdir1/subdir2/schema.yml index cb4b5a13a52..9d2c459d876 100644 --- a/test/integration/015_cli_invocation_tests/models/subdir1/subdir2/schema.yml +++ b/test/integration/015_cli_invocation_tests/models/subdir1/subdir2/schema.yml @@ -1,5 +1,9 @@ - -model: - constraints: - accepted_values: - - { field: id, values: [1] } +version: 2 +models: +- name: model + columns: + - name: id + tests: + - accepted_values: + values: + - 1 diff --git a/test/integration/022_bigquery_test/adapter-models/schema.yml b/test/integration/022_bigquery_test/adapter-models/schema.yml index 31a8249b31f..97a54b3c763 100644 --- a/test/integration/022_bigquery_test/adapter-models/schema.yml +++ b/test/integration/022_bigquery_test/adapter-models/schema.yml @@ -1,21 +1,39 @@ - - -test_get_columns_in_table: - constraints: - not_null: - - field_1 - - field_2 - - field_3 - - nested_field - - repeated_column - - -test_flattened_get_columns_in_table: - constraints: - not_null: - - field_1 - - field_2 - - field_3 - - field_4 - - field_5 - - repeated_column +version: 2 +models: +- name: test_get_columns_in_table + columns: + - name: field_1 + tests: + - not_null + - name: field_2 + tests: + - not_null + - name: field_3 + tests: + - not_null + - name: nested_field + tests: + - not_null + - name: repeated_column + tests: + - not_null +- name: test_flattened_get_columns_in_table + columns: + - name: field_1 + tests: + - not_null + - name: field_2 + tests: + - not_null + - name: field_3 + tests: + - not_null + - name: field_4 + tests: + - not_null + - name: field_5 + tests: + - not_null + - name: repeated_column + tests: + - not_null diff --git a/test/integration/022_bigquery_test/dp-models/schema.yml b/test/integration/022_bigquery_test/dp-models/schema.yml index d699498135c..f054dee50f5 100644 --- a/test/integration/022_bigquery_test/dp-models/schema.yml +++ b/test/integration/022_bigquery_test/dp-models/schema.yml @@ -1,18 +1,30 @@ - -# check that this exists -partitioned_simple: - constraints: - unique: - - id - not_null: - - id - -confirmation: - constraints: - accepted_values: - - {field: cast(day_1 as string), values:[1] } - - {field: cast(day_2 as string), values:[1] } - - {field: cast(day_3 as string), values:[1] } - - {field: cast(count_days as string), values:[3] } - - +version: 2 +models: +- name: partitioned_simple + columns: + - name: id + tests: + - not_null + - unique +- name: confirmation + columns: + - name: cast(count_days as string) + tests: + - accepted_values: + values: + - 3 + - name: cast(day_1 as string) + tests: + - accepted_values: + values: + - 1 + - name: cast(day_2 as string) + tests: + - accepted_values: + values: + - 1 + - name: cast(day_3 as string) + tests: + - accepted_values: + values: + - 1 diff --git a/test/integration/022_bigquery_test/models/schema.yml b/test/integration/022_bigquery_test/models/schema.yml index 88114ee08a4..30601d388da 100644 --- a/test/integration/022_bigquery_test/models/schema.yml +++ b/test/integration/022_bigquery_test/models/schema.yml @@ -1,21 +1,27 @@ - -view_model: - constraints: - not_null: - - id - - updated_at - - unique: - - id - - dupe # fails - - was_materialized: - - {name: view_model, type: view} - -table_model: - constraints: - not_null: - - id - - was_materialized: - - {name: table_model, type: table} +version: 2 +models: +- name: view_model + columns: + - name: dupe + tests: + - unique + - name: id + tests: + - not_null + - unique + - name: updated_at + tests: + - not_null + tests: + - was_materialized: + name: view_model + type: view +- name: table_model + columns: + - name: id + tests: + - not_null + tests: + - was_materialized: + name: table_model + type: table diff --git a/test/integration/023_exit_codes_test/models/schema.yml b/test/integration/023_exit_codes_test/models/schema.yml index fe15de22e69..f7243286b7b 100644 --- a/test/integration/023_exit_codes_test/models/schema.yml +++ b/test/integration/023_exit_codes_test/models/schema.yml @@ -1,15 +1,17 @@ - -good: - constraints: - not_null: - - updated_at - -bad: - constraints: - not_null: - - updated_at - -dupe: - constraints: - unique: - - updated_at # fails +version: 2 +models: +- name: good + columns: + - name: updated_at + tests: + - not_null +- name: bad + columns: + - name: updated_at + tests: + - not_null +- name: dupe + columns: + - name: updated_at + tests: + - unique diff --git a/test/integration/026_aliases_test/models-dupe-custom-schema/schema.yml b/test/integration/026_aliases_test/models-dupe-custom-schema/schema.yml index 10168fd13c8..d1e13ebcc68 100644 --- a/test/integration/026_aliases_test/models-dupe-custom-schema/schema.yml +++ b/test/integration/026_aliases_test/models-dupe-custom-schema/schema.yml @@ -1,18 +1,17 @@ - -# All of these models have the same alias, but are materialized -# in different schemas, so they should all succeed - -model_a: - constraints: - expect_value: - - {field: tablename, value: 'duped_alias'} - -model_b: - constraints: - expect_value: - - {field: tablename, value: duped_alias} - -model_c: - constraints: - expect_value: - - {field: tablename, value: 'duped_alias'} +version: 2 +models: +- name: model_a + tests: + - expect_value: + field: tablename + value: duped_alias +- name: model_b + tests: + - expect_value: + field: tablename + value: duped_alias +- name: model_c + tests: + - expect_value: + field: tablename + value: duped_alias diff --git a/test/integration/026_aliases_test/models/schema.yml b/test/integration/026_aliases_test/models/schema.yml index 83a2e331830..b3a82faad31 100644 --- a/test/integration/026_aliases_test/models/schema.yml +++ b/test/integration/026_aliases_test/models/schema.yml @@ -1,20 +1,22 @@ - -foo_alias: - constraints: - expect_value: - - {field: 'tablename', value: 'foo'} - -ref_foo_alias: - constraints: - expect_value: - - {field: 'tablename', value: 'ref_foo_alias'} - -alias_in_project: - constraints: - expect_value: - - {field: 'tablename', value: 'project_alias'} - -alias_in_project_with_override: - constraints: - expect_value: - - {field: 'tablename', value: 'override_alias'} +version: 2 +models: +- name: foo_alias + tests: + - expect_value: + field: tablename + value: foo +- name: ref_foo_alias + tests: + - expect_value: + field: tablename + value: ref_foo_alias +- name: alias_in_project + tests: + - expect_value: + field: tablename + value: project_alias +- name: alias_in_project_with_override + tests: + - expect_value: + field: tablename + value: override_alias diff --git a/test/integration/028_cli_vars/models_complex/schema.yml b/test/integration/028_cli_vars/models_complex/schema.yml index 1b9f3156167..7000d58c951 100644 --- a/test/integration/028_cli_vars/models_complex/schema.yml +++ b/test/integration/028_cli_vars/models_complex/schema.yml @@ -1,7 +1,19 @@ - -complex_model: - constraints: - accepted_values: - - {field: var_1, values: ["abc"]} - - {field: var_2, values: ["def"]} - - {field: var_3, values: ["jkl"]} +version: 2 +models: +- name: complex_model + columns: + - name: var_1 + tests: + - accepted_values: + values: + - abc + - name: var_2 + tests: + - accepted_values: + values: + - def + - name: var_3 + tests: + - accepted_values: + values: + - jkl diff --git a/test/integration/028_cli_vars/models_override/schema.yml b/test/integration/028_cli_vars/models_override/schema.yml index 7bc9ee11d56..44209f278b8 100644 --- a/test/integration/028_cli_vars/models_override/schema.yml +++ b/test/integration/028_cli_vars/models_override/schema.yml @@ -1,6 +1,9 @@ - -test_vars: - constraints: - accepted_values: - - {field: 'field', values: ['override']} - +version: 2 +models: +- name: test_vars + columns: + - name: field + tests: + - accepted_values: + values: + - override diff --git a/test/integration/028_cli_vars/models_simple/schema.yml b/test/integration/028_cli_vars/models_simple/schema.yml index 98e80979a55..cda0fea6381 100644 --- a/test/integration/028_cli_vars/models_simple/schema.yml +++ b/test/integration/028_cli_vars/models_simple/schema.yml @@ -1,5 +1,9 @@ - -simple_model: - constraints: - accepted_values: - - {field: simple, values: ["abc"]} +version: 2 +models: +- name: simple_model + columns: + - name: simple + tests: + - accepted_values: + values: + - abc diff --git a/test/integration/029_docs_generate_tests/ref_models/docs.md b/test/integration/029_docs_generate_tests/ref_models/docs.md index 8215c323476..b3bf0e74a48 100644 --- a/test/integration/029_docs_generate_tests/ref_models/docs.md +++ b/test/integration/029_docs_generate_tests/ref_models/docs.md @@ -14,4 +14,10 @@ The number of instances of the first name A view of the summary of the ephemeral copy of the seed data {% enddocs %} +{% docs source_info %} +My source +{% enddocs %} +{% docs table_info %} +My table +{% enddocs %} diff --git a/test/integration/029_docs_generate_tests/ref_models/ephemeral_copy.sql b/test/integration/029_docs_generate_tests/ref_models/ephemeral_copy.sql index c6c2c53e900..3f7e698ce4d 100644 --- a/test/integration/029_docs_generate_tests/ref_models/ephemeral_copy.sql +++ b/test/integration/029_docs_generate_tests/ref_models/ephemeral_copy.sql @@ -4,4 +4,4 @@ ) }} -select * from {{ this.schema }}.seed +select * from {{ source("my_source", "my_table") }} diff --git a/test/integration/029_docs_generate_tests/ref_models/schema.yml b/test/integration/029_docs_generate_tests/ref_models/schema.yml index c2b50e1e47d..7c133dbe5a2 100644 --- a/test/integration/029_docs_generate_tests/ref_models/schema.yml +++ b/test/integration/029_docs_generate_tests/ref_models/schema.yml @@ -11,3 +11,15 @@ models: - name: view_summary description: "{{ doc('view_summary') }}" columns: *summary_columns + +sources: + - name: my_source + description: "{{ doc('source_info') }}" + loader: a_loader + tables: + - name: my_table + description: "{{ doc('table_info') }}" + sql_table_name: "{{ var('test_schema') }}.seed" + columns: + - name: id + description: "An ID field" diff --git a/test/integration/029_docs_generate_tests/test_docs_generate.py b/test/integration/029_docs_generate_tests/test_docs_generate.py index 10cc512d077..7d848267f03 100644 --- a/test/integration/029_docs_generate_tests/test_docs_generate.py +++ b/test/integration/029_docs_generate_tests/test_docs_generate.py @@ -95,12 +95,14 @@ def run_and_generate(self, extra=None, seed_count=1, model_count=1, alternate_db project.update(extra) self.use_default_project(project) - self.assertEqual(len(self.run_dbt(["seed"])), seed_count) - self.assertEqual(len(self.run_dbt()), model_count) + vars_arg = '--vars={{test_schema: {}}}'.format(self.unique_schema()) + + self.assertEqual(len(self.run_dbt(["seed", vars_arg])), seed_count) + self.assertEqual(len(self.run_dbt(['run', vars_arg])), model_count) os.remove(_normalize('target/manifest.json')) os.remove(_normalize('target/run_results.json')) self.generate_start_time = datetime.utcnow() - self.run_dbt(['docs', 'generate']) + self.run_dbt(['docs', 'generate', vars_arg]) def assertBetween(self, timestr, start, end=None): if end is None: @@ -820,6 +822,7 @@ def expected_seeded_manifest(self, model_database=None): 'package_name': 'test', 'raw_sql': _read_file(model_sql_path).rstrip('\n'), 'refs': [['seed']], + 'sources': [], 'depends_on': {'nodes': ['seed.test.seed'], 'macros': []}, 'unique_id': 'model.test.model', 'empty': False, @@ -865,6 +868,7 @@ def expected_seeded_manifest(self, model_database=None): 'original_file_path': self.dir(os.path.join('seed', 'seed.csv')), 'refs': [], + 'sources': [], 'depends_on': {'nodes': [], 'macros': []}, 'unique_id': 'seed.test.seed', 'empty': False, @@ -900,6 +904,7 @@ def expected_seeded_manifest(self, model_database=None): 'vars': config_vars, 'tags': [], }, + 'sources': [], 'depends_on': {'macros': [], 'nodes': ['model.test.model']}, 'description': '', 'empty': False, @@ -930,6 +935,7 @@ def expected_seeded_manifest(self, model_database=None): 'vars': config_vars, 'tags': [], }, + 'sources': [], 'depends_on': {'macros': [], 'nodes': ['model.test.model']}, 'description': '', 'empty': False, @@ -938,7 +944,7 @@ def expected_seeded_manifest(self, model_database=None): 'original_file_path': schema_yml_path, 'package_name': 'test', 'path': _normalize('schema_test/nothing_model_.sql'), - 'raw_sql': "{{ test_nothing(model=ref('model'), ) }}", + 'raw_sql': "{{ test.test_nothing(model=ref('model'), ) }}", 'refs': [['model']], 'resource_type': 'test', 'root_path': os.getcwd(), @@ -961,6 +967,7 @@ def expected_seeded_manifest(self, model_database=None): 'vars': config_vars, 'tags': [], }, + 'sources': [], 'depends_on': {'macros': [], 'nodes': ['model.test.model']}, 'description': '', 'empty': False, @@ -1030,7 +1037,11 @@ def expected_postgres_references_manifest(self, model_database=None): 'vars': config_vars, 'tags': [], }, - 'depends_on': {'macros': [], 'nodes': []}, + 'sources': [['my_source', 'my_table']], + 'depends_on': { + 'macros': [], + 'nodes': ['source.test.my_source.my_table'] + }, 'description': '', 'empty': False, 'fqn': ['test', 'ephemeral_copy'], @@ -1040,7 +1051,7 @@ def expected_postgres_references_manifest(self, model_database=None): 'path': 'ephemeral_copy.sql', 'raw_sql': ( '{{\n config(\n materialized = "ephemeral"\n )\n}}' - '\n\nselect * from {{ this.schema }}.seed' + '\n\nselect * from {{ source("my_source", "my_table") }}' ), 'refs': [], 'resource_type': 'model', @@ -1072,6 +1083,7 @@ def expected_postgres_references_manifest(self, model_database=None): 'vars': config_vars, 'tags': [], }, + 'sources': [], 'depends_on': { 'macros': [], 'nodes': ['model.test.ephemeral_copy'] @@ -1136,6 +1148,7 @@ def expected_postgres_references_manifest(self, model_database=None): 'vars': config_vars, 'tags': [], }, + 'sources': [], 'depends_on': { 'macros': [], 'nodes': ['model.test.ephemeral_summary'] @@ -1190,6 +1203,7 @@ def expected_postgres_references_manifest(self, model_database=None): 'vars': {}, 'tags': [], }, + 'sources': [], 'depends_on': {'macros': [], 'nodes': []}, 'description': '', 'empty': False, @@ -1207,6 +1221,38 @@ def expected_postgres_references_manifest(self, model_database=None): 'tags': [], 'unique_id': 'seed.test.seed' }, + 'source.test.my_source.my_table': { + 'columns': { + 'id': { + 'description': 'An ID field', + 'name': 'id' + } + }, + 'description': 'My table', + 'docrefs': [ + { + 'documentation_name': 'table_info', + 'documentation_package': '' + }, + { + 'documentation_name': 'source_info', + 'documentation_package': '' + } + ], + 'freshness': {}, + 'loaded_at_field': None, + 'loader': 'a_loader', + 'name': 'my_table', + 'original_file_path': self.dir('ref_models/schema.yml'), + 'package_name': 'test', + 'path': self.dir('ref_models/schema.yml'), + 'resource_type': 'source', + 'root_path': os.getcwd(), + 'source_description': "{{ doc('source_info') }}", + 'source_name': 'my_source', + 'sql_table_name': '{}.seed'.format(my_schema_name), + 'unique_id': 'source.test.my_source.my_table' + } }, 'docs': { 'dbt.__overview__': ANY, @@ -1223,6 +1269,17 @@ def expected_postgres_references_manifest(self, model_database=None): 'root_path': os.getcwd(), 'unique_id': 'test.ephemeral_summary' }, + 'test.source_info': { + 'block_contents': 'My source', + 'file_contents': docs_file, + 'name': 'source_info', + 'original_file_path': docs_path, + 'package_name': 'test', + 'path': 'docs.md', + 'resource_type': 'documentation', + 'root_path': os.getcwd(), + 'unique_id': 'test.source_info', + }, 'test.summary_count': { 'block_contents': 'The number of instances of the first name', 'file_contents': docs_file, @@ -1245,6 +1302,17 @@ def expected_postgres_references_manifest(self, model_database=None): 'root_path': os.getcwd(), 'unique_id': 'test.summary_first_name' }, + 'test.table_info': { + 'block_contents': 'My table', + 'file_contents': docs_file, + 'name': 'table_info', + 'original_file_path': docs_path, + 'package_name': 'test', + 'path': 'docs.md', + 'resource_type': 'documentation', + 'root_path': os.getcwd(), + 'unique_id': 'test.table_info' + }, 'test.view_summary': { 'block_contents': ( 'A view of the summary of the ephemeral copy of the ' @@ -1265,12 +1333,14 @@ def expected_postgres_references_manifest(self, model_database=None): 'model.test.ephemeral_summary': ['model.test.view_summary'], 'model.test.view_summary': [], 'seed.test.seed': [], + 'source.test.my_source.my_table': ['model.test.ephemeral_copy'], }, 'parent_map': { - 'model.test.ephemeral_copy': [], + 'model.test.ephemeral_copy': ['source.test.my_source.my_table'], 'model.test.ephemeral_summary': ['model.test.ephemeral_copy'], 'model.test.view_summary': ['model.test.ephemeral_summary'], 'seed.test.seed': [], + 'source.test.my_source.my_table': [], }, 'metadata': { 'project_id': '098f6bcd4621d373cade4e832627b4f6', @@ -1303,6 +1373,7 @@ def expected_bigquery_complex_manifest(self): 'vars': config_vars, 'tags': [], }, + 'sources': [], 'depends_on': {'macros': [], 'nodes': ['seed.test.seed']}, 'empty': False, 'fqn': ['test', 'clustered'], @@ -1358,6 +1429,7 @@ def expected_bigquery_complex_manifest(self): 'tags': [], 'vars': config_vars }, + 'sources': [], 'depends_on': {'macros': [], 'nodes': ['seed.test.seed']}, 'empty': False, 'fqn': ['test', 'multi_clustered'], @@ -1411,6 +1483,7 @@ def expected_bigquery_complex_manifest(self): 'vars': config_vars, 'tags': [], }, + 'sources': [], 'depends_on': { 'macros': [], 'nodes': ['model.test.nested_table'] @@ -1467,6 +1540,7 @@ def expected_bigquery_complex_manifest(self): 'vars': config_vars, 'tags': [], }, + 'sources': [], 'depends_on': { 'macros': [], 'nodes': [] @@ -1497,6 +1571,7 @@ def expected_bigquery_complex_manifest(self): 'package_name': 'test', 'original_file_path': self.dir('seed/seed.csv'), 'refs': [], + 'sources': [], 'depends_on': { 'nodes': [], 'macros': [], @@ -1562,6 +1637,7 @@ def expected_redshift_incremental_view_manifest(self): "package_name": "test", "raw_sql": _read_file(model_sql_path).rstrip('\n'), "refs": [["seed"]], + "sources": [], "depends_on": { "nodes": ["seed.test.seed"], "macros": [], @@ -1619,6 +1695,7 @@ def expected_redshift_incremental_view_manifest(self): "package_name": "test", "original_file_path": self.dir("seed/seed.csv"), "refs": [], + "sources": [], "depends_on": { "nodes": [], "macros": [], @@ -1753,6 +1830,7 @@ def expected_run_results(self, quote_schema=True, quote_model=False, 'compiled': True, 'compiled_sql': compiled_sql, 'config': model_config, + 'sources': [], 'depends_on': { 'macros': [], 'nodes': ['seed.test.seed'] @@ -1804,6 +1882,7 @@ def expected_run_results(self, quote_schema=True, quote_model=False, 'vars': {}, 'tags': [], }, + 'sources': [], 'depends_on': {'macros': [], 'nodes': []}, 'description': '', 'empty': False, @@ -1849,6 +1928,7 @@ def expected_run_results(self, quote_schema=True, quote_model=False, 'vars': config_vars, 'tags': [], }, + 'sources': [], 'depends_on': {'macros': [], 'nodes': ['model.test.model']}, 'description': '', 'empty': False, @@ -1893,6 +1973,7 @@ def expected_run_results(self, quote_schema=True, quote_model=False, 'vars': config_vars, 'tags': [], }, + 'sources': [], 'depends_on': {'macros': [], 'nodes': ['model.test.model']}, 'description': '', 'empty': False, @@ -1904,7 +1985,7 @@ def expected_run_results(self, quote_schema=True, quote_model=False, 'original_file_path': schema_yml_path, 'package_name': 'test', 'path': _normalize('schema_test/nothing_model_.sql'), - 'raw_sql': "{{ test_nothing(model=ref('model'), ) }}", + 'raw_sql': "{{ test.test_nothing(model=ref('model'), ) }}", 'refs': [['model']], 'resource_type': 'test', 'root_path': os.getcwd(), @@ -1938,6 +2019,7 @@ def expected_run_results(self, quote_schema=True, quote_model=False, 'vars': config_vars, 'tags': [], }, + 'sources': [], 'depends_on': {'macros': [], 'nodes': ['model.test.model']}, 'description': '', 'empty': False, @@ -2020,6 +2102,7 @@ def expected_postgres_references_run_results(self): 'quoting': {}, 'tags': [], }, + 'sources': [], 'depends_on': { 'nodes': ['model.test.ephemeral_copy'], 'macros': [] @@ -2105,6 +2188,7 @@ def expected_postgres_references_run_results(self): 'quoting': {}, 'tags': [], }, + 'sources': [], 'depends_on': { 'nodes': ['model.test.ephemeral_summary'], 'macros': [] @@ -2178,6 +2262,7 @@ def expected_postgres_references_run_results(self): 'vars': {}, 'tags': [], }, + 'sources': [], 'depends_on': {'macros': [], 'nodes': []}, 'description': '', 'empty': False, diff --git a/test/integration/033_event_tracking_test/models/schema.yml b/test/integration/033_event_tracking_test/models/schema.yml index 2cdc6c0f498..5ac3436dc22 100644 --- a/test/integration/033_event_tracking_test/models/schema.yml +++ b/test/integration/033_event_tracking_test/models/schema.yml @@ -1,13 +1,12 @@ - -example: - constraints: - unique: - - id - - -# This fails -example_2: - constraints: - unique: - - id - +version: 2 +models: +- name: example + columns: + - name: id + tests: + - unique +- name: example_2 + columns: + - name: id + tests: + - unique diff --git a/test/integration/042_sources_test/models/descendant_model.sql b/test/integration/042_sources_test/models/descendant_model.sql new file mode 100644 index 00000000000..55bbcba67b4 --- /dev/null +++ b/test/integration/042_sources_test/models/descendant_model.sql @@ -0,0 +1 @@ +select * from {{ source('test_source', 'test_table') }} diff --git a/test/integration/042_sources_test/models/schema.yml b/test/integration/042_sources_test/models/schema.yml new file mode 100644 index 00000000000..e9de2d06ef1 --- /dev/null +++ b/test/integration/042_sources_test/models/schema.yml @@ -0,0 +1,39 @@ +version: 2 +models: + - name: descendant_model + columns: + - name: favorite_color + tests: + - relationships: + to: source('test_source', 'test_table') + field: favorite_color +sources: + - name: test_source + loader: custom + tables: + - name: test_table + # I don't know what would compel you to do this, but the important thing is that you can + sql_table_name: "{{ var(env_var('DBT_TEST_SCHEMA_NAME_VARIABLE')) }}.source" + columns: + - name: favorite_color + description: The favorite color + - name: id + description: The user ID + tests: + - unique + - not_null + - name: first_name + description: The first name of the user + tests: [] + - name: email + description: The email address of the user + - name: ip_address + description: The last IP address the user logged in from + - name: updated_at + description: The last update time for this user + tests: + - relationships: + # do this as a table-level test, just to test out that aspect + column_name: favorite_color + to: ref('descendant_model') + field: favorite_color diff --git a/test/integration/042_sources_test/source.sql b/test/integration/042_sources_test/source.sql new file mode 100644 index 00000000000..6e3fba91226 --- /dev/null +++ b/test/integration/042_sources_test/source.sql @@ -0,0 +1,113 @@ +create table {schema}.source ( + favorite_color VARCHAR(10), + id INTEGER, + first_name VARCHAR(11), + email VARCHAR(31), + ip_address VARCHAR(15), + updated_at TIMESTAMP WITHOUT TIME ZONE +); + + +INSERT INTO {schema}.source + (favorite_color, id, first_name, email, ip_address, updated_at) +VALUES + ('blue', 1,'Larry','lking0@miitbeian.gov.cn','69.135.206.194','2008-09-12 19:08:31'), + ('blue', 2,'Larry','lperkins1@toplist.cz','64.210.133.162','1978-05-09 04:15:14'), + ('blue', 3,'Anna','amontgomery2@miitbeian.gov.cn','168.104.64.114','2011-10-16 04:07:57'), + ('blue', 4,'Sandra','sgeorge3@livejournal.com','229.235.252.98','1973-07-19 10:52:43'), + ('blue', 5,'Fred','fwoods4@google.cn','78.229.170.124','2012-09-30 16:38:29'), + ('blue', 6,'Stephen','shanson5@livejournal.com','182.227.157.105','1995-11-07 21:40:50'), + ('blue', 7,'William','wmartinez6@upenn.edu','135.139.249.50','1982-09-05 03:11:59'), + ('blue', 8,'Jessica','jlong7@hao123.com','203.62.178.210','1991-10-16 11:03:15'), + ('blue', 9,'Douglas','dwhite8@tamu.edu','178.187.247.1','1979-10-01 09:49:48'), + ('blue', 10,'Lisa','lcoleman9@nydailynews.com','168.234.128.249','2011-05-26 07:45:49'), + ('blue', 11,'Ralph','rfieldsa@home.pl','55.152.163.149','1972-11-18 19:06:11'), + ('blue', 12,'Louise','lnicholsb@samsung.com','141.116.153.154','2014-11-25 20:56:14'), + ('blue', 13,'Clarence','cduncanc@sfgate.com','81.171.31.133','2011-11-17 07:02:36'), + ('blue', 14,'Daniel','dfranklind@omniture.com','8.204.211.37','1980-09-13 00:09:04'), + ('blue', 15,'Katherine','klanee@auda.org.au','176.96.134.59','1997-08-22 19:36:56'), + ('blue', 16,'Billy','bwardf@wikia.com','214.108.78.85','2003-10-19 02:14:47'), + ('blue', 17,'Annie','agarzag@ocn.ne.jp','190.108.42.70','1988-10-28 15:12:35'), + ('blue', 18,'Shirley','scolemanh@fastcompany.com','109.251.164.84','1988-08-24 10:50:57'), + ('blue', 19,'Roger','rfrazieri@scribd.com','38.145.218.108','1985-12-31 15:17:15'), + ('blue', 20,'Lillian','lstanleyj@goodreads.com','47.57.236.17','1970-06-08 02:09:05'), + ('blue', 21,'Aaron','arodriguezk@nps.gov','205.245.118.221','1985-10-11 23:07:49'), + ('blue', 22,'Patrick','pparkerl@techcrunch.com','19.8.100.182','2006-03-29 12:53:56'), + ('blue', 23,'Phillip','pmorenom@intel.com','41.38.254.103','2011-11-07 15:35:43'), + ('blue', 24,'Henry','hgarcian@newsvine.com','1.191.216.252','2008-08-28 08:30:44'), + ('blue', 25,'Irene','iturnero@opera.com','50.17.60.190','1994-04-01 07:15:02'), + ('blue', 26,'Andrew','adunnp@pen.io','123.52.253.176','2000-11-01 06:03:25'), + ('blue', 27,'David','dgutierrezq@wp.com','238.23.203.42','1988-01-25 07:29:18'), + ('blue', 28,'Henry','hsanchezr@cyberchimps.com','248.102.2.185','1983-01-01 13:36:37'), + ('blue', 29,'Evelyn','epetersons@gizmodo.com','32.80.46.119','1979-07-16 17:24:12'), + ('blue', 30,'Tammy','tmitchellt@purevolume.com','249.246.167.88','2001-04-03 10:00:23'), + ('blue', 31,'Jacqueline','jlittleu@domainmarket.com','127.181.97.47','1986-02-11 21:35:50'), + ('blue', 32,'Earl','eortizv@opera.com','166.47.248.240','1996-07-06 08:16:27'), + ('blue', 33,'Juan','jgordonw@sciencedirect.com','71.77.2.200','1987-01-31 03:46:44'), + ('blue', 34,'Diane','dhowellx@nyu.edu','140.94.133.12','1994-06-11 02:30:05'), + ('blue', 35,'Randy','rkennedyy@microsoft.com','73.255.34.196','2005-05-26 20:28:39'), + ('blue', 36,'Janice','jriveraz@time.com','22.214.227.32','1990-02-09 04:16:52'), + ('blue', 37,'Laura','lperry10@diigo.com','159.148.145.73','2015-03-17 05:59:25'), + ('blue', 38,'Gary','gray11@statcounter.com','40.193.124.56','1970-01-27 10:04:51'), + ('blue', 39,'Jesse','jmcdonald12@typepad.com','31.7.86.103','2009-03-14 08:14:29'), + ('blue', 40,'Sandra','sgonzalez13@goodreads.com','223.80.168.239','1993-05-21 14:08:54'), + ('blue', 41,'Scott','smoore14@archive.org','38.238.46.83','1980-08-30 11:16:56'), + ('blue', 42,'Phillip','pevans15@cisco.com','158.234.59.34','2011-12-15 23:26:31'), + ('blue', 43,'Steven','sriley16@google.ca','90.247.57.68','2011-10-29 19:03:28'), + ('blue', 44,'Deborah','dbrown17@hexun.com','179.125.143.240','1995-04-10 14:36:07'), + ('blue', 45,'Lori','lross18@ow.ly','64.80.162.180','1980-12-27 16:49:15'), + ('blue', 46,'Sean','sjackson19@tumblr.com','240.116.183.69','1988-06-12 21:24:45'), + ('blue', 47,'Terry','tbarnes1a@163.com','118.38.213.137','1997-09-22 16:43:19'), + ('blue', 48,'Dorothy','dross1b@ebay.com','116.81.76.49','2005-02-28 13:33:24'), + ('blue', 49,'Samuel','swashington1c@house.gov','38.191.253.40','1989-01-19 21:15:48'), + ('blue', 50,'Ralph','rcarter1d@tinyurl.com','104.84.60.174','2007-08-11 10:21:49'), + ('green', 51,'Wayne','whudson1e@princeton.edu','90.61.24.102','1983-07-03 16:58:12'), + ('green', 52,'Rose','rjames1f@plala.or.jp','240.83.81.10','1995-06-08 11:46:23'), + ('green', 53,'Louise','lcox1g@theglobeandmail.com','105.11.82.145','2016-09-19 14:45:51'), + ('green', 54,'Kenneth','kjohnson1h@independent.co.uk','139.5.45.94','1976-08-17 11:26:19'), + ('green', 55,'Donna','dbrown1i@amazon.co.uk','19.45.169.45','2006-05-27 16:51:40'), + ('green', 56,'Johnny','jvasquez1j@trellian.com','118.202.238.23','1975-11-17 08:42:32'), + ('green', 57,'Patrick','pramirez1k@tamu.edu','231.25.153.198','1997-08-06 11:51:09'), + ('green', 58,'Helen','hlarson1l@prweb.com','8.40.21.39','1993-08-04 19:53:40'), + ('green', 59,'Patricia','pspencer1m@gmpg.org','212.198.40.15','1977-08-03 16:37:27'), + ('green', 60,'Joseph','jspencer1n@marriott.com','13.15.63.238','2005-07-23 20:22:06'), + ('green', 61,'Phillip','pschmidt1o@blogtalkradio.com','177.98.201.190','1976-05-19 21:47:44'), + ('green', 62,'Joan','jwebb1p@google.ru','105.229.170.71','1972-09-07 17:53:47'), + ('green', 63,'Phyllis','pkennedy1q@imgur.com','35.145.8.244','2000-01-01 22:33:37'), + ('green', 64,'Katherine','khunter1r@smh.com.au','248.168.205.32','1991-01-09 06:40:24'), + ('green', 65,'Laura','lvasquez1s@wiley.com','128.129.115.152','1997-10-23 12:04:56'), + ('green', 66,'Juan','jdunn1t@state.gov','44.228.124.51','2004-11-10 05:07:35'), + ('green', 67,'Judith','jholmes1u@wiley.com','40.227.179.115','1977-08-02 17:01:45'), + ('green', 68,'Beverly','bbaker1v@wufoo.com','208.34.84.59','2016-03-06 20:07:23'), + ('green', 69,'Lawrence','lcarr1w@flickr.com','59.158.212.223','1988-09-13 06:07:21'), + ('green', 70,'Gloria','gwilliams1x@mtv.com','245.231.88.33','1995-03-18 22:32:46'), + ('green', 71,'Steven','ssims1y@cbslocal.com','104.50.58.255','2001-08-05 21:26:20'), + ('green', 72,'Betty','bmills1z@arstechnica.com','103.177.214.220','1981-12-14 21:26:54'), + ('green', 73,'Mildred','mfuller20@prnewswire.com','151.158.8.130','2000-04-19 10:13:55'), + ('green', 74,'Donald','dday21@icq.com','9.178.102.255','1972-12-03 00:58:24'), + ('green', 75,'Eric','ethomas22@addtoany.com','85.2.241.227','1992-11-01 05:59:30'), + ('green', 76,'Joyce','jarmstrong23@sitemeter.com','169.224.20.36','1985-10-24 06:50:01'), + ('green', 77,'Maria','mmartinez24@amazonaws.com','143.189.167.135','2005-10-05 05:17:42'), + ('green', 78,'Harry','hburton25@youtube.com','156.47.176.237','1978-03-26 05:53:33'), + ('green', 79,'Kevin','klawrence26@hao123.com','79.136.183.83','1994-10-12 04:38:52'), + ('green', 80,'David','dhall27@prweb.com','133.149.172.153','1976-12-15 16:24:24'), + ('green', 81,'Kathy','kperry28@twitter.com','229.242.72.228','1979-03-04 02:58:56'), + ('green', 82,'Adam','aprice29@elegantthemes.com','13.145.21.10','1982-11-07 11:46:59'), + ('green', 83,'Brandon','bgriffin2a@va.gov','73.249.128.212','2013-10-30 05:30:36'), + ('green', 84,'Henry','hnguyen2b@discovery.com','211.36.214.242','1985-01-09 06:37:27'), + ('green', 85,'Eric','esanchez2c@edublogs.org','191.166.188.251','2004-05-01 23:21:42'), + ('green', 86,'Jason','jlee2d@jimdo.com','193.92.16.182','1973-01-08 09:05:39'), + ('green', 87,'Diana','drichards2e@istockphoto.com','19.130.175.245','1994-10-05 22:50:49'), + ('green', 88,'Andrea','awelch2f@abc.net.au','94.155.233.96','2002-04-26 08:41:44'), + ('green', 89,'Louis','lwagner2g@miitbeian.gov.cn','26.217.34.111','2003-08-25 07:56:39'), + ('green', 90,'Jane','jsims2h@seesaa.net','43.4.220.135','1987-03-20 20:39:04'), + ('green', 91,'Larry','lgrant2i@si.edu','97.126.79.34','2000-09-07 20:26:19'), + ('green', 92,'Louis','ldean2j@prnewswire.com','37.148.40.127','2011-09-16 20:12:14'), + ('green', 93,'Jennifer','jcampbell2k@xing.com','38.106.254.142','1988-07-15 05:06:49'), + ('green', 94,'Wayne','wcunningham2l@google.com.hk','223.28.26.187','2009-12-15 06:16:54'), + ('green', 95,'Lori','lstevens2m@icq.com','181.250.181.58','1984-10-28 03:29:19'), + ('green', 96,'Judy','jsimpson2n@marriott.com','180.121.239.219','1986-02-07 15:18:10'), + ('green', 97,'Phillip','phoward2o@usa.gov','255.247.0.175','2002-12-26 08:44:45'), + ('green', 98,'Gloria','gwalker2p@usa.gov','156.140.7.128','1997-10-04 07:58:58'), + ('green', 99,'Paul','pjohnson2q@umn.edu','183.59.198.197','1991-11-14 12:33:55'), + ('green', 100,'Frank','fgreene2r@blogspot.com','150.143.68.121','2010-06-12 23:55:39'); diff --git a/test/integration/042_sources_test/test_sources.py b/test/integration/042_sources_test/test_sources.py new file mode 100644 index 00000000000..9dbe924274f --- /dev/null +++ b/test/integration/042_sources_test/test_sources.py @@ -0,0 +1,31 @@ +from nose.plugins.attrib import attr +from test.integration.base import DBTIntegrationTest, use_profile +import os + +class TestSources(DBTIntegrationTest): + @property + def schema(self): + return "sources_042" + + @property + def models(self): + return "test/integration/042_sources_test/models" + + @use_profile('postgres') + def test_basic_source_def(self): + os.environ['DBT_TEST_SCHEMA_NAME_VARIABLE'] = 'test_run_schema' + self.run_sql_file('test/integration/042_sources_test/source.sql', + kwargs={'schema':self.unique_schema()}) + self.run_dbt([ + 'run', + '--vars', + '{{test_run_schema: {}}}'.format(self.unique_schema()) + ]) + self.assertTablesEqual('source', 'descendant_model') + results = self.run_dbt([ + 'test', + '--vars', + '{{test_run_schema: {}}}'.format(self.unique_schema()) + ]) + self.assertEqual(len(results), 4) + del os.environ['DBT_TEST_SCHEMA_NAME_VARIABLE'] diff --git a/test/unit/test_compiler.py b/test/unit/test_compiler.py index 2bfad06a548..bc8ca7ac914 100644 --- a/test/unit/test_compiler.py +++ b/test/unit/test_compiler.py @@ -63,6 +63,7 @@ def test__prepend_ctes__already_has_cte(self): package_name='root', root_path='/usr/src/app', refs=[], + sources=[], depends_on={ 'nodes': [ 'model.root.ephemeral' @@ -96,6 +97,7 @@ def test__prepend_ctes__already_has_cte(self): package_name='root', root_path='/usr/src/app', refs=[], + sources=[], depends_on={ 'nodes': [], 'macros': [] @@ -152,6 +154,7 @@ def test__prepend_ctes__no_ctes(self): package_name='root', root_path='/usr/src/app', refs=[], + sources=[], depends_on={ 'nodes': [], 'macros': [] @@ -181,6 +184,7 @@ def test__prepend_ctes__no_ctes(self): package_name='root', root_path='/usr/src/app', refs=[], + sources=[], depends_on={ 'nodes': [], 'macros': [] @@ -249,6 +253,7 @@ def test__prepend_ctes(self): package_name='root', root_path='/usr/src/app', refs=[], + sources=[], depends_on={ 'nodes': [ 'model.root.ephemeral' @@ -280,6 +285,7 @@ def test__prepend_ctes(self): package_name='root', root_path='/usr/src/app', refs=[], + sources=[], depends_on={ 'nodes': [], 'macros': [] @@ -342,6 +348,7 @@ def test__prepend_ctes__multiple_levels(self): package_name='root', root_path='/usr/src/app', refs=[], + sources=[], depends_on={ 'nodes': [ 'model.root.ephemeral' @@ -373,6 +380,7 @@ def test__prepend_ctes__multiple_levels(self): package_name='root', root_path='/usr/src/app', refs=[], + sources=[], depends_on={ 'nodes': [], 'macros': [] @@ -402,6 +410,7 @@ def test__prepend_ctes__multiple_levels(self): package_name='root', root_path='/usr/src/app', refs=[], + sources=[], depends_on={ 'nodes': [], 'macros': [] diff --git a/test/unit/test_context.py b/test/unit/test_context.py index 5b50f4506c7..c5d9a5c99f4 100644 --- a/test/unit/test_context.py +++ b/test/unit/test_context.py @@ -20,6 +20,7 @@ def setUp(self): original_file_path='model_one.sql', root_path='/usr/src/app', refs=[], + sources=[], depends_on={ 'nodes': [], 'macros': [] diff --git a/test/unit/test_manifest.py b/test/unit/test_manifest.py index 3fe4d75bcfc..cbd988c7474 100644 --- a/test/unit/test_manifest.py +++ b/test/unit/test_manifest.py @@ -41,6 +41,7 @@ def setUp(self): empty=False, package_name='snowplow', refs=[], + sources=[], depends_on={ 'nodes': [], 'macros': [] @@ -63,6 +64,7 @@ def setUp(self): empty=False, package_name='root', refs=[], + sources=[], depends_on={ 'nodes': [], 'macros': [] @@ -85,6 +87,7 @@ def setUp(self): empty=False, package_name='root', refs=[['events']], + sources=[], depends_on={ 'nodes': ['model.root.events'], 'macros': [] @@ -107,6 +110,7 @@ def setUp(self): empty=False, package_name='root', refs=[['events']], + sources=[], depends_on={ 'nodes': ['model.root.dep'], 'macros': [] @@ -129,6 +133,7 @@ def setUp(self): empty=False, package_name='root', refs=[['events']], + sources=[], depends_on={ 'nodes': ['model.root.events'], 'macros': [] @@ -151,6 +156,7 @@ def setUp(self): empty=False, package_name='root', refs=[['events']], + sources=[], depends_on={ 'nodes': ['model.root.nested', 'model.root.sibling'], 'macros': [] @@ -332,6 +338,7 @@ def test_get_resource_fqns(self): empty=False, package_name='root', refs=[['events']], + sources=[], depends_on={ 'nodes': [], 'macros': [] @@ -389,6 +396,7 @@ def setUp(self): empty=False, package_name='snowplow', refs=[], + sources=[], depends_on={ 'nodes': [], 'macros': [] @@ -416,6 +424,7 @@ def setUp(self): empty=False, package_name='root', refs=[], + sources=[], depends_on={ 'nodes': [], 'macros': [] @@ -443,6 +452,7 @@ def setUp(self): empty=False, package_name='root', refs=[['events']], + sources=[], depends_on={ 'nodes': ['model.root.events'], 'macros': [] @@ -465,6 +475,7 @@ def setUp(self): empty=False, package_name='root', refs=[['events']], + sources=[], depends_on={ 'nodes': ['model.root.dep'], 'macros': [] @@ -487,6 +498,7 @@ def setUp(self): empty=False, package_name='root', refs=[['events']], + sources=[], depends_on={ 'nodes': ['model.root.events'], 'macros': [] @@ -509,6 +521,7 @@ def setUp(self): empty=False, package_name='root', refs=[['events']], + sources=[], depends_on={ 'nodes': ['model.root.nested', 'model.root.sibling'], 'macros': [] diff --git a/test/unit/test_parser.py b/test/unit/test_parser.py index 60e1376b2bb..00c9d62a2b0 100644 --- a/test/unit/test_parser.py +++ b/test/unit/test_parser.py @@ -13,7 +13,8 @@ from dbt.node_types import NodeType from dbt.contracts.graph.manifest import Manifest -from dbt.contracts.graph.parsed import ParsedNode, ParsedMacro, ParsedNodePatch +from dbt.contracts.graph.parsed import ParsedNode, ParsedMacro, \ + ParsedNodePatch, ParsedSourceDefinition from dbt.contracts.graph.unparsed import UnparsedNode from .utils import config_from_parts_or_dicts @@ -55,7 +56,8 @@ def setUp(self): self.root_project_config = config_from_parts_or_dicts( project=root_project, - profile=profile_data + profile=profile_data, + cli_vars='{"test_schema_name": "foo"}' ) snowplow_project = { @@ -138,15 +140,13 @@ def test_source_config_all_keys_accounted_for(self): self.assertEqual(used_keys, frozenset(SourceConfig.ConfigKeys)) -class ParserTest(BaseParserTest): - - def find_input_by_name(self, models, name): - return next( - (model for model in models if model.get('name') == name), - {}) +class SchemaParserTest(BaseParserTest): + maxDiff = None def setUp(self): - super(ParserTest, self).setUp() + super(SchemaParserTest, self).setUp() + self.maxDiff = None + self.macro_manifest = Manifest(macros={}, nodes={}, docs={}, generated_at=timestring(), disabled=[]) @@ -173,53 +173,655 @@ def setUp(self): 'tags': [], } + self._expected_source = ParsedSourceDefinition( + unique_id='source.root.my_source.my_table', + name='my_table', + description='my table description', + source_name='my_source', + source_description='my source description', + loader='some_loader', + package_name='root', + root_path=get_os_path('/usr/src/app'), + path='test_one.yml', + original_file_path='test_one.yml', + columns={ + 'id': { + 'name': 'id', + 'description': 'user ID', + }, + }, + docrefs=[], + freshness={ + 'warn_after': { + 'count': 7, + 'period': 'hour' + }, + 'error_after': { + 'count': 20, + 'period': 'hour' + }, + }, + loaded_at_field='something', + sql_table_name='foo.bar', + resource_type='source' + ) - def test__single_model(self): - models = [{ - 'name': 'model_one', - 'resource_type': 'model', - 'package_name': 'root', - 'original_file_path': 'model_one.sql', - 'root_path': get_os_path('/usr/src/app'), - 'path': 'model_one.sql', - 'raw_sql': ("select * from events"), - }] - parser = ModelParser( + self._expected_source_tests = [ + ParsedNode( + alias='source_accepted_values_my_source_my_table_id__a__b', + name='source_accepted_values_my_source_my_table_id__a__b', + database='test', + schema='analytics', + resource_type='test', + unique_id='test.root.source_accepted_values_my_source_my_table_id__a__b', + fqn=['root', 'schema_test', + 'source_accepted_values_my_source_my_table_id__a__b'], + empty=False, + package_name='root', + original_file_path='test_one.yml', + root_path=get_os_path('/usr/src/app'), + refs=[], + sources=[['my_source', 'my_table']], + depends_on={'nodes': [], 'macros': []}, + config=self.model_config, + path=get_os_path( + 'schema_test/source_accepted_values_my_source_my_table_id__a__b.sql'), + tags=['schema'], + raw_sql="{{ test_accepted_values(model=source('my_source', 'my_table'), column_name='id', values=['a', 'b']) }}", + description='', + columns={}, + column_name='id' + ), + ParsedNode( + alias='source_not_null_my_source_my_table_id', + name='source_not_null_my_source_my_table_id', + database='test', + schema='analytics', + resource_type='test', + unique_id='test.root.source_not_null_my_source_my_table_id', + fqn=['root', 'schema_test', 'source_not_null_my_source_my_table_id'], + empty=False, + package_name='root', + root_path=get_os_path('/usr/src/app'), + refs=[], + sources=[['my_source', 'my_table']], + depends_on={'nodes': [], 'macros': []}, + config=self.model_config, + original_file_path='test_one.yml', + path=get_os_path('schema_test/source_not_null_my_source_my_table_id.sql'), + tags=['schema'], + raw_sql="{{ test_not_null(model=source('my_source', 'my_table'), column_name='id') }}", + description='', + columns={}, + column_name='id' + ), + ParsedNode( + alias='source_relationships_my_source_my_table_id__id__ref_model_two_', + name='source_relationships_my_source_my_table_id__id__ref_model_two_', + database='test', + schema='analytics', + resource_type='test', + unique_id='test.root.source_relationships_my_source_my_table_id__id__ref_model_two_', # noqa + fqn=['root', 'schema_test', + 'source_relationships_my_source_my_table_id__id__ref_model_two_'], + empty=False, + package_name='root', + original_file_path='test_one.yml', + root_path=get_os_path('/usr/src/app'), + refs=[['model_two']], + sources=[['my_source', 'my_table']], + depends_on={'nodes': [], 'macros': []}, + config=self.model_config, + path=get_os_path('schema_test/source_relationships_my_source_my_table_id__id__ref_model_two_.sql'), # noqa + tags=['schema'], + raw_sql="{{ test_relationships(model=source('my_source', 'my_table'), column_name='id', from='id', to=ref('model_two')) }}", + description='', + columns={}, + column_name='id' + ), + ParsedNode( + alias='source_some_test_my_source_my_table_value', + name='source_some_test_my_source_my_table_value', + database='test', + schema='analytics', + resource_type='test', + unique_id='test.root.source_some_test_my_source_my_table_value', + fqn=['root', 'schema_test', 'source_some_test_my_source_my_table_value'], + empty=False, + package_name='root', + original_file_path='test_one.yml', + root_path=get_os_path('/usr/src/app'), + refs=[], + sources=[['my_source', 'my_table']], + depends_on={'nodes': [], 'macros': []}, + config=self.model_config, + path=get_os_path('schema_test/source_some_test_my_source_my_table_value.sql'), + tags=['schema'], + raw_sql="{{ test_some_test(model=source('my_source', 'my_table'), key='value') }}", + description='', + columns={} + ), + ParsedNode( + alias='source_unique_my_source_my_table_id', + name='source_unique_my_source_my_table_id', + database='test', + schema='analytics', + resource_type='test', + unique_id='test.root.source_unique_my_source_my_table_id', + fqn=['root', 'schema_test', 'source_unique_my_source_my_table_id'], + empty=False, + package_name='root', + root_path=get_os_path('/usr/src/app'), + refs=[], + sources=[['my_source', 'my_table']], + depends_on={'nodes': [], 'macros': []}, + config=self.model_config, + original_file_path='test_one.yml', + path=get_os_path('schema_test/source_unique_my_source_my_table_id.sql'), + tags=['schema'], + raw_sql="{{ test_unique(model=source('my_source', 'my_table'), column_name='id') }}", + description='', + columns={}, + column_name='id' + ), + ] + + self._expected_model_tests = [ + ParsedNode( + alias='accepted_values_model_one_id__a__b', + name='accepted_values_model_one_id__a__b', + database='test', + schema='analytics', + resource_type='test', + unique_id='test.root.accepted_values_model_one_id__a__b', + fqn=['root', 'schema_test', + 'accepted_values_model_one_id__a__b'], + empty=False, + package_name='root', + original_file_path='test_one.yml', + root_path=get_os_path('/usr/src/app'), + refs=[['model_one']], + sources=[], + depends_on={'nodes': [], 'macros': []}, + config=self.model_config, + path=get_os_path( + 'schema_test/accepted_values_model_one_id__a__b.sql'), + tags=['schema'], + raw_sql="{{ test_accepted_values(model=ref('model_one'), column_name='id', values=['a', 'b']) }}", + description='', + columns={}, + column_name='id' + ), + ParsedNode( + alias='not_null_model_one_id', + name='not_null_model_one_id', + database='test', + schema='analytics', + resource_type='test', + unique_id='test.root.not_null_model_one_id', + fqn=['root', 'schema_test', 'not_null_model_one_id'], + empty=False, + package_name='root', + root_path=get_os_path('/usr/src/app'), + refs=[['model_one']], + sources=[], + depends_on={'nodes': [], 'macros': []}, + config=self.model_config, + original_file_path='test_one.yml', + path=get_os_path('schema_test/not_null_model_one_id.sql'), + tags=['schema'], + raw_sql="{{ test_not_null(model=ref('model_one'), column_name='id') }}", + description='', + columns={}, + column_name='id' + ), + ParsedNode( + alias='relationships_model_one_id__id__ref_model_two_', + name='relationships_model_one_id__id__ref_model_two_', + database='test', + schema='analytics', + resource_type='test', + unique_id='test.root.relationships_model_one_id__id__ref_model_two_', # noqa + fqn=['root', 'schema_test', + 'relationships_model_one_id__id__ref_model_two_'], + empty=False, + package_name='root', + original_file_path='test_one.yml', + root_path=get_os_path('/usr/src/app'), + refs=[['model_one'], ['model_two']], + sources=[], + depends_on={'nodes': [], 'macros': []}, + config=self.model_config, + path=get_os_path('schema_test/relationships_model_one_id__id__ref_model_two_.sql'), # noqa + tags=['schema'], + raw_sql="{{ test_relationships(model=ref('model_one'), column_name='id', from='id', to=ref('model_two')) }}", + description='', + columns={}, + column_name='id' + ), + ParsedNode( + alias='some_test_model_one_value', + name='some_test_model_one_value', + database='test', + schema='analytics', + resource_type='test', + unique_id='test.root.some_test_model_one_value', + fqn=['root', 'schema_test', 'some_test_model_one_value'], + empty=False, + package_name='root', + original_file_path='test_one.yml', + root_path=get_os_path('/usr/src/app'), + refs=[['model_one']], + sources=[], + depends_on={'nodes': [], 'macros': []}, + config=self.model_config, + path=get_os_path('schema_test/some_test_model_one_value.sql'), + tags=['schema'], + raw_sql="{{ test_some_test(model=ref('model_one'), key='value') }}", + description='', + columns={} + ), + ParsedNode( + alias='unique_model_one_id', + name='unique_model_one_id', + database='test', + schema='analytics', + resource_type='test', + unique_id='test.root.unique_model_one_id', + fqn=['root', 'schema_test', 'unique_model_one_id'], + empty=False, + package_name='root', + root_path=get_os_path('/usr/src/app'), + refs=[['model_one']], + sources=[], + depends_on={'nodes': [], 'macros': []}, + config=self.model_config, + original_file_path='test_one.yml', + path=get_os_path('schema_test/unique_model_one_id.sql'), + tags=['schema'], + raw_sql="{{ test_unique(model=ref('model_one'), column_name='id') }}", + description='', + columns={}, + column_name='id' + ), + ] + + self._expected_patch = ParsedNodePatch( + name='model_one', + description='blah blah', + original_file_path='test_one.yml', + columns={ + 'id': { + 'name': 'id', + 'description': 'user ID', + }}, + docrefs=[], + ) + + def test__source_schema(self): + test_yml = yaml.safe_load(''' + version: 2 + sources: + - name: my_source + loader: some_loader + description: my source description + freshness: + warn_after: + count: 10 + period: hour + error_after: + count: 20 + period: hour + loaded_at_field: something + tables: + - name: my_table + description: "my table description" + sql_table_name: foo.bar + freshness: + warn_after: + count: 7 + period: hour + columns: + - name: id + description: user ID + tests: + - unique + - not_null + - accepted_values: + values: + - a + - b + - relationships: + from: id + to: ref('model_two') + tests: + - some_test: + key: value + ''') + parser = SchemaParser( self.root_project_config, self.all_projects, self.macro_manifest ) + root_dir = get_os_path('/usr/src/app') + results = list(parser.parse_schema( + path='test_one.yml', + test_yml=test_yml, + package_name='root', + root_dir=root_dir + )) - self.assertEqual( - parser.parse_sql_nodes(models), - ({ - 'model.root.model_one': ParsedNode( - alias='model_one', - name='model_one', - database='test', - schema='analytics', - resource_type='model', - unique_id='model.root.model_one', - fqn=['root', 'model_one'], - empty=False, - package_name='root', - original_file_path='model_one.sql', - root_path=get_os_path('/usr/src/app'), - refs=[], - depends_on={ - 'nodes': [], - 'macros': [] - }, - config=self.model_config, - tags=[], - path='model_one.sql', - raw_sql=self.find_input_by_name( - models, 'model_one').get('raw_sql'), - description='', - columns={} - ) - }, []) - ) + tests = sorted((node for t, node in results if t == 'test'), + key=lambda n: n.name) + patches = sorted((node for t, node in results if t == 'patch'), + key=lambda n: n.name) + sources = sorted((node for t, node in results if t == 'source'), + key=lambda n: n.name) + self.assertEqual(len(tests), 5) + self.assertEqual(len(patches), 0) + self.assertEqual(len(sources), 1) + self.assertEqual(len(results), 6) + + for test, expected in zip(tests, self._expected_source_tests): + self.assertEqual(test, expected) + + self.assertEqual(sources[0], self._expected_source) + + def test__model_schema(self): + test_yml = yaml.safe_load(''' + version: 2 + models: + - name: model_one + description: blah blah + columns: + - name: id + description: user ID + tests: + - unique + - not_null + - accepted_values: + values: + - a + - b + - relationships: + from: id + to: ref('model_two') + tests: + - some_test: + key: value + ''') + parser = SchemaParser( + self.root_project_config, + self.all_projects, + self.macro_manifest + ) + results = list(parser.parse_schema( + path='test_one.yml', + test_yml=test_yml, + package_name='root', + root_dir=get_os_path('/usr/src/app') + )) + + tests = sorted((node for t, node in results if t == 'test'), + key=lambda n: n.name) + patches = sorted((node for t, node in results if t == 'patch'), + key=lambda n: n.name) + sources = sorted((node for t, node in results if t == 'source'), + key=lambda n: n.name) + self.assertEqual(len(tests), 5) + self.assertEqual(len(patches), 1) + self.assertEqual(len(sources), 0) + self.assertEqual(len(results), 6) + + for test, expected in zip(tests, self._expected_model_tests): + self.assertEqual(test, expected) + + + self.assertEqual(patches[0], self._expected_patch) + + def test__mixed_schema(self): + test_yml = yaml.safe_load(''' + version: 2 + models: + - name: model_one + description: blah blah + columns: + - name: id + description: user ID + tests: + - unique + - not_null + - accepted_values: + values: + - a + - b + - relationships: + from: id + to: ref('model_two') + tests: + - some_test: + key: value + sources: + - name: my_source + loader: some_loader + description: my source description + freshness: + warn_after: + count: 10 + period: hour + error_after: + count: 20 + period: hour + loaded_at_field: something + tables: + - name: my_table + description: "my table description" + sql_table_name: '{{ var("test_schema_name") }}.bar' + freshness: + warn_after: + count: 7 + period: hour + columns: + - name: id + description: user ID + tests: + - unique + - not_null + - accepted_values: + values: + - a + - b + - relationships: + from: id + to: ref('model_two') + tests: + - some_test: + key: value + ''') + parser = SchemaParser( + self.root_project_config, + self.all_projects, + self.macro_manifest + ) + results = list(parser.parse_schema( + path='test_one.yml', + test_yml=test_yml, + package_name='root', + root_dir=get_os_path('/usr/src/app') + )) + + tests = sorted((node for t, node in results if t == 'test'), + key=lambda n: n.name) + patches = sorted((node for t, node in results if t == 'patch'), + key=lambda n: n.name) + sources = sorted((node for t, node in results if t == 'source'), + key=lambda n: n.name) + self.assertEqual(len(tests), 10) + self.assertEqual(len(patches), 1) + self.assertEqual(len(sources), 1) + self.assertEqual(len(results), 12) + + expected_tests = self._expected_model_tests + self._expected_source_tests + expected_tests.sort(key=lambda n: n.name) + for test, expected in zip(tests, expected_tests): + self.assertEqual(test, expected) + + self.assertEqual(patches[0], self._expected_patch) + self.assertEqual(sources[0], self._expected_source) + + @mock.patch.object(SchemaParser, 'find_schema_yml') + @mock.patch.object(dbt.parser.schemas, 'logger') + def test__schema_v2_as_v1(self, mock_logger, find_schema_yml): + test_yml = yaml.safe_load( + '{models: [{name: model_one, description: "blah blah", columns: [' + '{name: id, description: "user ID", tests: [unique, not_null, ' + '{accepted_values: {values: ["a", "b"]}},' + '{relationships: {from: id, to: ref(\'model_two\')}}]' + '}], tests: [some_test: { key: value }]}]}' + ) + find_schema_yml.return_value = [('/some/path/schema.yml', test_yml)] + root_project = {} + all_projects = {} + root_dir = '/some/path' + relative_dirs = ['a', 'b'] + parser = dbt.parser.schemas.SchemaParser(root_project, all_projects, None) + with self.assertRaises(dbt.exceptions.CompilationException) as cm: + parser.load_and_parse( + 'test', root_dir, relative_dirs + ) + self.assertIn('https://docs.getdbt.com/v0.11/docs/schemayml-files', + str(cm.exception)) + + @mock.patch.object(SchemaParser, 'find_schema_yml') + @mock.patch.object(dbt.parser.schemas, 'logger') + def test__schema_v1_version_model(self, mock_logger, find_schema_yml): + test_yml = yaml.safe_load( + '{model_one: {constraints: {not_null: [id],' + 'unique: [id],' + 'accepted_values: [{field: id, values: ["a","b"]}],' + 'relationships: [{from: id, to: ref(\'model_two\'), field: id}]' # noqa + '}}, version: {constraints: {not_null: [id]}}}' + ) + find_schema_yml.return_value = [('/some/path/schema.yml', test_yml)] + root_project = {} + all_projects = {} + root_dir = '/some/path' + relative_dirs = ['a', 'b'] + parser = dbt.parser.schemas.SchemaParser(root_project, all_projects, None) + with self.assertRaises(dbt.exceptions.CompilationException) as cm: + parser.load_and_parse( + 'test', root_dir, relative_dirs + ) + self.assertIn('https://docs.getdbt.com/v0.11/docs/schemayml-files', + str(cm.exception)) + + @mock.patch.object(SchemaParser, 'find_schema_yml') + @mock.patch.object(dbt.parser.schemas, 'logger') + def test__schema_v1_version_1(self, mock_logger, find_schema_yml): + test_yml = yaml.safe_load( + '{model_one: {constraints: {not_null: [id],' + 'unique: [id],' + 'accepted_values: [{field: id, values: ["a","b"]}],' + 'relationships: [{from: id, to: ref(\'model_two\'), field: id}]' # noqa + '}}, version: 1}' + ) + find_schema_yml.return_value = [('/some/path/schema.yml', test_yml)] + root_project = {} + all_projects = {} + root_dir = '/some/path' + relative_dirs = ['a', 'b'] + parser = dbt.parser.schemas.SchemaParser(root_project, all_projects, None) + with self.assertRaises(dbt.exceptions.CompilationException) as cm: + parser.load_and_parse( + 'test', root_dir, relative_dirs + ) + self.assertIn('https://docs.getdbt.com/v0.11/docs/schemayml-files', + str(cm.exception)) + + +class ParserTest(BaseParserTest): + + def find_input_by_name(self, models, name): + return next( + (model for model in models if model.get('name') == name), + {}) + + def setUp(self): + super(ParserTest, self).setUp() + + self.macro_manifest = Manifest(macros={}, nodes={}, docs={}, + generated_at=timestring(), disabled=[]) + + self.model_config = { + 'enabled': True, + 'materialized': 'view', + 'post-hook': [], + 'pre-hook': [], + 'vars': {}, + 'quoting': {}, + 'column_types': {}, + 'tags': [], + } + + self.disabled_config = { + 'enabled': False, + 'materialized': 'view', + 'post-hook': [], + 'pre-hook': [], + 'vars': {}, + 'quoting': {}, + 'column_types': {}, + 'tags': [], + } + + + def test__single_model(self): + models = [{ + 'name': 'model_one', + 'resource_type': 'model', + 'package_name': 'root', + 'original_file_path': 'model_one.sql', + 'root_path': get_os_path('/usr/src/app'), + 'path': 'model_one.sql', + 'raw_sql': ("select * from events"), + }] + parser = ModelParser( + self.root_project_config, + self.all_projects, + self.macro_manifest + ) + + self.assertEqual( + parser.parse_sql_nodes(models), + ({ + 'model.root.model_one': ParsedNode( + alias='model_one', + name='model_one', + database='test', + schema='analytics', + resource_type='model', + unique_id='model.root.model_one', + fqn=['root', 'model_one'], + empty=False, + package_name='root', + original_file_path='model_one.sql', + root_path=get_os_path('/usr/src/app'), + refs=[], + sources=[], + depends_on={ + 'nodes': [], + 'macros': [] + }, + config=self.model_config, + tags=[], + path='model_one.sql', + raw_sql=self.find_input_by_name( + models, 'model_one').get('raw_sql'), + description='', + columns={} + ) + }, []) + ) def test__single_model__nested_configuration(self): models = [{ @@ -269,6 +871,7 @@ def test__single_model__nested_configuration(self): original_file_path='nested/path/model_one.sql', root_path=get_os_path('/usr/src/app'), refs=[], + sources=[], depends_on={ 'nodes': [], 'macros': [] @@ -316,6 +919,7 @@ def test__empty_model(self): empty=True, package_name='root', refs=[], + sources=[], depends_on={ 'nodes': [], 'macros': [], @@ -372,6 +976,7 @@ def test__simple_dependency(self): empty=False, package_name='root', refs=[], + sources=[], depends_on={ 'nodes': [], 'macros': [] @@ -398,6 +1003,7 @@ def test__simple_dependency(self): empty=False, package_name='root', refs=[['base']], + sources=[], depends_on={ 'nodes': [], 'macros': [] @@ -482,6 +1088,7 @@ def test__multiple_dependencies(self): empty=False, package_name='root', refs=[], + sources=[], depends_on={ 'nodes': [], 'macros': [] @@ -507,6 +1114,7 @@ def test__multiple_dependencies(self): empty=False, package_name='root', refs=[], + sources=[], depends_on={ 'nodes': [], 'macros': [] @@ -532,6 +1140,7 @@ def test__multiple_dependencies(self): empty=False, package_name='root', refs=[['events']], + sources=[], depends_on={ 'nodes': [], 'macros': [] @@ -557,6 +1166,7 @@ def test__multiple_dependencies(self): empty=False, package_name='root', refs=[['sessions']], + sources=[], depends_on={ 'nodes': [], 'macros': [] @@ -582,6 +1192,7 @@ def test__multiple_dependencies(self): empty=False, package_name='root', refs=[['sessions_tx'], ['events_tx']], + sources=[], depends_on={ 'nodes': [], 'macros': [] @@ -668,6 +1279,7 @@ def test__multiple_dependencies__packages(self): empty=False, package_name='snowplow', refs=[], + sources=[], depends_on={ 'nodes': [], 'macros': [] @@ -693,6 +1305,7 @@ def test__multiple_dependencies__packages(self): empty=False, package_name='snowplow', refs=[], + sources=[], depends_on={ 'nodes': [], 'macros': [] @@ -718,6 +1331,7 @@ def test__multiple_dependencies__packages(self): empty=False, package_name='snowplow', refs=[['events']], + sources=[], depends_on={ 'nodes': [], 'macros': [] @@ -743,6 +1357,7 @@ def test__multiple_dependencies__packages(self): empty=False, package_name='snowplow', refs=[['sessions']], + sources=[], depends_on={ 'nodes': [], 'macros': [] @@ -769,6 +1384,7 @@ def test__multiple_dependencies__packages(self): package_name='root', refs=[['snowplow', 'sessions_tx'], ['snowplow', 'events_tx']], + sources=[], depends_on={ 'nodes': [], 'macros': [] @@ -801,6 +1417,7 @@ def test__process_refs__packages(self): 'empty': False, 'package_name': 'snowplow', 'refs': [], + 'sources': [], 'depends_on': { 'nodes': [], 'macros': [] @@ -823,6 +1440,7 @@ def test__process_refs__packages(self): 'empty': False, 'package_name': 'root', 'refs': [], + 'sources': [], 'depends_on': { 'nodes': [], 'macros': [] @@ -845,6 +1463,7 @@ def test__process_refs__packages(self): 'empty': False, 'package_name': 'root', 'refs': [['events']], + 'sources': [], 'depends_on': { 'nodes': [], 'macros': [] @@ -884,6 +1503,7 @@ def test__process_refs__packages(self): 'empty': False, 'package_name': 'snowplow', 'refs': [], + 'sources': [], 'depends_on': { 'nodes': [], 'macros': [] @@ -909,6 +1529,7 @@ def test__process_refs__packages(self): 'empty': False, 'package_name': 'root', 'refs': [], + 'sources': [], 'depends_on': { 'nodes': [], 'macros': [] @@ -934,6 +1555,7 @@ def test__process_refs__packages(self): 'empty': False, 'package_name': 'root', 'refs': [['events']], + 'sources': [], 'depends_on': { 'nodes': ['model.root.events'], 'macros': [] @@ -988,6 +1610,7 @@ def test__in_model_config(self): empty=False, package_name='root', refs=[], + sources=[], depends_on={ 'nodes': [], 'macros': [], @@ -1076,6 +1699,7 @@ def test__root_project_config(self): empty=False, package_name='root', refs=[], + sources=[], depends_on={ 'nodes': [], 'macros': [] @@ -1101,6 +1725,7 @@ def test__root_project_config(self): empty=False, package_name='root', refs=[], + sources=[], depends_on={ 'nodes': [], 'macros': [] @@ -1126,6 +1751,7 @@ def test__root_project_config(self): empty=False, package_name='root', refs=[], + sources=[], depends_on={ 'nodes': [], 'macros': [] @@ -1281,6 +1907,7 @@ def test__other_project_config(self): empty=False, package_name='root', refs=[], + sources=[], depends_on={ 'nodes': [], 'macros': [] @@ -1306,6 +1933,7 @@ def test__other_project_config(self): empty=False, package_name='root', refs=[], + sources=[], depends_on={ 'nodes': [], 'macros': [] @@ -1331,6 +1959,7 @@ def test__other_project_config(self): empty=False, package_name='root', refs=[], + sources=[], depends_on={ 'nodes': [], 'macros': [] @@ -1356,6 +1985,7 @@ def test__other_project_config(self): empty=False, package_name='snowplow', refs=[], + sources=[], depends_on={ 'nodes': [], 'macros': [] @@ -1383,6 +2013,7 @@ def test__other_project_config(self): database='test', schema='analytics', refs=[], + sources=[], depends_on={ 'nodes': [], 'macros': [] @@ -1406,6 +2037,7 @@ def test__other_project_config(self): database='test', schema='analytics', refs=[], + sources=[], depends_on={ 'nodes': [], 'macros': [] @@ -1421,349 +2053,6 @@ def test__other_project_config(self): ]) ) - def test__simple_schema_v1_test(self): - test_yml = yaml.safe_load( - '{model_one: {constraints: {not_null: [id],' - 'unique: [id],' - 'accepted_values: [{field: id, values: ["a","b"]}],' - 'relationships: [{from: id, to: ref(\'model_two\'), field: id}]' # noqa - '}}}' - ) - - parser = SchemaParser( - self.root_project_config, - self.all_projects, - self.macro_manifest - ) - results = list(parser.parse_v1_test_yml( - original_file_path='test_one.yml', - test_yml=test_yml, - package_name='root', - root_dir=get_os_path('/usr/src/app') - )) - results.sort(key=lambda n: n.name) - - not_null_sql = "{{ test_not_null(model=ref('model_one'), arg='id') }}" - unique_sql = "{{ test_unique(model=ref('model_one'), arg='id') }}" - accepted_values_sql = "{{ test_accepted_values(model=ref('model_one'), field='id', values=['a', 'b']) }}" # noqa - relationships_sql = "{{ test_relationships(model=ref('model_one'), field='id', from='id', to=ref('model_two')) }}" # noqa - - expected = [ - ParsedNode( - alias='accepted_values_model_one_id__a__b', - name='accepted_values_model_one_id__a__b', - database='test', - schema='analytics', - resource_type='test', - unique_id='test.root.accepted_values_model_one_id__a__b', # noqa - fqn=['root', 'schema_test', - 'accepted_values_model_one_id__a__b'], - empty=False, - package_name='root', - original_file_path='test_one.yml', - root_path=get_os_path('/usr/src/app'), - refs=[['model_one']], - depends_on={ - 'nodes': [], - 'macros': [] - }, - config=self.model_config, - path=get_os_path( - 'schema_test/accepted_values_model_one_id__a__b.sql'), - tags=['schema'], - raw_sql=accepted_values_sql, - description='', - columns={} - ), - ParsedNode( - alias='not_null_model_one_id', - name='not_null_model_one_id', - database='test', - schema='analytics', - resource_type='test', - unique_id='test.root.not_null_model_one_id', - fqn=['root', 'schema_test', 'not_null_model_one_id'], - empty=False, - package_name='root', - root_path=get_os_path('/usr/src/app'), - refs=[['model_one']], - depends_on={ - 'nodes': [], - 'macros': [] - }, - config=self.model_config, - original_file_path='test_one.yml', - path=get_os_path( - 'schema_test/not_null_model_one_id.sql'), - tags=['schema'], - raw_sql=not_null_sql, - description='', - columns={} - ), - ParsedNode( - alias='relationships_model_one_id__id__ref_model_two_', - name='relationships_model_one_id__id__ref_model_two_', - database='test', - schema='analytics', - resource_type='test', - unique_id='test.root.relationships_model_one_id__id__ref_model_two_', # noqa - fqn=['root', 'schema_test', - 'relationships_model_one_id__id__ref_model_two_'], - empty=False, - package_name='root', - original_file_path='test_one.yml', - root_path=get_os_path('/usr/src/app'), - refs=[['model_one'], ['model_two']], - depends_on={ - 'nodes': [], - 'macros': [] - }, - config=self.model_config, - path=get_os_path('schema_test/relationships_model_one_id__id__ref_model_two_.sql'), # noqa - tags=['schema'], - raw_sql=relationships_sql, - description='', - columns={} - ), - ParsedNode( - alias='unique_model_one_id', - name='unique_model_one_id', - database='test', - schema='analytics', - resource_type='test', - unique_id='test.root.unique_model_one_id', - fqn=['root', 'schema_test', 'unique_model_one_id'], - empty=False, - package_name='root', - root_path=get_os_path('/usr/src/app'), - refs=[['model_one']], - depends_on={ - 'nodes': [], - 'macros': [] - }, - config=self.model_config, - original_file_path='test_one.yml', - path=get_os_path('schema_test/unique_model_one_id.sql'), - tags=['schema'], - raw_sql=unique_sql, - description='', - columns={} - ), - ] - self.assertEqual( - results, - expected, - ) - - def test__simple_schema_v2(self): - test_yml = yaml.safe_load( - '{models: [{name: model_one, description: "blah blah", columns: [' - '{name: id, description: "user ID", tests: [unique, not_null, ' - '{accepted_values: {values: ["a", "b"]}},' - '{relationships: {from: id, to: ref(\'model_two\')}}]' - '}], tests: [some_test: { key: value }]}]}' - ) - parser = SchemaParser( - self.root_project_config, - self.all_projects, - self.macro_manifest - ) - results = list(parser.parse_v2_yml( - original_file_path='test_one.yml', - test_yml=test_yml, - package_name='root', - root_dir=get_os_path('/usr/src/app') - )) - - # split this into tests and patches, assert there's nothing else - tests = sorted((node for t, node in results if t == 'test'), - key=lambda n: n.name) - patches = sorted((node for t, node in results if t == 'patch'), - key=lambda n: n.name) - self.assertEqual(len(tests)+len(patches), len(results)) - - not_null_sql = "{{ test_not_null(model=ref('model_one'), column_name='id') }}" - unique_sql = "{{ test_unique(model=ref('model_one'), column_name='id') }}" - accepted_values_sql = "{{ test_accepted_values(model=ref('model_one'), column_name='id', values=['a', 'b']) }}" # noqa - relationships_sql = "{{ test_relationships(model=ref('model_one'), column_name='id', from='id', to=ref('model_two')) }}" # noqa - some_test_sql = "{{ test_some_test(model=ref('model_one'), key='value') }}" - - expected_tests = [ - ParsedNode( - alias='accepted_values_model_one_id__a__b', - name='accepted_values_model_one_id__a__b', - database='test', - schema='analytics', - resource_type='test', - unique_id='test.root.accepted_values_model_one_id__a__b', - fqn=['root', 'schema_test', - 'accepted_values_model_one_id__a__b'], - empty=False, - package_name='root', - original_file_path='test_one.yml', - root_path=get_os_path('/usr/src/app'), - refs=[['model_one']], - depends_on={'nodes': [], 'macros': []}, - config=self.model_config, - path=get_os_path( - 'schema_test/accepted_values_model_one_id__a__b.sql'), - tags=['schema'], - raw_sql=accepted_values_sql, - description='', - columns={}, - column_name='id' - ), - ParsedNode( - alias='not_null_model_one_id', - name='not_null_model_one_id', - database='test', - schema='analytics', - resource_type='test', - unique_id='test.root.not_null_model_one_id', - fqn=['root', 'schema_test', 'not_null_model_one_id'], - empty=False, - package_name='root', - root_path=get_os_path('/usr/src/app'), - refs=[['model_one']], - depends_on={'nodes': [], 'macros': []}, - config=self.model_config, - original_file_path='test_one.yml', - path=get_os_path('schema_test/not_null_model_one_id.sql'), - tags=['schema'], - raw_sql=not_null_sql, - description='', - columns={}, - column_name='id' - ), - ParsedNode( - alias='relationships_model_one_id__id__ref_model_two_', - name='relationships_model_one_id__id__ref_model_two_', - database='test', - schema='analytics', - resource_type='test', - unique_id='test.root.relationships_model_one_id__id__ref_model_two_', # noqa - fqn=['root', 'schema_test', - 'relationships_model_one_id__id__ref_model_two_'], - empty=False, - package_name='root', - original_file_path='test_one.yml', - root_path=get_os_path('/usr/src/app'), - refs=[['model_one'], ['model_two']], - depends_on={'nodes': [], 'macros': []}, - config=self.model_config, - path=get_os_path('schema_test/relationships_model_one_id__id__ref_model_two_.sql'), # noqa - tags=['schema'], - raw_sql=relationships_sql, - description='', - columns={}, - column_name='id' - ), - ParsedNode( - alias='some_test_model_one_value', - name='some_test_model_one_value', - database='test', - schema='analytics', - resource_type='test', - unique_id='test.root.some_test_model_one_value', - fqn=['root', 'schema_test', 'some_test_model_one_value'], - empty=False, - package_name='root', - original_file_path='test_one.yml', - root_path=get_os_path('/usr/src/app'), - refs=[['model_one']], - depends_on={'nodes': [], 'macros': []}, - config=self.model_config, - path=get_os_path('schema_test/some_test_model_one_value.sql'), - tags=['schema'], - raw_sql=some_test_sql, - description='', - columns={} - ), - ParsedNode( - alias='unique_model_one_id', - name='unique_model_one_id', - database='test', - schema='analytics', - resource_type='test', - unique_id='test.root.unique_model_one_id', - fqn=['root', 'schema_test', 'unique_model_one_id'], - empty=False, - package_name='root', - root_path=get_os_path('/usr/src/app'), - refs=[['model_one']], - depends_on={'nodes': [], 'macros': []}, - config=self.model_config, - original_file_path='test_one.yml', - path=get_os_path('schema_test/unique_model_one_id.sql'), - tags=['schema'], - raw_sql=unique_sql, - description='', - columns={}, - column_name='id' - ), - ] - for test, expected in zip(tests, expected_tests): - self.assertEqual(test, expected) - - expected_patches = [ - ParsedNodePatch(name='model_one', - description='blah blah', - original_file_path='test_one.yml', - columns={ - 'id': { - 'name': 'id', - 'description': 'user ID', - }}, - docrefs=[], - ), - ] - for patch, expected in zip(patches, expected_patches): - self.assertEqual(patch, expected) - - @mock.patch.object(SchemaParser, 'find_schema_yml') - @mock.patch.object(dbt.parser.schemas, 'logger') - def test__schema_v2_as_v1(self, mock_logger, find_schema_yml): - test_yml = yaml.safe_load( - '{models: [{name: model_one, description: "blah blah", columns: [' - '{name: id, description: "user ID", tests: [unique, not_null, ' - '{accepted_values: {values: ["a", "b"]}},' - '{relationships: {from: id, to: ref(\'model_two\')}}]' - '}], tests: [some_test: { key: value }]}]}' - ) - find_schema_yml.return_value = [('/some/path/schema.yml', test_yml)] - root_project = {} - all_projects = {} - root_dir = '/some/path' - relative_dirs = ['a', 'b'] - parser = dbt.parser.schemas.SchemaParser(root_project, all_projects, None) - with self.assertRaises(dbt.exceptions.CompilationException) as cm: - parser.load_and_parse( - 'test', root_dir, relative_dirs - ) - self.assertIn('https://docs.getdbt.com/v0.11/docs/schemayml-files', - str(cm.exception)) - - @mock.patch.object(SchemaParser, 'build_node') - @mock.patch.object(SchemaParser, 'find_schema_yml') - @mock.patch.object(dbt.parser.schemas, 'logger') - def test__schema_v1_version_model(self, mock_logger, find_schema_yml, build_node): - test_yml = yaml.safe_load( - '{model_one: {constraints: {not_null: [id],' - 'unique: [id],' - 'accepted_values: [{field: id, values: ["a","b"]}],' - 'relationships: [{from: id, to: ref(\'model_two\'), field: id}]' # noqa - '}}, version: {constraints: {not_null: [id]}}}' - ) - find_schema_yml.return_value = [('/some/path/schema.yml', test_yml)] - root_project = {} - all_projects = {} - root_dir = '/some/path' - relative_dirs = ['a', 'b'] - parser = dbt.parser.schemas.SchemaParser(root_project, all_projects, None) - parser.load_and_parse( - 'test', root_dir, relative_dirs - ) - def test__simple_data_test(self): tests = [{ 'name': 'no_events', @@ -1795,6 +2084,7 @@ def test__simple_data_test(self): empty=False, package_name='root', refs=[['base']], + sources=[], depends_on={ 'nodes': [], 'macros': [] @@ -1917,6 +2207,7 @@ def test__simple_macro_used_in_model(self): original_file_path='model_one.sql', root_path=get_os_path('/usr/src/app'), refs=[], + sources=[], depends_on={ 'nodes': [], 'macros': [] @@ -1964,6 +2255,7 @@ def test__macro_no_explicit_project_used_in_model(self): package_name='root', root_path=get_os_path('/usr/src/app'), refs=[], + sources=[], depends_on={ 'nodes': [], 'macros': [] diff --git a/test/unit/test_snowflake_adapter.py b/test/unit/test_snowflake_adapter.py index d340908c271..0ee65d05759 100644 --- a/test/unit/test_snowflake_adapter.py +++ b/test/unit/test_snowflake_adapter.py @@ -236,7 +236,7 @@ def test_authenticator_private_key_authentication(self, mock_get_private_key): self.snowflake.assert_has_calls([ mock.call( account='test_account', autocommit=False, - client_session_keep_alive=False, database='test_databse', + client_session_keep_alive=False, database='test_database', role=None, schema='public', user='test_user', warehouse='test_warehouse', private_key='test_key') ]) diff --git a/test/unit/utils.py b/test/unit/utils.py index 7e977763dbf..fafb89484e7 100644 --- a/test/unit/utils.py +++ b/test/unit/utils.py @@ -21,7 +21,7 @@ def config_from_parts_or_dicts(project, profile, packages=None, cli_vars='{}'): project.profile_name, cli_vars) args = Obj() - args.cli_vars = cli_vars + args.vars = repr(cli_vars) return RuntimeConfig.from_parts( project=project, profile=profile,