Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Added warning to nodes selector if nothing was matched #2343

Merged
merged 4 commits into from
May 7, 2020
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
@@ -1,12 +1,16 @@
## dbt 0.17.0 (Release TBD)

### Features
- Added warning to nodes selector if nothing was matched ([#2115](https://github.com/fishtown-analytics/dbt/issues/2115), [#2343](https://github.com/fishtown-analytics/dbt/pull/2343))
- Suport column descriptions for BigQuery models ([#2335](https://github.com/fishtown-analytics/dbt/issues/2335), [#2402](https://github.com/fishtown-analytics/dbt/pull/2402))

### Fixes
- When tracking is disabled due to errors, do not reset the invocation ID ([#2398](https://github.com/fishtown-analytics/dbt/issues/2398), [#2400](https://github.com/fishtown-analytics/dbt/pull/2400))
- Fix for logic error in compilation errors for duplicate data test names ([#2406](https://github.com/fishtown-analytics/dbt/issues/2406), [#2407](https://github.com/fishtown-analytics/dbt/pull/2407))

Contributors:
- [@raalsky](https://github.com/Raalsky) ([#2343](https://github.com/fishtown-analytics/dbt/pull/2343))

## dbt 0.17.0b1 (May 5, 2020)

### Breaking changes
Expand Down
71 changes: 53 additions & 18 deletions core/dbt/graph/selector.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@
from dbt.logger import GLOBAL_LOGGER as logger
from dbt.utils import coalesce
from dbt.node_types import NodeType
import dbt.exceptions
from dbt.exceptions import RuntimeException, InternalException, warn_or_error

SELECTOR_PARENTS = '+'
SELECTOR_CHILDREN = '+'
Expand Down Expand Up @@ -50,7 +50,7 @@ def __init__(self, node_spec: str):
node_spec = node_spec[:-1]

if self.select_children and self.select_childrens_parents:
raise dbt.exceptions.RuntimeException(
raise RuntimeException(
'Invalid node spec {} - "@" prefix and "+" suffix are '
'incompatible'.format(self.raw)
)
Expand Down Expand Up @@ -79,6 +79,14 @@ def __str__(self):
return self._value_


def alert_non_existence(raw_spec, nodes):
if len(nodes) == 0:
warn_or_error(
f"The selector '{str(raw_spec)}' does not match any nodes and will"
f" be ignored"
)


def split_specs(node_specs: Iterable[str]):
specs: Set[str] = set()
for spec in node_specs:
Expand All @@ -88,8 +96,8 @@ def split_specs(node_specs: Iterable[str]):
return specs


def get_package_names(graph):
return set([node.split(".")[1] for node in graph.nodes()])
def get_package_names(nodes):
return set([node.split(".")[1] for node in nodes])
Raalsky marked this conversation as resolved.
Show resolved Hide resolved


def is_selected_node(real_node, node_selector):
Expand Down Expand Up @@ -190,7 +198,7 @@ def search(self, included_nodes, selector):
:param str selector: The selector or node name
"""
qualified_name = selector.split(".")
package_names = {node.split(".")[1] for node in included_nodes}
package_names = get_package_names(included_nodes)
for node, real_node in self.parsed_nodes(included_nodes):
if _node_is_match(qualified_name, package_names, real_node.fqn):
yield node
Expand Down Expand Up @@ -228,7 +236,7 @@ def search(self, included_nodes, selector):
'`${{source_name}}.${{target_name}}`, or '
'`${{package_name}}.${{source_name}}.${{target_name}}'
).format(selector)
raise dbt.exceptions.RuntimeException(msg)
raise RuntimeException(msg)

for node, real_node in self.source_nodes(included_nodes):
if target_package not in (real_node.package_name, SELECTOR_GLOB):
Expand Down Expand Up @@ -380,18 +388,47 @@ def get_nodes_from_spec(self, graph, spec):

return collected

def select_nodes(self, graph, raw_include_specs, raw_exclude_specs):
selected_nodes: Set[str] = set()
def get_nodes_from_multiple_specs(
self,
graph,
specs,
nodes=None,
check_existence=False,
exclude=False
):
selected_nodes: Set[str] = coalesce(nodes, set())
operator = set.difference_update if exclude else set.update

for raw_spec in split_specs(raw_include_specs):
for raw_spec in split_specs(specs):
spec = SelectionCriteria(raw_spec)
included_nodes = self.get_nodes_from_spec(graph, spec)
selected_nodes.update(included_nodes)
nodes = self.get_nodes_from_spec(graph, spec)

for raw_spec in split_specs(raw_exclude_specs):
spec = SelectionCriteria(raw_spec)
excluded_nodes = self.get_nodes_from_spec(graph, spec)
selected_nodes.difference_update(excluded_nodes)
if check_existence:
alert_non_existence(raw_spec, nodes)

operator(selected_nodes, nodes)

return selected_nodes

def select_nodes(self, graph, raw_include_specs, raw_exclude_specs):
raw_exclude_specs = coalesce(raw_exclude_specs, [])
check_existence = True

if not raw_include_specs:
check_existence = False
raw_include_specs = ['fqn:*', 'source:*']

selected_nodes = self.get_nodes_from_multiple_specs(
graph,
raw_include_specs,
check_existence=check_existence
)
selected_nodes = self.get_nodes_from_multiple_specs(
graph,
raw_exclude_specs,
nodes=selected_nodes,
exclude=True
)

return selected_nodes

Expand All @@ -407,7 +444,7 @@ def _is_match(self, node_name, resource_types, tags, required):
elif node_name in self.manifest.sources:
node = self.manifest.sources[node_name]
else:
raise dbt.exceptions.InternalException(
raise InternalException(
f'Node {node_name} not found in the manifest!'
)
if node.resource_type not in resource_types:
Expand All @@ -422,8 +459,6 @@ def _is_match(self, node_name, resource_types, tags, required):
return True

def get_selected(self, include, exclude, resource_types, tags, required):
include = coalesce(include, ['fqn:*', 'source:*'])
exclude = coalesce(exclude, [])
tags = coalesce(tags, [])

graph_members = {
Expand Down
2 changes: 1 addition & 1 deletion test/unit/test_graph_selection.py
Original file line number Diff line number Diff line change
Expand Up @@ -173,7 +173,7 @@ def test__spec_parsing(self):
self.invalid_spec('@source:a+')

def test__package_name_getter(self):
found = graph_selector.get_package_names(self.package_graph)
found = graph_selector.get_package_names(self.package_graph.nodes())

expected = set(['X', 'Y'])
self.assertEqual(found, expected)
Expand Down