From 061a9abbf2f4d789e02d780503523c09dc9c9108 Mon Sep 17 00:00:00 2001 From: Gerda Shank Date: Mon, 11 Mar 2024 18:20:46 -0400 Subject: [PATCH 01/11] Add --exclude-resource-types param --- core/dbt/cli/params.py | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/core/dbt/cli/params.py b/core/dbt/cli/params.py index 291ac7b44c4..9abd54182a1 100644 --- a/core/dbt/cli/params.py +++ b/core/dbt/cli/params.py @@ -402,6 +402,7 @@ "analysis", "model", "test", + "unit_test", "exposure", "snapshot", "seed", @@ -415,6 +416,33 @@ default=(), ) +exclude_resource_type = click.option( + "--exclude-resource-types", + "--exclude-resource-type", + envvar="DBT_EXCLUDE_RESOURCE_TYPES", + help="Specify the types of resources that dbt will exclude", + type=ChoiceTuple( + [ + "metric", + "semantic_model", + "saved_query", + "source", + "analysis", + "model", + "test", + "unit_test", + "exposure", + "snapshot", + "seed", + "default", + ], + case_sensitive=False, + ), + cls=MultiOption, + multiple=True, + default=(), +) + # Renamed to --export-saved-queries deprecated_include_saved_query = click.option( "--include-saved-query/--no-include-saved-query", From 244bb80b0d754b651f10fa3a00a385b0bbe2e4aa Mon Sep 17 00:00:00 2001 From: Gerda Shank Date: Mon, 11 Mar 2024 22:39:36 -0400 Subject: [PATCH 02/11] Add exclude_resource_types to build command --- core/dbt/cli/main.py | 3 +++ core/dbt/cli/params.py | 2 +- core/dbt/task/build.py | 6 +++++- tests/functional/unit_testing/test_unit_testing.py | 5 ++++- 4 files changed, 13 insertions(+), 3 deletions(-) diff --git a/core/dbt/cli/main.py b/core/dbt/cli/main.py index 8b28224eac5..5dbeae13697 100644 --- a/core/dbt/cli/main.py +++ b/core/dbt/cli/main.py @@ -194,6 +194,7 @@ def cli(ctx, **kwargs): @p.profiles_dir @p.project_dir @p.resource_type +@p.exclude_resource_type @p.select @p.selector @p.show @@ -499,6 +500,7 @@ def init(ctx, **kwargs): @p.profiles_dir @p.project_dir @p.resource_type +@p.exclude_resource_type @p.raw_select @p.selector @p.target @@ -627,6 +629,7 @@ def retry(ctx, **kwargs): @p.profiles_dir @p.project_dir @p.resource_type +@p.exclude_resource_type @p.select @p.selector @p.target diff --git a/core/dbt/cli/params.py b/core/dbt/cli/params.py index 9abd54182a1..1ffeb53c94f 100644 --- a/core/dbt/cli/params.py +++ b/core/dbt/cli/params.py @@ -391,7 +391,7 @@ resource_type = click.option( "--resource-types", "--resource-type", - envvar=None, + envvar="DBT_RESOURCE_TYPES", help="Restricts the types of resources that dbt will include", type=ChoiceTuple( [ diff --git a/core/dbt/task/build.py b/core/dbt/task/build.py index 7c11839942a..00b93951a23 100644 --- a/core/dbt/task/build.py +++ b/core/dbt/task/build.py @@ -81,7 +81,7 @@ def __init__(self, args, config, manifest) -> None: def resource_types(self, no_unit_tests=False): if not self.args.resource_types: - resource_types = list(self.ALL_RESOURCE_VALUES) + resource_types = set(self.ALL_RESOURCE_VALUES) else: resource_types = set(self.args.resource_types) @@ -89,6 +89,10 @@ def resource_types(self, no_unit_tests=False): resource_types.remove("all") resource_types.update(self.ALL_RESOURCE_VALUES) + if self.args.exclude_resource_types: + exclude_resource_types = set(self.args.exclude_resource_types) + resource_types = resource_types - exclude_resource_types + # First we get selected_nodes including unit tests, then without, # and do a set difference. if no_unit_tests is True and NodeType.Unit in resource_types: diff --git a/tests/functional/unit_testing/test_unit_testing.py b/tests/functional/unit_testing/test_unit_testing.py index ffa3d0e34b2..6cfaad4b8ea 100644 --- a/tests/functional/unit_testing/test_unit_testing.py +++ b/tests/functional/unit_testing/test_unit_testing.py @@ -50,7 +50,10 @@ def test_basic(self, project): results = run_dbt(["test", "--select", "my_model"], expect_pass=False) assert len(results) == 5 - results = run_dbt(["build", "--select", "my_model"], expect_pass=False) + results = run_dbt( + ["build", "--select", "my_model", "--resource-types", "model unit_test"], + expect_pass=False, + ) assert len(results) == 6 for result in results: if result.node.unique_id == "model.test.my_model": From c0acc918609f7b12ee3aebdd6df404e1913d2d32 Mon Sep 17 00:00:00 2001 From: Gerda Shank Date: Tue, 12 Mar 2024 13:36:29 -0400 Subject: [PATCH 03/11] Update build, list and clone resource_types methods --- core/dbt/task/base.py | 25 ++++++++++++++++++++++++- core/dbt/task/build.py | 17 ++++------------- core/dbt/task/clone.py | 18 ++++++------------ core/dbt/task/list.py | 17 ++++++----------- 4 files changed, 40 insertions(+), 37 deletions(-) diff --git a/core/dbt/task/base.py b/core/dbt/task/base.py index 690ae36a71b..28a6feed837 100644 --- a/core/dbt/task/base.py +++ b/core/dbt/task/base.py @@ -6,7 +6,7 @@ from contextlib import nullcontext from datetime import datetime from pathlib import Path -from typing import Any, Dict, List, Optional, Type, Union +from typing import Any, Dict, List, Optional, Type, Union, Set from dbt.compilation import Compiler import dbt_common.exceptions.base @@ -480,3 +480,26 @@ def on_skip(self): def do_skip(self, cause=None): self.skip = True self.skip_cause = cause + + +def resource_types_from_args( + args, default_resource_values: Set[str], all_resource_values: Set[str] +) -> Set: + + if not args.resource_types: + resource_types = default_resource_values + else: + resource_types = set(args.resource_types) + + if "all" in resource_types: + resource_types.remove("all") + resource_types.update(all_resource_values) + if "default" in resource_types: + resource_types.remove("default") + resource_types.update(default_resource_values) + + if args.exclude_resource_types: + exclude_resource_types = set(args.exclude_resource_types) + resource_types = resource_types - exclude_resource_types + + return resource_types diff --git a/core/dbt/task/build.py b/core/dbt/task/build.py index 00b93951a23..5d3a42b3b9f 100644 --- a/core/dbt/task/build.py +++ b/core/dbt/task/build.py @@ -11,7 +11,7 @@ from dbt.graph import ResourceTypeSelector, GraphQueue, Graph from dbt.node_types import NodeType from dbt.task.test import TestSelector -from dbt.task.base import BaseRunner +from dbt.task.base import BaseRunner, resource_types_from_args from dbt_common.events.functions import fire_event from dbt.events.types import LogNodeNoOpResult from dbt.exceptions import DbtInternalError @@ -80,18 +80,9 @@ def __init__(self, args, config, manifest) -> None: self.model_to_unit_test_map: Dict[str, List] = {} def resource_types(self, no_unit_tests=False): - if not self.args.resource_types: - resource_types = set(self.ALL_RESOURCE_VALUES) - else: - resource_types = set(self.args.resource_types) - - if "all" in resource_types: - resource_types.remove("all") - resource_types.update(self.ALL_RESOURCE_VALUES) - - if self.args.exclude_resource_types: - exclude_resource_types = set(self.args.exclude_resource_types) - resource_types = resource_types - exclude_resource_types + resource_types = resource_types_from_args( + self.args, set(self.ALL_RESOURCE_VALUES), set(self.ALL_RESOURCE_VALUES) + ) # First we get selected_nodes including unit tests, then without, # and do a set difference. diff --git a/core/dbt/task/clone.py b/core/dbt/task/clone.py index 53c322211cb..379ab1ab550 100644 --- a/core/dbt/task/clone.py +++ b/core/dbt/task/clone.py @@ -10,7 +10,7 @@ from dbt_common.exceptions import DbtInternalError, CompilationError from dbt.graph import ResourceTypeSelector from dbt.node_types import NodeType, REFABLE_NODE_TYPES -from dbt.task.base import BaseRunner +from dbt.task.base import BaseRunner, resource_types_from_args from dbt.task.run import _validate_materialization_relations_dict from dbt.task.runnable import GraphRunnableTask @@ -132,18 +132,12 @@ def before_run(self, adapter, selected_uids: AbstractSet[str]): @property def resource_types(self): - if not self.args.resource_types: - return REFABLE_NODE_TYPES - - values = set(self.args.resource_types) - - if "all" in values: - values.remove("all") - values.update(REFABLE_NODE_TYPES) - - values = [NodeType(val) for val in values if val in REFABLE_NODE_TYPES] + resource_types = resource_types_from_args( + self.args, set(REFABLE_NODE_TYPES), set(REFABLE_NODE_TYPES) + ) - return list(values) + resource_types = [NodeType(rt) for rt in resource_types if rt in REFABLE_NODE_TYPES] + return list(resource_types) def get_node_selector(self) -> ResourceTypeSelector: resource_types = self.resource_types diff --git a/core/dbt/task/list.py b/core/dbt/task/list.py index d71ff3faf18..ff6fea3447e 100644 --- a/core/dbt/task/list.py +++ b/core/dbt/task/list.py @@ -10,6 +10,7 @@ ) from dbt.flags import get_flags from dbt.graph import ResourceTypeSelector +from dbt.task.base import resource_types_from_args from dbt.task.runnable import GraphRunnableTask from dbt.task.test import TestSelector from dbt.node_types import NodeType @@ -183,17 +184,11 @@ def resource_types(self): if self.args.models: return [NodeType.Model] - if not self.args.resource_types: - return list(self.DEFAULT_RESOURCE_VALUES) - - values = set(self.args.resource_types) - if "default" in values: - values.remove("default") - values.update(self.DEFAULT_RESOURCE_VALUES) - if "all" in values: - values.remove("all") - values.update(self.ALL_RESOURCE_VALUES) - return list(values) + resource_types = resource_types_from_args( + self.args, set(self.ALL_RESOURCE_VALUES), set(self.DEFAULT_RESOURCE_VALUES) + ) + + return list(resource_types) @property def selection_arg(self): From 87d2e40e547a96d6fff6e41ad3d0106ad61aef48 Mon Sep 17 00:00:00 2001 From: Gerda Shank Date: Tue, 12 Mar 2024 14:01:50 -0400 Subject: [PATCH 04/11] Add test for --exclude-resource-types unit_test --- tests/functional/unit_testing/test_unit_testing.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tests/functional/unit_testing/test_unit_testing.py b/tests/functional/unit_testing/test_unit_testing.py index 6cfaad4b8ea..c1cb048dbfa 100644 --- a/tests/functional/unit_testing/test_unit_testing.py +++ b/tests/functional/unit_testing/test_unit_testing.py @@ -59,6 +59,13 @@ def test_basic(self, project): if result.node.unique_id == "model.test.my_model": result.status == NodeStatus.Skipped + # Run build command but specify no unit tests + results = run_dbt( + ["build", "--select", "my_model", "--exclude-resource-types", "unit_test"], + expect_pass=True, + ) + assert len(results) == 1 + # Test select by test name results = run_dbt(["test", "--select", "test_name:test_my_model_string_concat"]) assert len(results) == 1 From c5444a619ab3955cb78c31223f0fbb42bb3aef2a Mon Sep 17 00:00:00 2001 From: Gerda Shank Date: Tue, 12 Mar 2024 14:04:14 -0400 Subject: [PATCH 05/11] Changie --- .changes/unreleased/Features-20240312-140407.yaml | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 .changes/unreleased/Features-20240312-140407.yaml diff --git a/.changes/unreleased/Features-20240312-140407.yaml b/.changes/unreleased/Features-20240312-140407.yaml new file mode 100644 index 00000000000..5486d560aec --- /dev/null +++ b/.changes/unreleased/Features-20240312-140407.yaml @@ -0,0 +1,6 @@ +kind: Features +body: Allow excluding resource types for build, list, and clone commands +time: 2024-03-12T14:04:07.086017-04:00 +custom: + Author: gshank + Issue: "9237" From 9b8cbe86d91c373edf21addc56976eb8f8833766 Mon Sep 17 00:00:00 2001 From: Gerda Shank Date: Tue, 12 Mar 2024 14:53:04 -0400 Subject: [PATCH 06/11] Fix order of params in resource_types_from_args --- core/dbt/task/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/dbt/task/base.py b/core/dbt/task/base.py index 28a6feed837..87cf873bb15 100644 --- a/core/dbt/task/base.py +++ b/core/dbt/task/base.py @@ -483,7 +483,7 @@ def do_skip(self, cause=None): def resource_types_from_args( - args, default_resource_values: Set[str], all_resource_values: Set[str] + args, all_resource_values: Set[str], default_resource_values: Set[str] ) -> Set: if not args.resource_types: From 510bb33fabacdbf1c8121e522fdff00949bd8ef2 Mon Sep 17 00:00:00 2001 From: Gerda Shank Date: Tue, 12 Mar 2024 15:46:06 -0400 Subject: [PATCH 07/11] Skip MultiOptions with no values in Flags construction --- core/dbt/cli/flags.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/core/dbt/cli/flags.py b/core/dbt/cli/flags.py index 3cfafc9296f..d688d3e4fa4 100644 --- a/core/dbt/cli/flags.py +++ b/core/dbt/cli/flags.py @@ -399,7 +399,10 @@ def add_fn(x): # MultiOption flags come back as lists, but we want to pass them as space separated strings if isinstance(v, list): - v = " ".join(v) + if len(v) > 0: + v = " ".join(v) + else: + continue if k == "macro" and command == CliCommand.RUN_OPERATION: add_fn(v) From 5a8afb1fc6785b6d5fd613c2816a51b3bffb9df3 Mon Sep 17 00:00:00 2001 From: Gerda Shank Date: Wed, 13 Mar 2024 10:10:02 -0400 Subject: [PATCH 08/11] Rearrange typing in resource_types_from_args function --- core/dbt/task/base.py | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/core/dbt/task/base.py b/core/dbt/task/base.py index 87cf873bb15..a126bc9c054 100644 --- a/core/dbt/task/base.py +++ b/core/dbt/task/base.py @@ -16,6 +16,7 @@ from dbt.config.profile import read_profile from dbt.constants import DBT_PROJECT_FILE_NAME from dbt.contracts.graph.manifest import Manifest +from dbt.artifacts.resources.types import NodeType from dbt.artifacts.schemas.results import TimingInfo, collect_timing_info from dbt.artifacts.schemas.results import NodeStatus, RunningStatus, RunStatus from dbt.artifacts.schemas.run import RunResult @@ -483,23 +484,27 @@ def do_skip(self, cause=None): def resource_types_from_args( - args, all_resource_values: Set[str], default_resource_values: Set[str] + args, all_resource_values: Set[NodeType], default_resource_values: Set[NodeType] ) -> Set: if not args.resource_types: resource_types = default_resource_values else: - resource_types = set(args.resource_types) - - if "all" in resource_types: - resource_types.remove("all") - resource_types.update(all_resource_values) - if "default" in resource_types: - resource_types.remove("default") - resource_types.update(default_resource_values) + # This is a list of strings, not NodeTypes + arg_resource_types = args.resource_types.copy() + + if "all" in arg_resource_types: + arg_resource_types.remove("all") + arg_resource_types.update(all_resource_values) + if "default" in arg_resource_types: + arg_resource_types.remove("default") + arg_resource_types.update(default_resource_values) + # Convert to a set of NodeTypes now that the non-NodeType strings are gone + resource_types = set([NodeType(rt) for rt in arg_resource_types]) if args.exclude_resource_types: - exclude_resource_types = set(args.exclude_resource_types) + # Convert from a list of strings to a set of NodeTypes + exclude_resource_types = set([NodeType(rt) for rt in args.exclude_resource_types]) resource_types = resource_types - exclude_resource_types return resource_types From 39cce08be38bf2992a1b1b4fe2e2c3033d358db1 Mon Sep 17 00:00:00 2001 From: Gerda Shank Date: Wed, 13 Mar 2024 10:28:04 -0400 Subject: [PATCH 09/11] Comment and remove copy --- core/dbt/task/base.py | 2 +- core/dbt/task/clone.py | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/core/dbt/task/base.py b/core/dbt/task/base.py index a126bc9c054..9794bacfe69 100644 --- a/core/dbt/task/base.py +++ b/core/dbt/task/base.py @@ -491,7 +491,7 @@ def resource_types_from_args( resource_types = default_resource_values else: # This is a list of strings, not NodeTypes - arg_resource_types = args.resource_types.copy() + arg_resource_types = args.resource_types if "all" in arg_resource_types: arg_resource_types.remove("all") diff --git a/core/dbt/task/clone.py b/core/dbt/task/clone.py index 379ab1ab550..44f6a30841d 100644 --- a/core/dbt/task/clone.py +++ b/core/dbt/task/clone.py @@ -9,7 +9,7 @@ from dbt_common.dataclass_schema import dbtClassMixin from dbt_common.exceptions import DbtInternalError, CompilationError from dbt.graph import ResourceTypeSelector -from dbt.node_types import NodeType, REFABLE_NODE_TYPES +from dbt.node_types import REFABLE_NODE_TYPES from dbt.task.base import BaseRunner, resource_types_from_args from dbt.task.run import _validate_materialization_relations_dict from dbt.task.runnable import GraphRunnableTask @@ -136,7 +136,8 @@ def resource_types(self): self.args, set(REFABLE_NODE_TYPES), set(REFABLE_NODE_TYPES) ) - resource_types = [NodeType(rt) for rt in resource_types if rt in REFABLE_NODE_TYPES] + # filter out any non-refable node types + resource_types = [rt for rt in resource_types if rt in REFABLE_NODE_TYPES] return list(resource_types) def get_node_selector(self) -> ResourceTypeSelector: From 50f4952cf06d448ccda9f4354464f7e4f0e73bf5 Mon Sep 17 00:00:00 2001 From: Gerda Shank Date: Wed, 13 Mar 2024 10:58:10 -0400 Subject: [PATCH 10/11] make args.resource_types a set --- core/dbt/task/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/dbt/task/base.py b/core/dbt/task/base.py index 9794bacfe69..3b893acdb54 100644 --- a/core/dbt/task/base.py +++ b/core/dbt/task/base.py @@ -491,7 +491,7 @@ def resource_types_from_args( resource_types = default_resource_values else: # This is a list of strings, not NodeTypes - arg_resource_types = args.resource_types + arg_resource_types = set(args.resource_types) if "all" in arg_resource_types: arg_resource_types.remove("all") From 4790a75d31a6b0bedf65037e70bcd3fb95ed8e14 Mon Sep 17 00:00:00 2001 From: Gerda Shank Date: Wed, 13 Mar 2024 11:26:58 -0400 Subject: [PATCH 11/11] Change return type from Set to Set[NodeType] --- core/dbt/task/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/dbt/task/base.py b/core/dbt/task/base.py index 3b893acdb54..fcc03adb170 100644 --- a/core/dbt/task/base.py +++ b/core/dbt/task/base.py @@ -485,7 +485,7 @@ def do_skip(self, cause=None): def resource_types_from_args( args, all_resource_values: Set[NodeType], default_resource_values: Set[NodeType] -) -> Set: +) -> Set[NodeType]: if not args.resource_types: resource_types = default_resource_values