Refactor deferral: always merge_from_artifact & support all commands

.changes/unreleased/Under the Hood-20231108-163613.yaml

@@ -0,0 +1,6 @@
+kind: Under the Hood
+body: Consolidate deferral methods & flags
+time: 2023-11-08T16:36:13.234324-05:00
+custom:
+  Author: jtcohen6
+  Issue: 7965 8715

core/dbt/cli/flags.py

@@ -113,6 +113,7 @@ def _get_params_by_source(ctx: Context, source_type: ParameterSource):
         def _assign_params(
             ctx: Context,
             params_assigned_from_default: set,
+            params_assigned_from_user: set,
             deprecated_env_vars: Dict[str, Callable],
         ):
             """Recursively adds all click params to flag object"""
@@ -178,15 +179,30 @@ def _assign_params(
                     object.__setattr__(self, flag_name, param_value)
 
                 # Track default assigned params.
-                if is_default:
+                # For flags that are accepted at both 'parent' and 'child' levels,
+                # we need to track user-provided and default values across both,
+                # to support detection of mutually exclusive flags later on.
+                if not is_default:
+                    params_assigned_from_user.add(param_name)
+                    if param_name in params_assigned_from_default:
+                        params_assigned_from_default.remove(param_name)
+                if is_default and param_name not in params_assigned_from_user:
                     params_assigned_from_default.add(param_name)
 
             if ctx.parent:
-                _assign_params(ctx.parent, params_assigned_from_default, deprecated_env_vars)
+                _assign_params(
+                    ctx.parent,
+                    params_assigned_from_default,
+                    params_assigned_from_user,
+                    deprecated_env_vars,
+                )
 
+        params_assigned_from_user = set()  # type: Set[str]
         params_assigned_from_default = set()  # type: Set[str]
         deprecated_env_vars: Dict[str, Callable] = {}
-        _assign_params(ctx, params_assigned_from_default, deprecated_env_vars)
+        _assign_params(
+            ctx, params_assigned_from_default, params_assigned_from_user, deprecated_env_vars
+        )
 
         # Set deprecated_env_var_warnings to be fired later after events have been init.
         object.__setattr__(
@@ -203,7 +219,10 @@ def _assign_params(
             invoked_subcommand.ignore_unknown_options = True
             invoked_subcommand_ctx = invoked_subcommand.make_context(None, sys.argv)
             _assign_params(
-                invoked_subcommand_ctx, params_assigned_from_default, deprecated_env_vars
+                invoked_subcommand_ctx,
+                params_assigned_from_default,
+                params_assigned_from_user,
+                deprecated_env_vars,
             )
 
         if not project_flags:
@@ -378,7 +397,11 @@ def add_fn(x):
         if k == "macro" and command == CliCommand.RUN_OPERATION:
             add_fn(v)
         # None is a Singleton, False is a Flyweight, only one instance of each.
-        elif v is None or v is False:
+        elif (v is None or v is False) and k not in (
+            # These are None by default but they do not support --no-{flag}
+            "defer_state",
+            "log_format",
+        ):
             add_fn(f"--no-{spinal_cased}")
         elif v is True:
             add_fn(f"--{spinal_cased}")

core/dbt/cli/main.py

@@ -121,11 +121,19 @@ def invoke(self, args: List[str], **kwargs) -> dbtRunnerResult:
 def global_flags(func):
     @p.cache_selected_only
     @p.debug
+    @p.defer
+    @p.deprecated_defer
+    @p.defer_state
+    @p.deprecated_favor_state
     @p.deprecated_print
+    @p.deprecated_state
     @p.enable_legacy_logger
     @p.fail_fast
+    @p.favor_state
+    @p.indirect_selection
     @p.log_cache_events
     @p.log_file_max_bytes
+    @p.log_format
     @p.log_format_file
     @p.log_level
     @p.log_level_file
@@ -141,12 +149,15 @@ def global_flags(func):
     @p.record_timing_info
     @p.send_anonymous_usage_stats
     @p.single_threaded
+    @p.state
     @p.static_parser
     @p.use_colors
     @p.use_colors_file
     @p.use_experimental_parser
     @p.version
     @p.version_check
+    @p.warn_error
+    @p.warn_error_options
     @p.write_json
     @functools.wraps(func)
     def wrapper(*args, **kwargs):
@@ -164,9 +175,6 @@ def wrapper(*args, **kwargs):
 )
 @click.pass_context
 @global_flags
-@p.warn_error
-@p.warn_error_options
-@p.log_format
 @p.show_resource_report
 def cli(ctx, **kwargs):
     """An ELT tool for managing your SQL transformations and data models.
@@ -178,24 +186,16 @@ def cli(ctx, **kwargs):
 @cli.command("build")
 @click.pass_context
 @global_flags
-@p.defer
-@p.deprecated_defer
 @p.exclude
-@p.favor_state
-@p.deprecated_favor_state
 @p.full_refresh
 @p.include_saved_query
-@p.indirect_selection
 @p.profile
 @p.profiles_dir
 @p.project_dir
 @p.resource_type
 @p.select
 @p.selector
 @p.show
-@p.state
-@p.defer_state
-@p.deprecated_state
 @p.store_failures
 @p.target
 @p.target_path
@@ -257,21 +257,14 @@ def docs(ctx, **kwargs):
 @click.pass_context
 @global_flags
 @p.compile_docs
-@p.defer
-@p.deprecated_defer
 @p.exclude
-@p.favor_state
-@p.deprecated_favor_state
 @p.profile
 @p.profiles_dir
 @p.project_dir
 @p.select
 @p.selector
 @p.empty_catalog
 @p.static
-@p.state
-@p.defer_state
-@p.deprecated_state
 @p.target
 @p.target_path
 @p.threads
@@ -328,14 +321,9 @@ def docs_serve(ctx, **kwargs):
 @cli.command("compile")
 @click.pass_context
 @global_flags
-@p.defer
-@p.deprecated_defer
 @p.exclude
-@p.favor_state
-@p.deprecated_favor_state
 @p.full_refresh
 @p.show_output_format
-@p.indirect_selection
 @p.introspect
 @p.profile
 @p.profiles_dir
@@ -344,9 +332,6 @@ def docs_serve(ctx, **kwargs):
 @p.select
 @p.selector
 @p.inline
-@p.state
-@p.defer_state
-@p.deprecated_state
 @p.compile_inject_ephemeral_ctes
 @p.target
 @p.target_path
@@ -376,25 +361,17 @@ def compile(ctx, **kwargs):
 @cli.command("show")
 @click.pass_context
 @global_flags
-@p.defer
-@p.deprecated_defer
 @p.exclude
-@p.favor_state
-@p.deprecated_favor_state
 @p.full_refresh
 @p.show_output_format
 @p.show_limit
-@p.indirect_selection
 @p.introspect
 @p.profile
 @p.profiles_dir
 @p.project_dir
 @p.select
 @p.selector
 @p.inline
-@p.state
-@p.defer_state
-@p.deprecated_state
 @p.target
 @p.target_path
 @p.threads
@@ -514,7 +491,6 @@ def init(ctx, **kwargs):
 @click.pass_context
 @global_flags
 @p.exclude
-@p.indirect_selection
 @p.models
 @p.output
 @p.output_keys
@@ -524,9 +500,6 @@ def init(ctx, **kwargs):
 @p.resource_type
 @p.raw_select
 @p.selector
-@p.state
-@p.defer_state
-@p.deprecated_state
 @p.target
 @p.target_path
 @p.vars
@@ -582,10 +555,6 @@ def parse(ctx, **kwargs):
 @cli.command("run")
 @click.pass_context
 @global_flags
-@p.defer
-@p.deprecated_defer
-@p.favor_state
-@p.deprecated_favor_state
 @p.exclude
 @p.full_refresh
 @p.profile
@@ -594,9 +563,6 @@ def parse(ctx, **kwargs):
 @p.empty
 @p.select
 @p.selector
-@p.state
-@p.defer_state
-@p.deprecated_state
 @p.target
 @p.target_path
 @p.threads
@@ -629,7 +595,6 @@ def run(ctx, **kwargs):
 @p.vars
 @p.profile
 @p.target
-@p.state
 @p.threads
 @p.full_refresh
 @requires.postflight
@@ -654,7 +619,6 @@ def retry(ctx, **kwargs):
 @cli.command("clone")
 @click.pass_context
 @global_flags
-@p.defer_state
 @p.exclude
 @p.full_refresh
 @p.profile
@@ -663,7 +627,6 @@ def retry(ctx, **kwargs):
 @p.resource_type
 @p.select
 @p.selector
-@p.state  # required
 @p.target
 @p.target_path
 @p.threads
@@ -731,9 +694,6 @@ def run_operation(ctx, **kwargs):
 @p.select
 @p.selector
 @p.show
-@p.state
-@p.defer_state
-@p.deprecated_state
 @p.target
 @p.target_path
 @p.threads
@@ -760,19 +720,12 @@ def seed(ctx, **kwargs):
 @cli.command("snapshot")
 @click.pass_context
 @global_flags
-@p.defer
-@p.deprecated_defer
 @p.exclude
-@p.favor_state
-@p.deprecated_favor_state
 @p.profile
 @p.profiles_dir
 @p.project_dir
 @p.select
 @p.selector
-@p.state
-@p.defer_state
-@p.deprecated_state
 @p.target
 @p.target_path
 @p.threads
@@ -815,9 +768,6 @@ def source(ctx, **kwargs):
 @p.project_dir
 @p.select
 @p.selector
-@p.state
-@p.defer_state
-@p.deprecated_state
 @p.target
 @p.target_path
 @p.threads
@@ -851,20 +801,12 @@ def freshness(ctx, **kwargs):
 @cli.command("test")
 @click.pass_context
 @global_flags
-@p.defer
-@p.deprecated_defer
 @p.exclude
-@p.favor_state
-@p.deprecated_favor_state
-@p.indirect_selection
 @p.profile
 @p.profiles_dir
 @p.project_dir
 @p.select
 @p.selector
-@p.state
-@p.defer_state
-@p.deprecated_state
 @p.store_failures
 @p.target
 @p.target_path

core/dbt/context/providers.py

@@ -1406,7 +1406,8 @@ def sql(self) -> Optional[str]:
         if self.model.language == ModelLanguage.sql:  # type: ignore[union-attr]
             # If the model is deferred and the adapter doesn't support zero-copy cloning, then select * from the prod
             # relation
-            if getattr(self.model, "defer_relation", None):
+            # TODO: avoid routing on args.which if possible
+            if getattr(self.model, "defer_relation", None) and self.config.args.which == "clone":
                 # TODO https://github.com/dbt-labs/dbt-core/issues/7976
                 return f"select * from {self.model.defer_relation.relation_name or str(self.defer_relation)}"  # type: ignore[union-attr]
             elif getattr(self.model, "extra_ctes_injected", None):