Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add enabled as a source config #5008

Merged
merged 22 commits into from
Apr 12, 2022
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions .changes/unreleased/Features-20220408-132725.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
kind: Features
body: add enabled as a source config
time: 2022-04-08T13:27:25.292454-04:00
custom:
Author: nathaniel-may
Issue: "3662"
PR: "5008"
34 changes: 34 additions & 0 deletions core/dbt/contracts/graph/model_config.py
Original file line number Diff line number Diff line change
Expand Up @@ -335,6 +335,40 @@ def replace(self, **kwargs):
@dataclass
class SourceConfig(BaseConfig):
enabled: bool = True
# to be implmented to complete CT-201
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

👍

# quoting: Dict[str, Any] = field(
# default_factory=dict,
# metadata=MergeBehavior.Update.meta(),
# )
# freshness: Optional[Dict[str, Any]] = field(
# default=None,
# metadata=CompareBehavior.Exclude.meta(),
# )
# loader: Optional[str] = field(
# default=None,
# metadata=CompareBehavior.Exclude.meta(),
# )
# # TODO what type is this? docs say: "<column_name_or_expression>"
# loaded_at_field: Optional[str] = field(
# default=None,
# metadata=CompareBehavior.Exclude.meta(),
# )
# database: Optional[str] = field(
# default=None,
# metadata=CompareBehavior.Exclude.meta(),
# )
# schema: Optional[str] = field(
# default=None,
# metadata=CompareBehavior.Exclude.meta(),
# )
# meta: Dict[str, Any] = field(
# default_factory=dict,
# metadata=MergeBehavior.Update.meta(),
# )
# tags: Union[List[str], str] = field(
# default_factory=list_str,
# metadata=metas(ShowBehavior.Hide, MergeBehavior.Append, CompareBehavior.Exclude),
# )


@dataclass
Expand Down
2 changes: 2 additions & 0 deletions core/dbt/contracts/graph/unparsed.py
Original file line number Diff line number Diff line change
Expand Up @@ -242,6 +242,7 @@ class Quoting(dbtClassMixin, Mergeable):

@dataclass
class UnparsedSourceTableDefinition(HasColumnTests, HasTests):
config: Dict[str, Any] = field(default_factory=dict)
gshank marked this conversation as resolved.
Show resolved Hide resolved
loaded_at_field: Optional[str] = None
identifier: Optional[str] = None
quoting: Quoting = field(default_factory=Quoting)
Expand Down Expand Up @@ -322,6 +323,7 @@ class SourcePatch(dbtClassMixin, Replaceable):
path: Path = field(
metadata=dict(description="The path to the patch-defining yml file"),
)
config: Dict[str, Any] = field(default_factory=dict)
description: Optional[str] = None
meta: Optional[Dict[str, Any]] = None
database: Optional[str] = None
Expand Down
2 changes: 2 additions & 0 deletions core/dbt/parser/manifest.py
Original file line number Diff line number Diff line change
Expand Up @@ -465,6 +465,8 @@ def parse_project(
else:
dct = block.file.dict_from_yaml
parser.parse_file(block, dct=dct)
# Came out of here with UnpatchedSourceDefinition containing configs at the source level
# and not configs at the table level (as expected)
else:
parser.parse_file(block)
project_parsed_path_count += 1
Expand Down
24 changes: 16 additions & 8 deletions core/dbt/parser/sources.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
import itertools
from pathlib import Path
from typing import Iterable, Dict, Optional, Set, List, Any
from typing import Iterable, Dict, Optional, Set, Any
from dbt.adapters.factory import get_adapter
from dbt.config import RuntimeConfig
from dbt.context.context_config import (
Expand Down Expand Up @@ -137,15 +137,13 @@ def parse_source(self, target: UnpatchedSourceDefinition) -> ParsedSourceDefinit
tags = sorted(set(itertools.chain(source.tags, table.tags)))

config = self._generate_source_config(
fqn=target.fqn,
target=target,
rendered=True,
project_name=target.package_name,
)

unrendered_config = self._generate_source_config(
fqn=target.fqn,
target=target,
rendered=False,
project_name=target.package_name,
)

if not isinstance(config, SourceConfig):
Expand Down Expand Up @@ -261,19 +259,29 @@ def parse_source_test(
)
return node

def _generate_source_config(self, fqn: List[str], rendered: bool, project_name: str):
def _generate_source_config(self, target: UnpatchedSourceDefinition, rendered: bool):
generator: BaseContextConfigGenerator
if rendered:
generator = ContextConfigGenerator(self.root_project)
else:
generator = UnrenderedConfigGenerator(self.root_project)

# configs with precendence set
precedence_configs = dict()
# first apply source configs
precedence_configs.update(target.source.config)
# then overrite anything that is defined on source tables
# this is not quite complex enough for configs that can be set as top-level node keys, but
# it works while source configs can only include `enabled`.
Comment on lines +274 to +275
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is so useful for future us.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

In the other nodes configs are merged by the code in the BaseConfig object, in the 'update_from' method, which handles merging things like tags and meta, etc. (MergeBehavior, etc). Which is called by 'calculate_node_config'. I imagine you'll need to use some of that code to handle merging the Source configs with the SourceTable configs?

precedence_configs.update(target.table.config)

return generator.calculate_node_config(
config_call_dict={},
fqn=fqn,
fqn=target.fqn,
resource_type=NodeType.Source,
project_name=project_name,
project_name=target.package_name,
base=False,
patch_config_dict=precedence_configs,
)

def _get_relation_name(self, node: ParsedSourceDefinition):
Expand Down
3 changes: 3 additions & 0 deletions test/unit/test_contracts_graph_unparsed.py
Original file line number Diff line number Diff line change
Expand Up @@ -308,6 +308,7 @@ def test_table_defaults(self):
to_dict = {
'name': 'foo',
'description': '',
'config': {},
'loader': '',
'freshness': {'error_after': {}, 'warn_after': {}},
'quoting': {},
Expand All @@ -316,6 +317,7 @@ def test_table_defaults(self):
{
'name': 'table1',
'description': '',
'config': {},
'docs': {'show': True},
'tests': [],
'columns': [],
Expand All @@ -327,6 +329,7 @@ def test_table_defaults(self):
{
'name': 'table2',
'description': 'table 2',
'config': {},
'docs': {'show': True},
'tests': [],
'columns': [],
Expand Down
50 changes: 0 additions & 50 deletions tests/functional/permission/test_permissions.py

This file was deleted.

Loading