Merge pull request #2015 from tayloramurphy/add-data-dictionary

Handle "meta:" key in schema/sources.yml
dbt-labs · Jan 7, 2020 · 0a5b436 · 0a5b436
2 parents 4ed1986 + dea5f98
commit 0a5b436
Show file tree

Hide file tree

Showing 10 changed files with 263 additions and 50 deletions.
diff --git a/core/dbt/contracts/graph/parsed.py b/core/dbt/contracts/graph/parsed.py
@@ -119,6 +119,7 @@ def __len__(self):
 class ColumnInfo(JsonSchemaMixin, Replaceable):
     name: str
     description: str = ''
+    meta: Dict[str, Any] = field(default_factory=dict)
     data_type: Optional[str] = None
 
 
@@ -180,6 +181,7 @@ def patch(self, patch):
         self.description = patch.description
         self.columns = patch.columns
         self.docrefs = patch.docrefs
+        self.meta = patch.meta
         if dbt.flags.STRICT_MODE:
             self.to_dict(validate=True)
 
@@ -215,6 +217,7 @@ class ParsedNodeDefaults(ParsedNodeMandatory):
     docrefs: List[Docref] = field(default_factory=list)
     description: str = field(default='')
     columns: Dict[str, ColumnInfo] = field(default_factory=dict)
+    meta: Dict[str, Any] = field(default_factory=dict)
     patch_path: Optional[str] = None
     build_path: Optional[str] = None
 
@@ -455,6 +458,7 @@ class ParsedNodePatch(JsonSchemaMixin, Replaceable):
     original_file_path: str
     columns: Dict[str, ColumnInfo]
     docrefs: List[Docref]
+    meta: Dict[str, Any]
 
 
 @dataclass
@@ -507,6 +511,8 @@ class ParsedSourceDefinition(
     docrefs: List[Docref] = field(default_factory=list)
     description: str = ''
     columns: Dict[str, ColumnInfo] = field(default_factory=dict)
+    meta: Dict[str, Any] = field(default_factory=dict)
+    source_meta: Dict[str, Any] = field(default_factory=dict)
 
     @property
     def is_ephemeral_model(self):

diff --git a/core/dbt/contracts/graph/unparsed.py b/core/dbt/contracts/graph/unparsed.py
@@ -58,6 +58,7 @@ class UnparsedRunHook(UnparsedNode):
 class NamedTested(JsonSchemaMixin, Replaceable):
     name: str
     description: str = ''
+    meta: Dict[str, Any] = field(default_factory=dict)
     data_type: Optional[str] = None
     tests: Optional[List[Union[Dict[str, Any], str]]] = None
 
@@ -159,6 +160,7 @@ class ExternalPartition(AdditionalPropertiesAllowed, Replaceable):
     name: str = ''
     description: str = ''
     data_type: str = ''
+    meta: Dict[str, Any] = field(default_factory=dict)
 
     def __post_init__(self):
         if self.name == '' or self.data_type == '':
@@ -206,6 +208,7 @@ def __post_init__(self):
 class UnparsedSourceDefinition(JsonSchemaMixin, Replaceable):
     name: str
     description: str = ''
+    meta: Dict[str, Any] = field(default_factory=dict)
     database: Optional[str] = None
     schema: Optional[str] = None
     loader: str = ''

diff --git a/core/dbt/parser/schemas.py b/core/dbt/parser/schemas.py
@@ -66,10 +66,11 @@ def __init__(self):
         self.column_info: Dict[str, ColumnInfo] = {}
         self.docrefs: List[Docref] = []
 
-    def add(self, column_name, description, data_type):
+    def add(self, column_name, description, data_type, meta):
         self.column_info[column_name] = ColumnInfo(name=column_name,
                                                    description=description,
-                                                   data_type=data_type)
+                                                   data_type=data_type,
+                                                   meta=meta)
 
 
 def collect_docrefs(
@@ -222,9 +223,10 @@ def parse_column(
         column_name = column.name
         description = column.description
         data_type = column.data_type
+        meta = column.meta
         collect_docrefs(block.target, refs, column_name, description)
 
-        refs.add(column_name, description, data_type)
+        refs.add(column_name, description, data_type, meta)
 
         if not column.tests:
             return
@@ -340,6 +342,7 @@ def generate_source_node(
             NodeType.Source, self.project.project_name, source.name, table.name
         ])
         description = table.description or ''
+        meta = table.meta or {}
         source_description = source.description or ''
         collect_docrefs(source, refs, None, description, source_description)
 
@@ -348,6 +351,7 @@ def generate_source_node(
         freshness = self._calculate_freshness(source, table)
         quoting = source.quoting.merged(table.quoting)
         path = block.path.original_file_path
+        source_meta = source.meta or {}
 
         return ParsedSourceDefinition(
             package_name=self.project.project_name,
@@ -364,6 +368,8 @@ def generate_source_node(
             external=table.external,
             source_name=source.name,
             source_description=source_description,
+            source_meta=source_meta,
+            meta=meta,
             loader=source.loader,
             docrefs=refs.docrefs,
             loaded_at_field=loaded_at_field,
@@ -385,7 +391,8 @@ def generate_node_patch(
             original_file_path=block.path.original_file_path,
             description=description,
             columns=refs.column_info,
-            docrefs=refs.docrefs
+            docrefs=refs.docrefs,
+            meta=block.target.meta,
         )
 
     def parse_target_model(