Fix: Export data caterer not using server details (#430)

* Add in export to data-caterer YAML format

* Fix: Use server details from data contract when exporting to data caterer format, use location as file pathway if available

* Allow for data caterer export to use specified server information, remove entry from changelog

datacontract/export/data_caterer_converter.py

@@ -16,15 +16,27 @@ def export(self, data_contract, model, server, sql_server_type, export_args) ->
         return to_data_caterer_generate_yaml(data_contract, server)
 
 
-def to_data_caterer_generate_yaml(data_contract_spec: DataContractSpecification, server: Server):
+def to_data_caterer_generate_yaml(data_contract_spec: DataContractSpecification, server):
     generation_task = {"name": data_contract_spec.info.title, "steps": []}
+    server_info = _get_server_info(data_contract_spec, server)
 
     for model_key, model_value in data_contract_spec.models.items():
-        odcs_table = _to_data_caterer_generate_step(model_key, model_value, server)
+        odcs_table = _to_data_caterer_generate_step(model_key, model_value, server_info)
         generation_task["steps"].append(odcs_table)
     return yaml.dump(generation_task, indent=2, sort_keys=False, allow_unicode=True)
 
 
+def _get_server_info(data_contract_spec: DataContractSpecification, server):
+    if server is not None and server in data_contract_spec.servers:
+        return data_contract_spec.servers.get(server)
+    elif server is not None:
+        raise Exception(f"Server name not found in servers list in data contract, server-name={server}")
+    elif len(data_contract_spec.servers.keys()) > 0:
+        return next(iter(data_contract_spec.servers.values()))
+    else:
+        return None
+
+
 def _to_data_caterer_generate_step(model_key, model_value: Model, server: Server) -> dict:
     step = {
         "name": model_key,
@@ -52,7 +64,12 @@ def _to_data_source_options(model_key, server: Server):
     options = {}
     if server is not None and server.type is not None:
         if server.type in ["s3", "gcs", "azure", "local"]:
-            options["path"] = server.path
+            if server.path is not None:
+                options["path"] = server.path
+            elif server.location is not None:
+                options["path"] = server.location
+            else:
+                options["path"] = "/tmp/data_caterer_data"
         elif server.type == "postgres":
             options["schema"] = server.schema_
             options["table"] = model_key

tests/fixtures/data-caterer/export/datacontract_nested.yaml

@@ -14,11 +14,16 @@ terms:
   billing: free
   noticePeriod: P3M
 servers:
-  production:
-    type: snowflake
-    account: my-account
-    database: my-database
-    schema: my-schema
+  s3-json:
+    type: s3
+    location: s3://covid19-lake/enigma-jhu/json/*.json
+    format: json
+    delimiter: new_line
+  s3-json-prod:
+    type: s3
+    location: s3://covid19-lake-prod/enigma-jhu/json/*.json
+    format: json
+    delimiter: new_line
 models:
   orders:
     description: The orders model

tests/test_export_data_caterer.py

@@ -6,7 +6,7 @@
 
 from datacontract.cli import app
 from datacontract.export.data_caterer_converter import to_data_caterer_generate_yaml
-from datacontract.model.data_contract_specification import DataContractSpecification, Server
+from datacontract.model.data_contract_specification import DataContractSpecification
 
 
 def test_cli():
@@ -19,12 +19,34 @@ def test_to_data_caterer():
     data_contract = DataContractSpecification.from_string(
         read_file("fixtures/data-caterer/export/datacontract_nested.yaml")
     )
-    expected_data_caterer_model = """
+    expected_data_caterer_model = _get_expected_data_caterer_yaml("s3://covid19-lake/enigma-jhu/json/*.json")
+
+    data_caterer_yaml = to_data_caterer_generate_yaml(data_contract, None)
+    result = yaml.safe_load(data_caterer_yaml)
+
+    assert result == yaml.safe_load(expected_data_caterer_model)
+
+
+def test_to_data_caterer_with_server():
+    data_contract = DataContractSpecification.from_string(
+        read_file("fixtures/data-caterer/export/datacontract_nested.yaml")
+    )
+    expected_data_caterer_model = _get_expected_data_caterer_yaml("s3://covid19-lake-prod/enigma-jhu/json/*.json")
+
+    data_caterer_yaml = to_data_caterer_generate_yaml(data_contract, "s3-json-prod")
+    result = yaml.safe_load(data_caterer_yaml)
+
+    assert result == yaml.safe_load(expected_data_caterer_model)
+
+
+def _get_expected_data_caterer_yaml(path: str):
+    return f"""
 name: Orders Unit Test
 steps:
 - name: orders
-  type: csv
-  options: {}
+  type: json
+  options:
+    path: {path}
   schema:
   - name: order_id
     type: string
@@ -58,11 +80,6 @@ def test_to_data_caterer():
         type: string
 """
 
-    data_caterer_yaml = to_data_caterer_generate_yaml(data_contract, Server())
-    result = yaml.safe_load(data_caterer_yaml)
-
-    assert result == yaml.safe_load(expected_data_caterer_model)
-
 
 def read_file(file):
     if not os.path.exists(file):