Import support

demo/demo_schematics_export.py

@@ -0,0 +1,88 @@
+from schematics import Model, types, models
+
+from export_util import normalize, writer, Exporter, utility
+
+
+@models.metaclass(utility.ExportableModelMeta)
+class OrderedModel(Model):
+    class Options:
+        export_order = True
+        serialize_when_none = False
+
+
+class Composer(OrderedModel):
+    first_name = types.StringType(serialized_name="First Name")
+    last_name = types.StringType()
+
+
+class Codes(OrderedModel):
+    isan = types.StringType(serialized_name="ISAN")
+    gema = types.StringType(serialized_name="GEMA")
+
+
+class Track(OrderedModel):
+    name = types.StringType(serialized_name="Name")
+    duration = types.IntType(serialized_name="Length")
+    modes = types.ListType(types.IntType)
+    composers = types.ListType(types.ModelType(Composer), serialized_name="Composers")
+    codes = types.ModelType(Codes)
+
+
+if __name__ == '__main__':
+    tracks = [
+        Track({
+            "name": "First Track",
+            "duration": 44,
+            "modes": [1, 2, 3],
+            "codes": {
+                "isan": "ISAN_CODE",
+                "GEMA": "GEMA_CODE",
+            },
+            "composers": [{
+                "first_name": "Composer1",
+                "last_name": "John",
+            }, {
+                "first_name": "Composer2",
+                "last_name": "Doe",
+            }]
+        }),
+        Track({
+            "name": "Second Track",
+            "duration": 21,
+            "modes": [2, 3, 4],
+            "codes": {
+                "isan": "ISAN_CODE2",
+                "GEMA": "GEMA_CODE2",
+            },
+            "composers": [{
+                "first_name": "Composer2.1",
+                "last_name": "John2",
+            }, {
+                "first_name": "Composer2.2",
+                "last_name": "Doe2",
+            }]
+        }),
+    ]
+
+    ex = Exporter(
+        normalizer=normalize.SchematicsNormalizer(
+            model=Track,
+            translate={
+                # If the field has `serialized_name` provided in a
+                # model property, it must be used as a key to rename
+                # it.
+                'Name': 'Track Name Renamed',
+                'First Name': 'First Name',
+
+                # Otherwise - use the property name as a key. So we can
+                # use `last_name` key to override this field name in
+                # a result table.
+                'last_name': 'Last Name',
+            }
+        ),
+        output=writer.XLSXBytesOutputWriter()
+    )
+
+    filename, mime, xls_data = ex.generate(tracks, "metadata")
+    with open('demo_schematics_export.xlsx', 'wb') as f:
+        f.write(xls_data)

demo/demo_schematics_import.py

@@ -0,0 +1,57 @@
+from schematics import Model, types, models
+
+from export_util import normalize, Importer, utility, value
+
+
+@models.metaclass(utility.ExportableModelMeta)
+class OrderedModel(Model):
+    class Options:
+        export_order = True
+        serialize_when_none = False
+
+
+class Composer(OrderedModel):
+    first_name = types.StringType(serialized_name="First Name")
+    last_name = types.StringType()
+
+
+class Codes(OrderedModel):
+    isan = types.StringType(serialized_name="ISAN")
+    gema = types.StringType(serialized_name="GEMA")
+
+
+class Track(OrderedModel):
+    name = types.StringType(serialized_name="Name")
+    duration = types.IntType(serialized_name="Length")
+    modes = types.ListType(types.IntType)
+    composers = types.ListType(types.ModelType(Composer), serialized_name="Composers")
+    codes = types.ModelType(Codes)
+
+    class Options:
+        normalize = {
+            'modes': value.string_to_any
+        }
+
+
+if __name__ == '__main__':
+    im = Importer(
+        normalizer=normalize.SchematicsParseNormalizer(
+            model=Track,
+            translate={
+                # If the field has `serialized_name` provided in a
+                # model property, it must be used as a key to rename
+                # it.
+                'Name': 'Track Name Renamed',
+                'First Name': 'First Name',
+
+                # Otherwise - use the property name as a key. So we can
+                # use `last_name` key to override this field name in
+                # a result table.
+                'last_name': 'Last Name',
+            }
+        )
+    )
+
+    objects = im.parse('demo/demo_schematics_import.xlsx')
+    for o in objects:
+        print(value.schema_model_to_dict(o))

export_util/__init__.py

@@ -102,4 +102,63 @@ def _get_content(self, data):
             yield row
 
 
-__all__ = ['Exporter']
+class Importer:
+    """
+    Import data utility.
+
+
+    """
+    def __init__(self, normalizer):
+        """
+        :param export_util.normalize.ParseNormalizer normalizer:
+        """
+        self.normal = normalizer
+
+    def parse(self, filename):
+        """
+        Parsing matrix and normalize it using normalizer.
+        :param filename:
+        :return:
+        """
+        for matrix in self._parse_matrix(filename):
+            yield self.normal.model(self.normal.read(matrix))
+
+    def _parse_matrix(self, filename):
+        """
+        Parser should separate objects and yield each of them
+        separately to be able to normalize output.
+
+        Each object should contain headers like it's a single
+        object for whole document.
+        :param filename:
+        :return:
+        """
+        from openpyxl import load_workbook
+        wb = load_workbook(filename=filename)
+        sheet = wb.active
+
+        matrix = []
+        headers = None
+        for i, row in enumerate(sheet.rows):
+            rowdata = list(map(lambda x: x.value, row))
+
+            if i == 0:
+                headers = rowdata
+                continue
+
+            if not len(matrix):
+                matrix.append(rowdata)
+                continue
+
+            if not rowdata[0]:
+                matrix.append(rowdata)
+                continue
+
+            yield [headers] + matrix
+            matrix = [rowdata]
+
+        if matrix:
+            yield [headers] + matrix
+
+
+__all__ = ['Exporter', 'Importer']

export_util/normalize.py

@@ -2,6 +2,7 @@
 
 import schematics
 
+from export_util.utility.misc import NestedDict
 from export_util import (
     template as tpl,
     value as val
@@ -393,6 +394,8 @@ def _get_model_renderable_fields(self, model: schematics.models.Model):
                     raise NotImplementedError(f'{model.__name__}.{getter} is not implemented')
 
                 getter = getattr(model, getter)
+                getter.name = field_name
+                getter.serialized_name = field_name.replace('_', ' ').capitalize()
                 getter.name = getter.serialized_name = field_name
 
                 # Define preformatters and prepend getter
@@ -443,7 +446,133 @@ def _get_field_path(self, parent, field_name):
         return '.'.join([parent or '', field_name]).strip('.')
 
 
+class ParseNormalizer(SchematicsNormalizer):
+    def _create_map(self):
+        """
+        Creates map to read matrix.
+        :return:
+        """
+
+    def read(self, matrix):
+        """
+        Reads list[shape(N, N)] using builded map.
+        :param matrix:
+        :return:
+        """
+        raise NotImplementedError()
+
+
+class SchematicsParseNormalizer(ParseNormalizer):
+    """
+    Parse normalizer allows to read exported schematics document.
+
+    Example:
+
+         >> im = Importer(normalize.SchematicsParseNormalizer(Track))
+         >> im.parse('doc.xlsx')
+         >>
+         [
+             {
+                'name': 'First Track',
+                'duration': 44,
+                'modes': [1, 2, 3],
+                'composers': [<Composer instance>, <Composer instance>],
+                'codes': <Codes instance>
+             },
+             {
+                'name': 'Second Track',
+                'duration': 21,
+                'modes': [2, 3, 4],
+                'composers': [<Composer instance>, <Composer instance>],
+                'codes': <Codes instance>
+             }
+         ]
+
+
+    """
+    def __init__(self, model: schematics.Model, *args, **kwargs):
+        """
+        Create objects template from schematics model.
+        """
+        self.model = model
+        super(SchematicsParseNormalizer, self).__init__(model, *args, **kwargs)
+
+    def read(self, matrix):
+        # Building parse map
+        pmap = NestedDict()
+        for field in self.template.fields:
+            pmap.update(self.parse_field(field, matrix))
+        return pmap
+
+    def parse_field(self, field, matrix):
+        if hasattr(field, 'fields'):
+            if self.model.fields[field.value_path].primitive_type == list:
+                return self.parse_list_related_field(
+                    field=field,
+                    matrix=self._crop_matrix(matrix, field.column-1, 0, field.length)
+                )
+
+            if self.model.fields[field.value_path].primitive_type == dict:
+                return self.parse_related_field(
+                    field=field,
+                    matrix=self._crop_matrix(matrix, field.column-1, 0, field.length)
+                )
+
+        return self.parse_normal_field(field, matrix)
+
+    def parse_normal_field(self, field, matrix):
+        matrix = self._crop_titles(matrix)
+        value = matrix[0][field.column-1]
+        return {field.value_path: self._get_model_normal_field(field, value)}
+
+    def parse_list_related_field(self, field, matrix):
+        matrix = self._crop_titles(matrix, field)
+
+        data = []
+        for row in matrix:
+            obj = NestedDict()
+            for nested in field.fields:
+                obj[nested.value_path] = self._get_model_normal_field(nested, row[nested.column-1])
+            data.append(obj)
+        return {field.value_path: data}
+
+    def parse_related_field(self, field, matrix):
+        matrix = self._crop_titles(matrix, field)
+
+        data = []
+        for row in matrix:
+            obj = NestedDict()
+            for nested in field.fields:
+                obj[nested.value_path] = self._get_model_normal_field(nested, row[nested.column-1])
+            data.append(obj)
+
+        return {field.value_path: data[0]}
+
+    def _get_model_normal_field(self, field, value):
+        if 'normalize' in dict(self.model._options) and self.model._options.normalize is not None:
+            normalizer = self.model._options.normalize.get(field.value_path)
+            if callable(normalizer):
+                return normalizer(value)
+            return value
+
+    def _crop_titles(self, matrix, field=None):
+        if self.template.render_titles:
+            matrix = self._crop_matrix(matrix, 0, 1)
+        if field and self.template.inline:
+            matrix = self._crop_matrix(matrix, 0, 1)
+        if field and hasattr(field, 'render_titles') and field.render_titles:
+            matrix = self._crop_matrix(matrix, 0, 1)
+        return matrix
+
+    def _crop_matrix(self, matrix, x, y, width=None, height=None):
+        result = []
+        for row in matrix[y:y+height if height else None]:
+            result.append(row[x:x+width if width else None])
+        return result
+
+
 __all__ = [
     'Normalizer',
     'SchematicsNormalizer',
+    'SchematicsParseNormalizer',
 ]

export_util/utility/misc.py

@@ -0,0 +1,17 @@
+
+
+class NestedDict(dict):
+    """
+    Returns soft python-dictionary each item of which
+    can be set easily without setting parent dictionary.
+
+        >> s = NestedDict()
+        >> s['a']['b']['c'] = 1
+        {'a': {'b': {'c': 1}}}
+
+    """
+    def __getitem__(self, item):
+        if item not in self:
+            self[item] = NestedDict()
+
+        return super(NestedDict, self).__getitem__(item)

export_util/utility/schematics.py

@@ -9,7 +9,7 @@ def __init__(self, namespace=None, roles=None, export_level=schema.DEFAULT,
                  serialize_when_none=None, export_order=False, fields=None,
                  verbose_name=None, preformat=None, offset_top=None, inline=None,
                  offset_item=None, titles=None, fold_nested=None, title_each=None,
-                 extras=None):
+                 extras=None, normalize=None):
         super(ExportLibSchemaOptions, self).__init__(namespace, roles, export_level, serialize_when_none, export_order,
                                                      extras)
 
@@ -26,6 +26,8 @@ def __init__(self, namespace=None, roles=None, export_level=schema.DEFAULT,
         self.fold_nested = fold_nested
         self.verbose_name = verbose_name
 
+        self.normalize = normalize
+
 
 class ExportableModelMeta(models.ModelMeta):
     def __new__(mcs, name, bases, attrs):