made better tests for ordered content sets import/export

home/export_ordered_sets.py

@@ -1,16 +1,15 @@
 import copy
 import csv
-import io
 from collections.abc import Iterable
 from dataclasses import asdict, astuple, dataclass, fields
 from math import ceil
 
 from django.http import HttpResponse
-from openpyxl.styles import Border, Font, NamedStyle, Side
+from openpyxl.styles import Font, NamedStyle
 from openpyxl.utils import get_column_letter
 from openpyxl.workbook import Workbook
 from openpyxl.worksheet.worksheet import Worksheet
-from wagtail.query import PageQuerySet  # type: ignore  # No typing available
+from wagtail.query import QuerySet  # type: ignore  # No typing available
 
 
 @dataclass
@@ -22,7 +21,7 @@ class ExportRow:
     name: str
     profile_field: str
     page_slugs: str
-    page: str | None
+    time: int | None
     unit: str | None
     before_or_after: str | None
     contact_field: str | None
@@ -44,7 +43,7 @@ def to_tuple(self) -> tuple[str]:
 class OrderedSetExporter:
     rows: list[ExportRow]
 
-    def __init__(self, queryset: PageQuerySet):
+    def __init__(self, queryset: QuerySet):
         self.queryset = queryset
         self.rows = []
 
@@ -54,23 +53,13 @@ def perform_export(self) -> Iterable[ExportRow]:
                 name=item.name,
                 profile_field=item.profile_field,
                 page_slugs=item.page_slugs,
-                page=item.page,
+                time=item.time,
                 unit=item.unit,
                 before_or_after=str(item.before_or_after),
                 contact_field=item.contact_field,
             )
 
 
-def serialize_list(items: Iterable[str]) -> str:
-    """
-    Uses CSV formatting to seralize a list of strings, handling escaping
-    """
-    output = io.StringIO()
-    writer = csv.writer(output)
-    writer.writerow(items)
-    return output.getvalue().rstrip("\r\n")
-
-
 class OrderedSetsExportWriter:
     rows: Iterable[ExportRow]
 
@@ -104,29 +93,13 @@ def _set_xlsx_styles(wb: Workbook, sheet: Worksheet) -> None:
 
     # Set columns based on best size
     column_widths_in_pts = {
-        "title": 110,
-        "question_type": 110,
-        "tags": 110,
-        "slug": 110,
-        "version": 90,
-        "locale": 50,
-        "high_result_page": 110,
-        "high_inflection": 110,
-        "medium_result_page": 120,
-        "medium_inflection": 110,
-        "low_result_page": 110,
-        "skip_threshold": 110,
-        "skip_high_result_page": 110,
-        "generic_error": 370,
-        "question": 370,
-        "explainer": 370,
-        "error": 370,
-        "min": 110,
-        "max": 110,
-        "answers": 370,
-        "scores": 110,
-        "answer_semantic_ids": 110,
-        "question_semantic_id": 110,
+        "name": 130,
+        "profile_field": 110,
+        "page_slugs": 100,
+        "time": 100,
+        "unit": 110,
+        "before_or_after": 120,
+        "contact_field": 100,
     }
     for column in sheet.iter_cols(max_row=1):
         [cell] = column
@@ -135,13 +108,6 @@ def _set_xlsx_styles(wb: Workbook, sheet: Worksheet) -> None:
             width / adjustment
         )
 
-    # Freeze heading row and side panel, 1 added because it freezes before the column
-    panel_column = get_column_letter(5)
-    sheet.freeze_panes = sheet[f"{panel_column}2"]
-
-    # Borders
-    left_border = Border(left=Side(border_style="thin", color="FF000000"))
-
     # Named Styles
     header_style = NamedStyle(name="header_style")
 
@@ -156,10 +122,6 @@ def _set_xlsx_styles(wb: Workbook, sheet: Worksheet) -> None:
         for cell in row:
             cell.style = header_style
 
-    # Set dividing border for side panel
-    for cell in sheet[f"{panel_column}:{panel_column}"]:
-        cell.border = left_border
-
     # set font on all cells initially to 10pt and row height
     general_font = Font(size=10)
     for index, row in enumerate(sheet.iter_rows()):

home/tests/test_content_import_export.py

@@ -9,6 +9,7 @@
 from queue import Queue
 from typing import Any
 
+import pandas as pd
 import pytest
 from django.core import serializers  # type: ignore
 from django.core.files.base import File  # type: ignore
@@ -124,6 +125,15 @@ def csv2dicts(csv_bytes: bytes) -> ExpDicts:
     return list(csv.DictReader(StringIO(csv_bytes.decode())))
 
 
+def xlsx2dicts(xlsx_bytes: bytes) -> ExpDicts:
+    # Read the Excel file from bytes into a pandas DataFrame
+    excel_file = BytesIO(xlsx_bytes)
+    df = pd.read_excel(excel_file)
+
+    # Convert the DataFrame into a list of dictionaries
+    return df.to_dict(orient="records")
+
+
 DbDict = dict[str, Any]
 DbDicts = Iterable[DbDict]
 
@@ -133,11 +143,14 @@ def _models2dicts(model_instances: Any) -> DbDicts:
 
 
 def get_page_json() -> DbDicts:
-    page_objs = Page.objects.type(ContentPage, ContentPageIndex).all()
+    page_objs = Page.objects.type(
+        ContentPage, ContentPageIndex, OrderedContentSet
+    ).all()
     pages = {p["pk"]: p["fields"] for p in _models2dicts(page_objs)}
     content_pages = [
         *_models2dicts(ContentPage.objects.all()),
         *_models2dicts(ContentPageIndex.objects.all()),
+        *_models2dicts(OrderedContentSet.objects.all()),
     ]
     return [p | {"fields": {**pages[p["pk"]], **p["fields"]}} for p in content_pages]
 
@@ -427,17 +440,14 @@ def export_content(self, locale: str | None = None) -> bytes:
             print("-^-CONTENT-^-")
         return content
 
-    def export_ordered_content(self, locale: str | None = None) -> bytes:
+    def export_ordered_content(self) -> bytes:
         """
         Export ordered content in the configured format.
         """
-        url = f"/admin/snippets/home/orderedcontentset/={self.format}"
-        content = self.admin_client.get(url).content
+        url = f"/admin/snippets/home/orderedcontentset/?export={self.format}"
 
-        if self.format == "csv":
-            print("-v-CONTENT-v-")
-            print(content.decode())
-            print("-^-CONTENT-^-")
+        stream = self.admin_client.get(url)
+        content = b"".join(stream.streaming_content)
         return content
 
     def import_content(self, content_bytes: bytes, **kw: Any) -> None:
@@ -462,6 +472,16 @@ def import_file(
         self.import_content(content, **kw)
         return content
 
+    def import_ordered_file(
+        self, path_str: str, path_base: Path = IMP_EXP_DATA_BASE, **kw: Any
+    ) -> bytes:
+        """
+        Import given ordered content file in the configured format with the configured importer.
+        """
+        content = self.read_bytes(path_str, path_base)
+        self.import_ordered_sets(content, **kw)
+        return content
+
     def export_reimport(self) -> None:
         """
         Export all content, then immediately reimport it.
@@ -486,6 +506,11 @@ def csvs2dicts(self, src_bytes: bytes, dst_bytes: bytes) -> ExpDictsPair:
         dst = csv2dicts(dst_bytes)
         return filter_exports(src, dst)
 
+    def xlsxs2dicts(self, src_bytes: bytes, dst_bytes: bytes) -> ExpDictsPair:
+        src = xlsx2dicts(src_bytes)
+        dst = xlsx2dicts(dst_bytes)
+        return filter_exports(src, dst)
+
 
 @pytest.fixture()
 def csv_impexp(request: Any, admin_client: Any) -> ImportExport:
@@ -1316,28 +1341,33 @@ class TestExport:
         container for related tests.
     """
 
-    def test_ordered_content_set_export(self, impexp: ImportExport) -> None:
+    def test_ordered_content_set_export(self, csv_impexp: ImportExport) -> None:
         """
         Ordered Content Sets should export all pages correctly
         """
-        ordered_content_set = OrderedContentSet(name="Test Title")
-        ordered_content_set.save()
+        csv_impexp.import_file("contentpage_required_fields.csv")
 
-        content = impexp.export_ordered_content()
-        # Export should succeed
-        assert content is not None
+        imported_content = csv_impexp.import_ordered_file("ordered_content.csv")
+
+        exported_content = csv_impexp.export_ordered_content()
+
+        src, dst = csv_impexp.csvs2dicts(imported_content, exported_content)
+        assert src == dst
 
-    def test_ordered_content_XLSX_export(self, xlsx_impexp: ImportExport) -> None:
+    def test_ordered_content_XLSX_export(
+        self, xlsx_impexp: ImportExport, csv_impexp: ImportExport
+    ) -> None:
         """
         Ordered Content Sets should export in XLSX format correctly
         """
-        ordered_content_set = OrderedContentSet(name="Test Title")
-        ordered_content_set.save()
+        csv_impexp.import_file("contentpage_required_fields.csv")
 
-        # exports in xlsx format
-        content = xlsx_impexp.export_ordered_content()
+        imported_content = xlsx_impexp.import_ordered_file("ordered_content.xlsx")
 
-        assert content is not None
+        exported_content = xlsx_impexp.export_ordered_content()
+
+        src, dst = csv_impexp.xlsxs2dicts(imported_content, exported_content)
+        assert src == dst
 
     def test_export_wa_with_image(self, impexp: ImportExport) -> None:
         img_path = Path("home/tests/test_static") / "test.jpeg"