[NEAT-222] DMS (and all other rules) dump order 🖶

Makefile

@@ -1,6 +1,6 @@
 .PHONY: run-explorer run-tests run-linters build-ui build-python build-docker run-docker compose-up
 
-version="0.75.9"
+version="0.77.0"
 run-explorer:
 	@echo "Running explorer API server..."
 	# open "http://localhost:8000/static/index.html" || true

cognite/neat/_version.py

@@ -1 +1 @@
-__version__ = "0.75.9"
+__version__ = "0.77.0"

cognite/neat/rules/exporters/_rules2dms.py

@@ -115,11 +115,11 @@ def export(self, rules: Rules) -> DMSSchema:
         )
         is_new_model = dms_rules.reference is None
         if is_new_model or is_solution_model:
-            return dms_rules.as_schema(self.export_pipeline, self.instance_space)
+            return dms_rules.as_schema(False, self.export_pipeline, self.instance_space)
 
         # This is an extension of an existing model.
         reference_rules = cast(DMSRules, dms_rules.reference).model_copy(deep=True)
-        reference_schema = reference_rules.as_schema(self.export_pipeline)
+        reference_schema = reference_rules.as_schema(include_ref=False, include_pipeline=self.export_pipeline)
 
         # Todo Move this to an appropriate location
         # Merging Reference with User Rules
@@ -142,7 +142,7 @@ def export(self, rules: Rules) -> DMSSchema:
                 property_.reference = None
                 combined_rules.properties.append(property_)
 
-        schema = combined_rules.as_schema(self.export_pipeline, self.instance_space)
+        schema = combined_rules.as_schema(True, self.export_pipeline, self.instance_space)
 
         if dms_rules.metadata.extension in (ExtensionCategory.addition, ExtensionCategory.reshape):
             # We do not freeze views as they might be changed, even for addition,

cognite/neat/rules/exporters/_rules2excel.py

@@ -120,13 +120,6 @@ def _write_sheets(self, workbook: Workbook, dumped_rules: dict[str, Any], rules:
             else:
                 sheet = workbook.create_sheet(sheet_name)
 
-            # Reorder such that the first column is class + the first field of the subclass
-            # of sheet entity. This is to make the properties/classes/views/containers sheet more readable.
-            # For example, for the properties these that means class, property, name, description
-            # instead of class, name, description, property
-            move = len(SheetEntity.model_fields) - 1  # -1 is for the class field
-            headers = headers[:1] + headers[move : move + 1] + headers[1:move] + headers[move + 1 :]
-
             main_header = self._main_header_by_sheet_name[sheet_name]
             sheet.append([main_header] + [""] * (len(headers) - 1))
             sheet.merge_cells(start_row=1, start_column=1, end_row=1, end_column=len(headers))
@@ -150,8 +143,6 @@ def _write_sheets(self, workbook: Workbook, dumped_rules: dict[str, Any], rules:
                         cell.border = Border(left=side, right=side, top=side, bottom=side)
                     fill_color = next(fill_colors)
 
-                # Need to do the same reordering as for the headers above
-                row = row[:1] + row[move : move + 1] + row[1:move] + row[move + 1 :]
                 sheet.append(row)
                 if self._styling_level > 2 and is_properties:
                     for cell in sheet[sheet.max_row]:

cognite/neat/rules/importers/_base.py

@@ -60,6 +60,12 @@ def _to_output(
         else:
             return output, issues
 
+    @classmethod
+    def _return_or_raise(cls, issue_list: IssueList, errors: Literal["raise", "continue"]) -> tuple[None, IssueList]:
+        if errors == "raise":
+            raise issue_list.as_errors()
+        return None, issue_list
+
     def _default_metadata(self):
         return {
             "prefix": "neat",

cognite/neat/rules/importers/_spreadsheet2rules.py

@@ -229,12 +229,6 @@ def to_rules(
             role=role,
         )
 
-    @classmethod
-    def _return_or_raise(cls, issue_list: IssueList, errors: Literal["raise", "continue"]) -> tuple[None, IssueList]:
-        if errors == "raise":
-            raise issue_list.as_errors()
-        return None, issue_list
-
 
 class GoogleSheetImporter(BaseImporter):
     def __init__(self, sheet_id: str, skiprows: int = 1):

cognite/neat/rules/issues/base.py

@@ -78,6 +78,9 @@ def from_pydantic_errors(cls, errors: list[ErrorDetails], **kwargs) -> "list[Nea
                 all_errors.append(DefaultPydanticError.from_pydantic_error(error))
         return all_errors
 
+    def as_exception(self) -> Exception:
+        return ValueError(self.message())
+
 
 @dataclass(frozen=True)
 class DefaultPydanticError(NeatValidationError):

cognite/neat/rules/issues/dms.py

@@ -19,12 +19,11 @@
     "DuplicatedViewInDataModelError",
     "DirectRelationMissingSourceWarning",
     "ContainerPropertyUsedMultipleTimesError",
-    "DirectRelationListWarning",
-    "ReverseOfDirectRelationListWarning",
     "EmptyContainerWarning",
     "UnsupportedRelationWarning",
     "MultipleReferenceWarning",
     "HasDataFilterOnNoPropertiesViewWarning",
+    "ReverseRelationMissingOtherSideWarning",
     "NodeTypeFilterOnParentViewWarning",
     "ChangingContainerError",
     "ChangingViewError",
@@ -297,54 +296,6 @@ def dump(self) -> dict[str, Any]:
         return output
 
 
-@dataclass(frozen=True)
-class DirectRelationListWarning(DMSSchemaWarning):
-    description = "The container property is set to a direct relation list, which is not supported by the CDF API"
-    fix = "Make the property into a multiedge connection instead"
-    error_name: ClassVar[str] = "DirectRelationListWarning"
-    view_id: dm.ViewId
-    container_id: dm.ContainerId
-    property: str
-
-    def message(self) -> str:
-        return (
-            f"The property in {self.container_id}.{self.property} is a list of direct relations. "
-            f"This is not supported by the API, so it will be converted to an MultiEdgeConnection on"
-            f"the view {self.view_id}.{self.property} instead"
-        )
-
-    def dump(self) -> dict[str, Any]:
-        output = super().dump()
-        output["view_id"] = self.view_id.dump()
-        output["container_id"] = self.container_id.dump()
-        output["property"] = self.property
-        return output
-
-
-@dataclass(frozen=True)
-class ReverseOfDirectRelationListWarning(DMSSchemaWarning):
-    description = (
-        "The view property is set to a reverse of a direct relation list, which is not supported by the CDF API"
-    )
-    fix = "Make the property into a multiedge connection instead"
-    error_name: ClassVar[str] = "ReverseOfDirectRelationListWarning"
-    view_id: dm.ViewId
-    property: str
-
-    def message(self) -> str:
-        return (
-            f"The property pointed to be {self.view_id}.{self.property} is a list of direct relations. "
-            f"This is not supported by the API, so the {self.view_id}.{self.property} "
-            "will be converted from a reverse direct relation to an MultiEdgeConnection instead"
-        )
-
-    def dump(self) -> dict[str, Any]:
-        output = super().dump()
-        output["view_id"] = self.view_id.dump()
-        output["property"] = self.property
-        return output
-
-
 @dataclass(frozen=True)
 class EmptyContainerWarning(DMSSchemaWarning):
     description = "The container is empty"
@@ -387,6 +338,24 @@ def dump(self) -> dict[str, Any]:
         return output
 
 
+@dataclass(frozen=True)
+class ReverseRelationMissingOtherSideWarning(DMSSchemaWarning):
+    description = "The relation is missing the other side"
+    fix = "Add the other side of the relation"
+    error_name: ClassVar[str] = "ReverseRelationMissingOtherSideWarning"
+    view_id: dm.ViewId
+    property: str
+
+    def message(self) -> str:
+        return f"The reverse relation specified in {self.view_id}.{self.property} is missing the other side."
+
+    def dump(self) -> dict[str, Any]:
+        output = super().dump()
+        output["view_id"] = self.view_id.dump()
+        output["property"] = self.property
+        return output
+
+
 @dataclass(frozen=True)
 class MultipleReferenceWarning(DMSSchemaWarning):
     description = "The view is implements multiple views from other spaces"
@@ -444,3 +413,28 @@ def dump(self) -> dict[str, Any]:
         output = super().dump()
         output["view_id"] = self.view_id.dump()
         return output
+
+
+@dataclass(frozen=True)
+class HasDataFilterOnViewWithReferencesWarning(DMSSchemaWarning):
+    description = (
+        "Setting a hasData filter on a solution view which reference other containers is not recommended."
+        "This will lead to no nodes being returned when querying the solution view."
+    )
+    fix = "Use a node type filter instead"
+    error_name: ClassVar[str] = "HasDataFilterOnReferencedViewWarning"
+
+    view_id: dm.ViewId
+    references: list[dm.ViewId]
+
+    def message(self) -> str:
+        return (
+            f"Setting a hasData filter on view {self.view_id} which references other views {self.references}. "
+            "This is not recommended as it will lead to no nodes being returned when querying the solution view."
+        )
+
+    def dump(self) -> dict[str, Any]:
+        output = super().dump()
+        output["view_id"] = self.view_id.dump()
+        output["references"] = [view.dump() for view in sorted(self.references, key=lambda x: x.as_tuple())]
+        return output

cognite/neat/rules/issues/fileread.py

@@ -9,10 +9,13 @@
     "InvalidFileFormatWarning",
     "UnsupportedSpecWarning",
     "UnknownItemWarning",
+    "FailedLoadWarning",
     "BugInImporterWarning",
     "FileReadError",
     "FileNotFoundError",
     "FileNotAFileError",
+    "InvalidFileFormatError",
+    "FailedStringLoadError",
 ]
 
 
@@ -96,6 +99,26 @@ def dump(self) -> dict[str, str | None]:
         return output
 
 
+@dataclass(frozen=True)
+class FailedLoadWarning(FileReadWarning):
+    description = "The file content is invalid"
+    fix = "Check if the file content is valid"
+
+    expected_format: str
+    error_message: str
+
+    def message(self) -> str:
+        return (
+            f"Failed to load {self.filepath.name}. Expected format: {self.expected_format}. Error: {self.error_message}"
+        )
+
+    def dump(self) -> dict[str, str | None]:
+        output = super().dump()
+        output["expected_format"] = self.expected_format
+        output["error_message"] = self.error_message
+        return output
+
+
 @dataclass(frozen=True)
 class BugInImporterWarning(FileReadWarning):
     description = "A bug was raised during reading."
@@ -143,6 +166,24 @@ def message(self) -> str:
         return f"{self.filepath} is not in the expected format. Expected format: {self.expected_format}."
 
 
+@dataclass(frozen=True)
+class FailedStringLoadError(NeatValidationError):
+    description = "The file content is invalid"
+    fix = "Check if the file content is valid"
+
+    expected_format: str
+    error_message: str
+
+    def message(self) -> str:
+        return f"Failed to load string. Expected format: {self.expected_format}. Error: {self.error_message}"
+
+    def dump(self) -> dict[str, str | None]:
+        output = super().dump()
+        output["expected_format"] = self.expected_format
+        output["error_message"] = self.error_message
+        return output
+
+
 @dataclass(frozen=True)
 class NoFilesFoundError(FileReadError):
     description = "No files were found in the directory"