Bug fixes in base workflows and improved data-exploration UI (#55)

* base wf fixes and data exploration UI extensions

* more data exploration improvements


* Updated example rules file and
regenerated test data for workflow test

* updated change log and removed commented code

---------

Co-authored-by: Aleksandrs Livincovs <aleksandrs.livincovs@cognite.com>

.gitignore

@@ -7,6 +7,7 @@ __pycache__/
 config.yaml
 real_data/
 data/
+dev-data/
 
 !cognite/neat/examples/config.yaml
 requirements.txt

Makefile

@@ -1,17 +1,21 @@
 .PHONY: run-explorer run-tests run-linters build-ui build-python build-docker run-docker compose-up
 
-version="0.14.0"
+version="0.14.1"
 run-explorer:
 	@echo "Running explorer API server..."
 	# open "http://localhost:8000/static/index.html" || true
 	mkdir -p ./data
-	export NEAT_CONFIG_PATH=./data/config.yaml && \
+	export NEAT_CONFIG_PATH=./dev-data/config.yaml && \
 	poetry run uvicorn --host 0.0.0.0 cognite.neat.explorer.explorer:app
 
 run-tests:
 	@echo "Running tests..."
 	poetry run pytest
 
+run-regen-test:
+	@echo "Regenerating test data for failed test. This will overwrite the existing test data !!!!!! Use with caution !!!!!!"
+	poetry run pytest --force-regen
+
 configure:
 	@echo "Configuring..."
 	cd cognite/neat/explorer-ui/neat-app; npm install

cognite/neat/__init__.py

@@ -1 +1 @@
-__version__ = "0.14.0"
+__version__ = "0.14.1"

cognite/neat/base_workflows/graph2assets_relationships.py

@@ -69,7 +69,7 @@ def step_prepare_cdf_assets(self, flow_msg: FlowMessage):
             logging.error(f"Found orphaned assets: {', '.join(orphan_assets)}")
 
             orphanage_asset_external_id = (
-                f"{self.transformation_rules.metadata.externalIdPrefix}orphanage"
+                f"{self.transformation_rules.metadata.externalIdPrefix}orphanage-{self.transformation_rules.metadata.data_set_id}"
                 if self.transformation_rules.metadata.externalIdPrefix
                 else "orphanage"
             )
@@ -93,7 +93,7 @@ def step_prepare_cdf_assets(self, flow_msg: FlowMessage):
             logging.info("No orphaned assets found, your assets look healthy !")
 
         if circular_assets:
-            msg = f"Found circular dependencies: {', '.join(circular_assets)}"
+            msg = f"Found circular dependencies: {str(circular_assets)}"
             logging.error(msg)
             raise Exception(msg)
         elif orphan_assets:

cognite/neat/base_workflows/graphs_and_rules.py

@@ -4,6 +4,7 @@
 from cognite.client import CogniteClient
 
 from cognite.neat.core import loader, parser
+from cognite.neat.core.configuration import PREFIXES
 from cognite.neat.core.data_classes.transformation_rules import TransformationRules
 from cognite.neat.core.loader.graph_store import NeatGraphStore, drop_graph_store
 from cognite.neat.core.transformer import RuleProcessingReport, domain2app_knowledge_graph
@@ -61,8 +62,9 @@ def step_configuring_stores(self, flow_msg: FlowMessage = None, clean_start: boo
             drop_graph_store(self.solution_graph, solution_store_dir, force=True)
 
         self.source_graph = loader.NeatGraphStore(
-            prefixes=self.transformation_rules.prefixes, base_prefix="nordic44", namespace="http://purl.org/nordic44#"
+            prefixes=self.transformation_rules.prefixes, base_prefix="neat", namespace=PREFIXES["neat"]
         )
+
         if self.get_config_item_value("source_rdf_store.type"):
             self.source_graph.init_graph(
                 self.get_config_item_value("source_rdf_store.type", self.graph_source_type),
@@ -73,7 +75,10 @@ def step_configuring_stores(self, flow_msg: FlowMessage = None, clean_start: boo
             )
 
         if self.get_config_item_value("solution_rdf_store.type"):
-            self.solution_graph = loader.NeatGraphStore(prefixes=self.transformation_rules.prefixes)
+            self.solution_graph = loader.NeatGraphStore(
+                prefixes=self.transformation_rules.prefixes, base_prefix="neat", namespace=PREFIXES["neat"]
+            )
+
             self.solution_graph.init_graph(
                 self.get_config_item_value("solution_rdf_store.type"),
                 self.get_config_item_value("solution_rdf_store.query_url"),

cognite/neat/core/configuration.py

@@ -28,6 +28,7 @@ def as_set(cls) -> set[str]:
     "md": Namespace("http://iec.ch/TC57/61970-552/ModelDescription/1#"),
     "pti": Namespace("http://www.pti-us.com/PTI_CIM-schema-cim16#"),
     "tnt": Namespace("http://purl.org/cognite/tnt#"),
+    "neat": Namespace("http://purl.org/cognite/neat#"),
 }
 
 DEFAULT_NAMESPACE = Namespace("http://purl.org/cognite/app#")

cognite/neat/core/extractors/rdf_to_assets.py

@@ -526,7 +526,15 @@ def _asset2dict(asset: Asset) -> dict:
 
 def _flatten_labels(labels: List[Dict[str, str]]) -> set[str]:
     """Flatten labels"""
-    return {label["externalId"] for label in labels}
+    result = set()
+    if labels is None:
+        return result
+    for label in labels:
+        if "externalId" in label:
+            result.add(label["externalId"])
+        else:
+            logging.warning(f"Label {label} does not have externalId")
+    return result
 
 
 def _is_historic(labels) -> bool:

cognite/neat/core/loader/graph_store.py

@@ -142,6 +142,7 @@ def init_graph(
             self.base_prefix = base_prefix
 
         self.graph.bind(self.base_prefix, self.namespace)
+        logging.info("Adding prefix %s with namespace %s", self.base_prefix, self.namespace)
         logging.info("Graph initialized")
 
     def close(self):

cognite/neat/explorer-ui/neat-app/build/asset-manifest.json

@@ -1,14 +1,14 @@
 {
   "files": {
     "main.css": "./static/css/main.38a62222.css",
-    "main.js": "./static/js/main.45df5ef5.js",
+    "main.js": "./static/js/main.16555d84.js",
     "static/media/logo.svg": "./static/media/logo.4bdbcf8396881de06a6723a92fed910b.svg",
     "index.html": "./index.html",
     "main.38a62222.css.map": "./static/css/main.38a62222.css.map",
-    "main.45df5ef5.js.map": "./static/js/main.45df5ef5.js.map"
+    "main.16555d84.js.map": "./static/js/main.16555d84.js.map"
   },
   "entrypoints": [
     "static/css/main.38a62222.css",
-    "static/js/main.45df5ef5.js"
+    "static/js/main.16555d84.js"
   ]
 }

cognite/neat/explorer-ui/neat-app/build/index.html

@@ -1 +1 @@
-<!doctype html><html lang="en"><head><meta charset="utf-8"/><link rel="icon" href="./favicon.ico"/><meta name="viewport" content="width=device-width,initial-scale=1"/><meta name="theme-color" content="#000000"/><meta name="description" content="NEAT controll panel"/><link rel="apple-touch-icon" href="./logo192.png"/><link rel="manifest" href="./manifest.json"/><title>React App</title><script defer="defer" src="./static/js/main.45df5ef5.js"></script><link href="./static/css/main.38a62222.css" rel="stylesheet"></head><body><noscript>You need to enable JavaScript to run this app.</noscript><div id="root"></div></body></html>
+<!doctype html><html lang="en"><head><meta charset="utf-8"/><link rel="icon" href="./favicon.ico"/><meta name="viewport" content="width=device-width,initial-scale=1"/><meta name="theme-color" content="#000000"/><meta name="description" content="NEAT controll panel"/><link rel="apple-touch-icon" href="./logo192.png"/><link rel="manifest" href="./manifest.json"/><title>React App</title><script defer="defer" src="./static/js/main.16555d84.js"></script><link href="./static/css/main.38a62222.css" rel="stylesheet"></head><body><noscript>You need to enable JavaScript to run this app.</noscript><div id="root"></div></body></html>

cognite/neat/explorer-ui/neat-app/src/App.js

@@ -5,6 +5,7 @@ import BasicTabs from './MainContainer.tsx';
 import CssBaseline from '@mui/material/CssBaseline';
 
 function App() {
+  window.addEventListener("contextmenu", e => e.preventDefault());
   return (
     <div >
       <BasicTabs/>

cognite/neat/explorer-ui/neat-app/src/components/NodeViewer.tsx

@@ -0,0 +1,86 @@
+import Button from "@mui/material/Button"
+import Dialog from "@mui/material/Dialog"
+import DialogActions from "@mui/material/DialogActions"
+import DialogContent from "@mui/material/DialogContent"
+import DialogTitle from "@mui/material/DialogTitle"
+import React, { useEffect, useState } from "react"
+import RemoveNsPrefix, { getNeatApiRootUrl, getSelectedWorkflowName } from "./Utils"
+import { Paper, Table, TableBody, TableCell, TableContainer, TableHead, TableRow, Typography } from "@mui/material"
+import { ExplorerContext } from "./Context"
+
+export default function NodeViewerDialog(props: any)
+{
+    const {hiddenNsPrefixModeCtx, graphNameCtx} = React.useContext(ExplorerContext);
+    const [dialogOpen, setDialogOpen] = useState(false);
+    const neatApiRootUrl = getNeatApiRootUrl();
+    const [nodeId, setNodeId] = useState("");
+    const [graphName, setGraphName] = graphNameCtx;
+    const [hiddenNsPrefixMode, setHiddenNsPrefixMode] = hiddenNsPrefixModeCtx;
+    const [data, setData] = useState([]);
+
+    const handleDialogCancel = () => {
+        setDialogOpen(false);
+        props.onClose();
+    };
+
+    useEffect(() => {
+        if (props.open){
+            setNodeId(props.nodeId);
+            setDialogOpen(true);
+            loadObjectProperties(props.nodeId);
+        }
+      }, [props.open]);
+
+      const loadObjectProperties = (reference:string) => {
+        const workflowName = getSelectedWorkflowName();
+        fetch(neatApiRootUrl+`/api/object-properties?`+new URLSearchParams({"reference":reference,"graph_name":graphName,"workflow_name":workflowName}).toString())
+         .then((response) => response.json())
+          .then((rdata) => {
+            let result = rdata.rows.map((row:any) => {
+                let prop = row.property;
+                if(hiddenNsPrefixMode)
+                  prop = RemoveNsPrefix(prop);
+                return {property:prop, value:row.value}
+            });
+            setData(result);
+          }).catch((error) => {
+            console.error('Error:', error);
+          });
+       }
+
+
+return (
+  <Dialog open={dialogOpen} onClose={handleDialogCancel} fullWidth = {true} maxWidth = "lg"  >
+        <DialogTitle>Node viewer</DialogTitle>
+        <DialogContent sx={{ minWidth: 650 }} >
+        <TableContainer component={Paper}>
+      <Table sx={{ minWidth: 650 }} aria-label="simple table">
+        <TableHead>
+          <TableRow>
+            <TableCell>Property</TableCell>
+            <TableCell align="left">Value</TableCell>
+           </TableRow>
+        </TableHead>
+        <TableBody>
+          {data.map((row) => (
+            <TableRow
+              key={row.name}
+              sx={{ '&:last-child td, &:last-child th': { border: 0 } }}
+            >
+              <TableCell component="th" scope="row">
+                {row.property}
+              </TableCell>
+              <TableCell align="left">{ row.value } </TableCell>
+            </TableRow>
+          ))}
+        </TableBody>
+      </Table>
+    </TableContainer>
+            <Typography variant="body1" gutterBottom> object : { nodeId} </Typography>
+        </DialogContent>
+        <DialogActions>
+          <Button variant="outlined" size="small" onClick={handleDialogCancel}>Cancel</Button>
+        </DialogActions>
+      </Dialog>
+)
+}

cognite/neat/explorer-ui/neat-app/src/components/Utils.tsx

@@ -29,6 +29,14 @@ export function convertMillisToStr(millis) {
   return isoString;
 }
 
+export function getShortenedString(str,len) {
+  str = RemoveNsPrefix(str);
+  if (str.length <= len) {
+    return str;
+  } else {
+    return str.slice(0,len)+"..."+ str.slice(-len);
+  }
+}
 
 export default function RemoveNsPrefix(strWithPrefix: string) {