Fix LUIS JSON to LU to handle hierarchical entities correctly

packages/lu/src/parser/luis/luConverter.js

@@ -92,7 +92,12 @@ const parseUtterancesToLu = function(utterances, luisJSON){
         if(utterance.entities.length >= 0) {
             // update utterance for each entity
             let text = utterance.text;
-            let sortedEntitiesList = objectSortByStartPos(utterance.entities);
+            // flatten entities
+            let flatEntities = [];
+            Object.assign([], utterance.entities).forEach(entity => flattenEntities(entity, flatEntities));
+            let sortedEntitiesList = objectSortByStartPos(flatEntities);
+            // remove all children
+            sortedEntitiesList.forEach(entity => delete entity.children);
             let tokenizedText = text.split('');
             // handle cases where we have both child as well as cases where more than one entity can have the same start position
             // if there are multiple entities in the same start position, then order them by composite, nDepth, regular entity
@@ -108,6 +113,14 @@ const parseUtterancesToLu = function(utterances, luisJSON){
     return fileContent
 }
 
+const flattenEntities = function(entity, flatEntities)
+{
+    if (entity.children !== undefined && Array.isArray(entity.children) && entity.children.length !== 0) {
+        entity.children.forEach(child => flattenEntities(child, flatEntities));
+    }
+    flatEntities.push(Object.assign({}, entity));
+}
+
 const getEntitiesByPositionList = function(entitiesList, tokenizedText) {
     (entitiesList || []).forEach(entity => {
         // does this entity have child labels?

packages/lu/test/fixtures/testcases/overlappingEntities.json

@@ -0,0 +1,92 @@
+{
+  "intents": [
+      {
+          "name": "None"
+      }
+  ],
+  "entities": [
+      {
+          "name": "add",
+          "roles": [],
+          "children": [
+              {
+                  "name": "count",
+                  "children": [],
+                  "features": [
+                      {
+                          "modelName": "globalCount",
+                          "isRequired": true
+                      }
+                  ]
+              }
+          ]
+      },
+      {
+          "name": "globalCount",
+          "roles": [],
+          "children": [
+              {
+                  "name": "countNumber",
+                  "children": [],
+                  "features": [
+                      {
+                          "modelName": "number",
+                          "isRequired": true
+                      }
+                  ]
+              }
+          ]
+      }
+  ],
+  "composites": [],
+  "closedLists": [],
+  "regex_entities": [],
+  "regex_features": [],
+  "utterances": [
+      {
+          "text": "add two apples",
+          "intent": "None",
+          "entities": [
+              {
+                  "entity": "add",
+                  "startPos": 0,
+                  "endPos": 13,
+                  "children": [
+                      {
+                          "entity": "count",
+                          "startPos": 4,
+                          "endPos": 13
+                      }
+                  ]
+              },
+              {
+                  "entity": "globalCount",
+                  "startPos": 4,
+                  "endPos": 13,
+                  "children": [
+                      {
+                          "entity": "countNumber",
+                          "startPos": 4,
+                          "endPos": 6
+                      }
+                  ]
+              }
+          ]
+      }
+  ],
+  "patterns": [],
+  "patternAnyEntities": [],
+  "prebuiltEntities": [
+      {
+          "name": "number",
+          "roles": []
+      }
+  ],
+  "luis_schema_version": "7.0.0",
+  "versionId": "0.1",
+  "name": "LuTest",
+  "desc": "",
+  "culture": "en-us",
+  "tokenizerVersion": "1.0.0",
+  "phraselists": []
+}

packages/lu/test/parser/luis/luisBuilder.test.js

@@ -1,6 +1,7 @@
 const LUISBuilder = require('./../../../src/parser/luis/luisBuilder')
 const LU = require('./../../../src/parser/lu/lu')
 var chai = require('chai');
+const luisobjenum = require('../../../src/parser/utils/enums/luisobjenum');
 var assert = chai.assert;
 
 describe('LUISBuilder', function() {
@@ -65,5 +66,11 @@ assert.isTrue(luisObject.validate())
         assert.equal(luisObject.content.includes('disabledForAllModels'), true);
     })
 
+    it('Overlapping entities are converted to LU correctly', async () => {
+        let testJSON = require('../../fixtures/testcases/overlappingEntities.json');
+        const luisObject = LUISBuilder.fromJson(testJSON).parseToLU();
+        assert.equal(luisObject.content.includes(`- {@add=add {@globalCount={@count={@countNumber=two} apples}}}`), true);
+    })
+
 
 });