diff --git a/packages/lu/src/parser/luis/luConverter.js b/packages/lu/src/parser/luis/luConverter.js index bb525816e..a44433547 100644 --- a/packages/lu/src/parser/luis/luConverter.js +++ b/packages/lu/src/parser/luis/luConverter.js @@ -92,7 +92,12 @@ const parseUtterancesToLu = function(utterances, luisJSON){ if(utterance.entities.length >= 0) { // update utterance for each entity let text = utterance.text; - let sortedEntitiesList = objectSortByStartPos(utterance.entities); + // flatten entities + let flatEntities = []; + Object.assign([], utterance.entities).forEach(entity => flattenEntities(entity, flatEntities)); + let sortedEntitiesList = objectSortByStartPos(flatEntities); + // remove all children + sortedEntitiesList.forEach(entity => delete entity.children); let tokenizedText = text.split(''); // handle cases where we have both child as well as cases where more than one entity can have the same start position // if there are multiple entities in the same start position, then order them by composite, nDepth, regular entity @@ -108,6 +113,14 @@ const parseUtterancesToLu = function(utterances, luisJSON){ return fileContent } +const flattenEntities = function(entity, flatEntities) +{ + if (entity.children !== undefined && Array.isArray(entity.children) && entity.children.length !== 0) { + entity.children.forEach(child => flattenEntities(child, flatEntities)); + } + flatEntities.push(Object.assign({}, entity)); +} + const getEntitiesByPositionList = function(entitiesList, tokenizedText) { (entitiesList || []).forEach(entity => { // does this entity have child labels? diff --git a/packages/lu/test/fixtures/testcases/overlappingEntities.json b/packages/lu/test/fixtures/testcases/overlappingEntities.json new file mode 100644 index 000000000..29ab6bbdf --- /dev/null +++ b/packages/lu/test/fixtures/testcases/overlappingEntities.json @@ -0,0 +1,92 @@ +{ + "intents": [ + { + "name": "None" + } + ], + "entities": [ + { + "name": "add", + "roles": [], + "children": [ + { + "name": "count", + "children": [], + "features": [ + { + "modelName": "globalCount", + "isRequired": true + } + ] + } + ] + }, + { + "name": "globalCount", + "roles": [], + "children": [ + { + "name": "countNumber", + "children": [], + "features": [ + { + "modelName": "number", + "isRequired": true + } + ] + } + ] + } + ], + "composites": [], + "closedLists": [], + "regex_entities": [], + "regex_features": [], + "utterances": [ + { + "text": "add two apples", + "intent": "None", + "entities": [ + { + "entity": "add", + "startPos": 0, + "endPos": 13, + "children": [ + { + "entity": "count", + "startPos": 4, + "endPos": 13 + } + ] + }, + { + "entity": "globalCount", + "startPos": 4, + "endPos": 13, + "children": [ + { + "entity": "countNumber", + "startPos": 4, + "endPos": 6 + } + ] + } + ] + } + ], + "patterns": [], + "patternAnyEntities": [], + "prebuiltEntities": [ + { + "name": "number", + "roles": [] + } + ], + "luis_schema_version": "7.0.0", + "versionId": "0.1", + "name": "LuTest", + "desc": "", + "culture": "en-us", + "tokenizerVersion": "1.0.0", + "phraselists": [] +} \ No newline at end of file diff --git a/packages/lu/test/parser/luis/luisBuilder.test.js b/packages/lu/test/parser/luis/luisBuilder.test.js index 359735b3e..3929a2c36 100644 --- a/packages/lu/test/parser/luis/luisBuilder.test.js +++ b/packages/lu/test/parser/luis/luisBuilder.test.js @@ -1,6 +1,7 @@ const LUISBuilder = require('./../../../src/parser/luis/luisBuilder') const LU = require('./../../../src/parser/lu/lu') var chai = require('chai'); +const luisobjenum = require('../../../src/parser/utils/enums/luisobjenum'); var assert = chai.assert; describe('LUISBuilder', function() { @@ -65,5 +66,11 @@ assert.isTrue(luisObject.validate()) assert.equal(luisObject.content.includes('disabledForAllModels'), true); }) + it('Overlapping entities are converted to LU correctly', async () => { + let testJSON = require('../../fixtures/testcases/overlappingEntities.json'); + const luisObject = LUISBuilder.fromJson(testJSON).parseToLU(); + assert.equal(luisObject.content.includes(`- {@add=add {@globalCount={@count={@countNumber=two} apples}}}`), true); + }) + }); \ No newline at end of file