Update to version v5.3.4

### Updated
- Security patches for npm and pip packages

### Fixed
- Fix Connect Voice response failure when QID has buttons Fix Connect Voice response failure when QID has buttons (#607 )

.gitignore

@@ -1,5 +1,7 @@
 build
 
+codescan-*
+
 # Logs
 logs
 *.log
@@ -63,9 +65,10 @@ config.json
 .vscode/
 utilities/migration.md
 
-# Temporary folders
+# Temporary folders and backup files
 tmp/
 temp/
+*.bak
 
 # derived build assets
 **/deployment/global-s3-assets

CHANGELOG.md

@@ -5,6 +5,16 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [5.3.4] - 2023-05-19
+
+### Updated
+
+- Security patches for npm and pip packages
+
+### Fixed
+
+- Fix Connect Voice response failure when QID has buttons (#607)
+
 ## [5.3.3] - 2023-04-20
 
 ### Updated

lambda/es-proxy-layer/package.json

@@ -11,6 +11,6 @@
     "handlebars": "^4.7.2",
     "linkifyjs": "^3.0.0-beta.3",
     "simple-encryptor": "^3.0.0",
-    "vm2": "^3.9.17"
+    "vm2": "^3.9.18"
   }
 }

lambda/fulfillment/lib/middleware/1_parse.js

@@ -65,11 +65,9 @@ module.exports = async function parse(req, res) {
     _.set(req, "_settings", settings);
 
     req._type = req._event.version ? "ALEXA" : "LEX"
-    req._clientType = getClientType(req)
 
     switch (req._type) {
         case 'LEX':
-            Object.assign(req, await lex.parse(req))
             _.set(req,"_preferredResponseType","PlainText") ;
             // Determine preferred response message type - PlainText, or SSML
             const outputDialogMode = _.get(req,"_event.outputDialogMode") || _.get(req,"_event.inputMode") ;
@@ -84,10 +82,13 @@ module.exports = async function parse(req, res) {
             } else {
                 qnabot.log("WARNING: Unrecognised value for outputDialogMode:", outputDialogMode);
             }
+            req._clientType = getClientType(req)
+            Object.assign(req, await lex.parse(req))
             break;
         case 'ALEXA':
+            _.set(req,"_preferredResponseType","SSML")
+            req._clientType = getClientType(req)
             Object.assign(req, await alexa.parse(req))
-            _.set(req,"_preferredResponseType","SSML") ;
             break;
     }
 

lambda/import/convert-xlsx.js

@@ -1,124 +1,173 @@
-var XLSX = require("xlsx")
+var XLSX = require("read-excel-file")
 const _ = require('lodash')
 
-exports.convertxlsx = function (content) {
-
-  var header_mapping = {
-    question: "q",
-    topic: "t",
-    markdown: "alt.markdown",
-    answer: "a",
-    "Answer": "a",
-    ssml: "alt.ssml",
+exports.convertxlsx = async function (content) {
+
+  //this headermap enabled customers to more conveniently
+  //map some of the more common fields using a 'friendly' name
+  let headerMapping = {
+    question: 'q',
+    topic: 't',
+    markdown: 'alt.markdown',
+    answer: 'a',
+    'Answer': 'a',
+    ssml: 'alt.ssml'
   };
 
   console.log('inside convert json')
   try {
-    var workbook = XLSX.read(content, {
-      type: 'array'
-    });
-    var valid_questions = []
-    workbook.SheetNames.forEach(function (sheetName) {
-      // Here is your object
-      var XL_row_object = XLSX.utils.sheet_to_row_object_array(workbook.Sheets[sheetName]);
-      var json_object = JSON.stringify(XL_row_object);
-      var question_number = 1
-      XL_row_object.forEach(question => {
-        console.log("Processing " + JSON.stringify(question))
-        for (const property in header_mapping) {
-          var dest_property = header_mapping[property]
-          if (question[dest_property] == undefined) {
-            _.set(question, dest_property.split("."), question[property])
-            console.log("Assigning value for " + dest_property)
-            delete question[property]
-          }
-        }
-        question_number++
-        if (question["cardtitle"] != undefined) {
-          console.log("processing response title")
-          question.r = {}
-          question.r.title = question["cardtitle"]
-          delete question["cardtitle"]
-          if (question["imageurl"] != undefined) {
-            question.r.imageUrl = question.imageurl
-            delete question.imageurl
-          }
-          if (question["cardsubtitle"] != undefined) {
-            question.r.subTitle = question.subtitle
-            delete question["cardsubtitle"]
-          }
-          question.r.buttons = []
-          let i = 1
-          while (true) {
+    let sheetNames = await XLSX.readSheetNames(content)
+    var valid_questions = [];
+    for(let i = 0; i < sheetNames.length; i++){
+        // Here is your object
+        let rows = await XLSX.default(content, {sheet: sheetNames[i]})
+        let headerRow = rows.shift()
+        let excelRowNumber = 1; //excel sheets start at index 1, which for us is the header
+        rows.forEach((question) => {
+            console.log('Processing ' + JSON.stringify(question));
+            excelRowNumber++;
+
+            //first let's remap the current row entry from an index array
+            //to a key value map for easier processing
+            let questionMap = {}
+            for(let j = 0; j < headerRow.length; j++){
+                questionMap[headerRow[j]] = question[j]
+            }
+            question = questionMap
+
+            //let's try and map a couple friendly column names into their
+            //actual property names using the header mapping (e.g. 'topic' to 't')
+            for (const property in headerMapping) {
+                let dest_property = headerMapping[property];
+                if (question[dest_property] == undefined) {
+                    console.log('Assigning value for ' + dest_property);
+                    _.set(question, dest_property, question[property]);
+                    delete question[property];
+                }
+            }
 
-            console.log("Processing Button" + i)
-            var buttonFieldTextName = "displaytext" + i
-            var buttonFieldValueName = "buttonvalue" + i
-            i++
-            var undefinedButtonFieldCount = (question[buttonFieldTextName] == undefined) + (question[buttonFieldValueName] == undefined)
-            console.log("ButtonName " + question[buttonFieldTextName] + " ButtonValue " + question[buttonFieldValueName])
-            console.log("Undefined field count " + undefinedButtonFieldCount)
 
-            if (undefinedButtonFieldCount == 2) {
-              break
+            //lets try to extract all of the user questions
+            question.q = question.q ? [question.q] : []
+            let counter = 1;
+            while (true) {
+                //users can import multiple utterances, be appending sequential numbers to
+                //the column 'question', e.g. question8
+                var userQuestion = question['question' + counter];
+                if(!userQuestion) {
+                    //break on the first instance of missing question number. For example,
+                    //if user has question1 and question3 in their excel file, but no question2
+                    //then we would never look at question3 because question2 is missing
+                    break
+                }
+                question.q.push(userQuestion.replace(/(\r\n|\n|\r)/gm, ' '));
+                delete question['question' + counter];
+                counter++;
             }
-            if (undefinedButtonFieldCount == 1) {
-              console.log(`Warning:  Both ${buttonFieldTextName} and ${buttonFieldValueName} must be defined for qid: "${question.qid}"`)
-              continue;
+
+            //validate mandatory fields of qid, question, and answer
+            //qid must exist
+            if (!question.qid) {
+                console.log(
+                    `Warning: No QID found for line ${excelRowNumber}. The question will be skipped.`
+                );
+                return;
             }
-            console.log("Found two values")
-            if (question[buttonFieldValueName].length > 80) {
-              console.log(`Warning: ${buttonFieldValueName} must be less than or equal to 80 characters for qid:"${question.qid}"`)
-              continue;
+            //must have atleast 1 question
+            if (question.q.length == 0) {
+                console.log(
+                    'Warning: No questions found for QID: "' +
+                        question.qid +
+                        '". The question will be skipped.'
+                );
+                return;
             }
-            if (question[buttonFieldTextName].length > 80) {
-              console.log(`Warning: ${buttonFieldTextName} must be less than or equal to 80 characters for qid:"${question.qid}"`)
-              continue;
+            //answer must exist and include valid characters
+            if (!question.a || question.a.replace(/[^a-zA-Z0-9-_]/g, '').trim().length == 0) {
+                console.log(
+                    'Warning: No answer found for QID:"' +
+                        question.qid +
+                        '". The question will be skipped.'
+                );
+                return;
             }
-            var button = {
-              "text": question[buttonFieldTextName],
-              "value": question[buttonFieldValueName]
+
+            if (question['cardtitle']) {
+                console.log('processing response title');
+                question.r = {};
+                question.r.title = question['cardtitle'];
+                delete question['cardtitle'];
+                if (question['imageurl']) {
+                    question.r.imageUrl = question.imageurl;
+                    delete question.imageurl;
+                }
+                if (question['cardsubtitle']) {
+                    question.r.subTitle = question.subtitle;
+                    delete question['cardsubtitle'];
+                }
+
+                //TODO, refactor to operate similar to import.vue
+                //better yet, move common xlsx validation into common-modules
+                question.r.buttons = []
+                let i = 1
+                while (true) {
+                  console.log("Processing Button" + i)
+                  var buttonFieldTextName = "displaytext" + i
+                  var buttonFieldValueName = "buttonvalue" + i
+                  i++
+                  var undefinedButtonFieldCount = (question[buttonFieldTextName] == undefined) + (question[buttonFieldValueName] == undefined)
+                  console.log("ButtonName " + question[buttonFieldTextName] + " ButtonValue " + question[buttonFieldValueName])
+                  console.log("Undefined field count " + undefinedButtonFieldCount)
+
+                  if (undefinedButtonFieldCount == 2) {
+                    break
+                  }
+                  if (undefinedButtonFieldCount == 1) {
+                    console.log(`Warning:  Both ${buttonFieldTextName} and ${buttonFieldValueName} must be defined for qid: "${question.qid}"`)
+                    continue;
+                  }
+                  console.log("Found two values")
+                  if (question[buttonFieldValueName].length > 80) {
+                    console.log(`Warning: ${buttonFieldValueName} must be less than or equal to 80 characters for qid:"${question.qid}"`)
+                    continue;
+                  }
+                  if (question[buttonFieldTextName].length > 80) {
+                    console.log(`Warning: ${buttonFieldTextName} must be less than or equal to 80 characters for qid:"${question.qid}"`)
+                    continue;
+                  }
+                  var button = {
+                    "text": question[buttonFieldTextName],
+                    "value": question[buttonFieldValueName]
+                  }
+                  console.log("Adding button " + JSON.stringify(button))
+                  question.r.buttons.push(button)
+                  delete question[buttonFieldTextName]
+                  delete question[buttonFieldValueName]
+                }
+            }
+
+            //properties with a '.' should be treated as nested properties
+            //let's set any that we find into their proper destination within the object
+            //e.g. 'botRouting.specialty_bot' ==> 'botRouting': { 'specialty_bot': value }
+            for (let property in question) {
+              if (property.includes('.')) {
+                  let value = question[property]
+                  //need to delete the property first to ensure lodash treats the property
+                  //variable as a path, and not just as a string key
+                  delete question[property];
+                  if(value != null){
+                      _.set(question, property, value);
+                  }
+              }
             }
-            console.log("Adding button " + JSON.stringify(button))
-            question.r.buttons.push(button)
-            delete question[buttonFieldTextName]
-            delete question[buttonFieldValueName]
-          }
-        }
-        let counter = 1
-        question.q = question.q == undefined ? [] : question.q
-        while (true) {
-          var userQuestion = question["question" + counter]
-          if (userQuestion != undefined) {
-            question.q.push(userQuestion.replace(/(\r\n|\n|\r)/gm, " "))
-            delete question["question" + counter]
-            counter++
-          } else {
-            break;
-          }
-        }
-        for (let property in question) {
-          if (property.includes(".")) {
-            _.set(question, property.split("."), question[property])
-          }
-        }
-        if (question.qid == undefined) {
-          console.log(`Warning: No QID found for line ${question_number}. The question will be skipped.`)
-          return []
-        }
 
-        if (question.a == undefined || question.a.replace(/[^a-zA-Z0-9-_]/g, '').trim().length == 0) {
-          console.log("Warning: No answer found for QID:\"" + question.qid + "\". The question will be skipped.")
-          return []
-        }
-        if (question.q.length == 0) {
-          console.log("Warning: No questions found for QID: \"" + question.qid + "\". The question will be skipped.")
-          return []
-        }
-        console.log("Processed " + JSON.stringify(question))
-        valid_questions.push(question)
-      })
-    })
+            //Note that at this point we have stopped processing the excel file and any additional
+            //fields will be left as is. This means that new or more advanced fields can be imported
+            //by directly referencing their schema id (e.g. 'kendraRedirectQueryArgs')
+            console.log('Processed ' + JSON.stringify(question));
+            valid_questions.push(question);
+        });
+    };
     return valid_questions
   } catch (err) {
     console.log("Parse error");