diff --git a/__tests__/algolia-cdk.test.js b/__tests__/algolia-cdk.test.js new file mode 100644 index 0000000000..f90c5ce9cd --- /dev/null +++ b/__tests__/algolia-cdk.test.js @@ -0,0 +1,64 @@ +const fs = require("fs"); +const path = require("path"); +const { TRANSFORMER_METRIC } = require("../v0/util/constant"); +const { processCdkV2Workflow } = require("../cdk/v2/handler"); + +const integration = "algolia"; +const name = "Algolia"; + +const inputDataFile = fs.readFileSync( + path.resolve(__dirname, `./data/${integration}_input.json`) +); +const outputDataFile = fs.readFileSync( + path.resolve(__dirname, `./data/${integration}_output.json`) +); +const inputData = JSON.parse(inputDataFile); +const expectedData = JSON.parse(outputDataFile); + +describe(`${name} Tests`, () => { + describe("Processor Tests", () => { + inputData.forEach((input, index) => { + it(`${name} - payload: ${index}`, async () => { + const expected = expectedData[index]; + try { + const output = await processCdkV2Workflow( + integration, + input, + TRANSFORMER_METRIC.ERROR_AT.PROC + ); + expect(output).toEqual(expected); + } catch (error) { + expect(error.message).toEqual(expected.message); + } + }); + }); + }); + + const inputRouterDataFile = fs.readFileSync( + path.resolve(__dirname, `./data/${integration}_router_input.json`) + ); + const outputRouterDataFile = fs.readFileSync( + path.resolve(__dirname, `./data/${integration}_router_output.json`) + ); + const inputRouterData = JSON.parse(inputRouterDataFile); + const expectedRouterData = JSON.parse(outputRouterDataFile); + + describe("Router Tests", () => { + inputRouterData.forEach((input, index) => { + it(`${name} - payload: ${index}`, async () => { + const expected = expectedRouterData[index]; + try { + const output = await processCdkV2Workflow( + integration, + input, + TRANSFORMER_METRIC.ERROR_AT.RT + ); + expect(output).toEqual(expected); + } catch (error) { + // console.log(error); + expect(error.message).toEqual(expected.message); + } + }); + }); + }); +}); diff --git a/__tests__/algolia.test.js b/__tests__/algolia.test.js index da0e1c371a..b835421dac 100644 --- a/__tests__/algolia.test.js +++ b/__tests__/algolia.test.js @@ -30,33 +30,28 @@ inputData.forEach((input, index) => { }); }); -const batchInputDataFile = fs.readFileSync( - path.resolve(__dirname, `./data/${integration}_batch_input.json`) +const routerInputDataFile = fs.readFileSync( + path.resolve(__dirname, `./data/${integration}_router_input.json`) ); -const batchOutputDataFile = fs.readFileSync( - path.resolve(__dirname, `./data/${integration}_batch_output.json`) +const routerInputData = JSON.parse(routerInputDataFile); +const routerOutputDataFile = fs.readFileSync( + path.resolve(__dirname, `./data/${integration}_router_output.json`) ); +const routerOutputData = JSON.parse(routerOutputDataFile); -// const batchInputData = JSON.parse(batchInputDataFile); -// const batchExpectedData = JSON.parse(batchOutputDataFile); - -// batchInputData.forEach((input, index) => { -// test(`${name} Batching ${index}`, () => { -// const output = transformer.batch(input); -// //console.log(output); -// expect(Array.isArray(output)).toEqual(true); -// expect(output.length).toEqual(batchExpectedData[index].length); -// output.forEach((input, indexInner) => { -// expect(output[indexInner]).toEqual(batchExpectedData[index][indexInner]); -// }) -// }); -// }); - -// Batching using routerTransform -test('Batching', async () => { - const batchInputData = JSON.parse(batchInputDataFile); - const batchExpectedData = JSON.parse(batchOutputDataFile); - const output = await transformer.processRouterDest(batchInputData); - expect(Array.isArray(output)).toEqual(true); - expect(output).toEqual(batchExpectedData); +describe('Router Tests', () => { + routerInputData.forEach((input, index) => { + it(`${name} Tests: payload - ${index}`, async () => { + let output, expected; + try { + output = await transformer.processRouterDest(input); + expected = routerOutputData[index] + } catch (error) { + output = error.message; + // console.log(output); + expected = routerOutputData[index].message; + } + expect(output).toEqual(expected); + }); + }); }); diff --git a/__tests__/data/algolia_batch_input.json b/__tests__/data/algolia_router_input.json similarity index 99% rename from __tests__/data/algolia_batch_input.json rename to __tests__/data/algolia_router_input.json index 188f64f4ee..281abe2dc5 100644 --- a/__tests__/data/algolia_batch_input.json +++ b/__tests__/data/algolia_router_input.json @@ -1,4 +1,5 @@ [ + [ { "message": { "type": "track", @@ -251,5 +252,8 @@ "IsProcessorEnabled": true } } - ] + ], + [], + {} +] \ No newline at end of file diff --git a/__tests__/data/algolia_batch_output.json b/__tests__/data/algolia_router_output.json similarity index 94% rename from __tests__/data/algolia_batch_output.json rename to __tests__/data/algolia_router_output.json index 72c89e3437..d1fffd4385 100644 --- a/__tests__/data/algolia_batch_output.json +++ b/__tests__/data/algolia_router_output.json @@ -1,4 +1,5 @@ [ + [ { "batchedRequest": { "version": "1", @@ -97,5 +98,14 @@ "IsProcessorEnabled": true } } - ] + ], + { + "message": "Invalid event array", + "statusCode": 400 + }, + { + "message": "Invalid event array", + "statusCode": 400 + } +] \ No newline at end of file diff --git a/__tests__/data/versioned_processor_algolia_input.json b/__tests__/data/versioned_processor_algolia_input.json new file mode 100644 index 0000000000..ad87d8524a --- /dev/null +++ b/__tests__/data/versioned_processor_algolia_input.json @@ -0,0 +1,720 @@ +{ + "request": { + "body": [ + { + "message": { + "channel": "web", + "context": { + "app": { + "build": "1.0.0", + "name": "RudderLabs JavaScript SDK", + "namespace": "com.rudderlabs.javascript", + "version": "1.0.0" + }, + "traits": { + "email": "testone@gmail.com", + "firstName": "test", + "lastName": "one" + }, + "library": { + "name": "RudderLabs JavaScript SDK", + "version": "1.0.0" + }, + "userAgent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.90 Safari/537.36", + "locale": "en-US", + "ip": "0.0.0.0", + "os": { + "name": "", + "version": "" + }, + "screen": { + "density": 2 + }, + "page": { + "path": "/destinations/ometria", + "referrer": "", + "search": "", + "title": "", + "url": "https://docs.rudderstack.com/destinations/ometria", + "category": "destination", + "initial_referrer": "https://docs.rudderstack.com", + "initial_referring_domain": "docs.rudderstack.com" + } + }, + "type": "track", + "messageId": "84e26acc-56a5-4835-8233-591137fca468", + "session_id": "3049dc4c-5a95-4ccd-a3e7-d74a7e411f22", + "originalTimestamp": "2019-10-14T09:03:17.562Z", + "anonymousId": "123456", + "event": "product clicked", + "userId": "testuserId1", + "properties": { + "index": "products", + "filters": [ + "field1:hello", + "val1:val2" + ] + }, + "integrations": { + "All": true + }, + "sentAt": "2019-10-14T09:03:22.563Z" + }, + "destination": { + "Config": { + "apiKey": "34d8efa09c5b048bbacc6af157f2e687", + "applicationId": "O2YARRI15I", + "eventTypeSettings": [ + { + "from": "product clicked", + "to": "cLick " + } + ] + }, + "DestinationDefinition": { + "Config": { + "cdkV2TestThreshold": 1 + } + } + } + }, + { + "message": { + "channel": "web", + "context": { + "app": { + "build": "1.0.0", + "name": "RudderLabs JavaScript SDK", + "namespace": "com.rudderlabs.javascript", + "version": "1.0.0" + }, + "traits": { + "email": "testone@gmail.com", + "firstName": "test", + "lastName": "one" + }, + "library": { + "name": "RudderLabs JavaScript SDK", + "version": "1.0.0" + }, + "userAgent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.90 Safari/537.36", + "locale": "en-US", + "ip": "0.0.0.0", + "os": { + "name": "", + "version": "" + }, + "screen": { + "density": 2 + }, + "page": { + "path": "/destinations/ometria", + "referrer": "", + "search": "", + "title": "", + "url": "https://docs.rudderstack.com/destinations/ometria", + "category": "destination", + "initial_referrer": "https://docs.rudderstack.com", + "initial_referring_domain": "docs.rudderstack.com" + } + }, + "type": "track", + "messageId": "84e26acc-56a5-4835-8233-591137fca468", + "session_id": "3049dc4c-5a95-4ccd-a3e7-d74a7e411f22", + "originalTimestamp": "2019-10-14T09:03:17.562Z", + "anonymousId": "123456", + "event": "product list viewed", + "userId": "testuserId1", + "properties": { + "index": "products", + "products": [ + { + "objectId": "ecommerce-sample-data-919", + "position": 7 + }, + { + "objectId": "9780439784542", + "position": 8 + } + ], + "queryId": "43b15df305339e827f0ac0bdc5ebcaa7" + }, + "integrations": { + "All": true + }, + "sentAt": "2019-10-14T09:03:22.563Z" + }, + "destination": { + "Config": { + "apiKey": "34d8efa09c5b048bbacc6af157f2e687", + "applicationId": "O2YARRI15I", + "eventTypeSettings": [ + { + "from": "product list viewed", + "to": "click" + } + ] + }, + "DestinationDefinition": { + "Config": { + "cdkV2TestThreshold": 1 + } + } + } + }, + { + "message": { + "channel": "web", + "context": { + "app": { + "build": "1.0.0", + "name": "RudderLabs JavaScript SDK", + "namespace": "com.rudderlabs.javascript", + "version": "1.0.0" + }, + "traits": { + "email": "testone@gmail.com", + "firstName": "test", + "lastName": "one" + }, + "library": { + "name": "RudderLabs JavaScript SDK", + "version": "1.0.0" + }, + "userAgent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.90 Safari/537.36", + "locale": "en-US", + "ip": "0.0.0.0", + "os": { + "name": "", + "version": "" + }, + "screen": { + "density": 2 + }, + "page": { + "path": "/destinations/ometria", + "referrer": "", + "search": "", + "title": "", + "url": "https://docs.rudderstack.com/destinations/ometria", + "category": "destination", + "initial_referrer": "https://docs.rudderstack.com", + "initial_referring_domain": "docs.rudderstack.com" + } + }, + "type": "track", + "messageId": "84e26acc-56a5-4835-8233-591137fca468", + "session_id": "3049dc4c-5a95-4ccd-a3e7-d74a7e411f22", + "originalTimestamp": "2019-10-14T09:03:17.562Z", + "anonymousId": "123456", + "event": "product clicked", + "userId": "testuserId1", + "properties": { + "eventType": "click", + "index": "products", + "queryId": "43b15df305339e827f0ac0bdc5ebcaa8" + }, + "integrations": { + "All": true + }, + "sentAt": "2019-10-14T09:03:22.563Z" + }, + "destination": { + "Config": { + "apiKey": "34d8efa09c5b048bbacc6af157f2e687", + "applicationId": "O2YARRI15I", + "eventTypeSettings": [] + }, + "DestinationDefinition": { + "Config": { + "cdkV2TestThreshold": 1 + } + } + } + }, + { + "message": { + "channel": "web", + "context": { + "app": { + "build": "1.0.0", + "name": "RudderLabs JavaScript SDK", + "namespace": "com.rudderlabs.javascript", + "version": "1.0.0" + }, + "traits": { + "email": "testone@gmail.com", + "firstName": "test", + "lastName": "one" + }, + "library": { + "name": "RudderLabs JavaScript SDK", + "version": "1.0.0" + }, + "userAgent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.90 Safari/537.36", + "locale": "en-US", + "ip": "0.0.0.0", + "os": { + "name": "", + "version": "" + }, + "screen": { + "density": 2 + }, + "page": { + "path": "/destinations/ometria", + "referrer": "", + "search": "", + "title": "", + "url": "https://docs.rudderstack.com/destinations/ometria", + "category": "destination", + "initial_referrer": "https://docs.rudderstack.com", + "initial_referring_domain": "docs.rudderstack.com" + } + }, + "type": "track", + "messageId": "84e26acc-56a5-4835-8233-591137fca468", + "session_id": "3049dc4c-5a95-4ccd-a3e7-d74a7e411f22", + "originalTimestamp": "2019-10-14T09:03:17.562Z", + "anonymousId": "123456", + "userId": "testuserId1", + "event": "product clicked", + "properties": { + "index": "products", + "queryId": "43b15df305339e827f0ac0bdc5ebcaa7" + }, + "integrations": { + "All": true + }, + "sentAt": "2019-10-14T09:03:22.563Z" + }, + "destination": { + "Config": { + "apiKey": "34d8efa09c5b048bbacc6af157f2e687", + "applicationId": "O2YARRI15I", + "eventTypeSettings": [] + }, + "DestinationDefinition": { + "Config": { + "cdkV2TestThreshold": 1 + } + } + } + }, + { + "message": { + "channel": "web", + "context": { + "app": { + "build": "1.0.0", + "name": "RudderLabs JavaScript SDK", + "namespace": "com.rudderlabs.javascript", + "version": "1.0.0" + }, + "traits": { + "email": "testone@gmail.com", + "firstName": "test", + "lastName": "one" + }, + "library": { + "name": "RudderLabs JavaScript SDK", + "version": "1.0.0" + }, + "userAgent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.90 Safari/537.36", + "locale": "en-US", + "ip": "0.0.0.0", + "os": { + "name": "", + "version": "" + }, + "screen": { + "density": 2 + }, + "page": { + "path": "/destinations/ometria", + "referrer": "", + "search": "", + "title": "", + "url": "https://docs.rudderstack.com/destinations/ometria", + "category": "destination", + "initial_referrer": "https://docs.rudderstack.com", + "initial_referring_domain": "docs.rudderstack.com" + } + }, + "type": "track", + "messageId": "84e26acc-56a5-4835-8233-591137fca468", + "session_id": "3049dc4c-5a95-4ccd-a3e7-d74a7e411f22", + "originalTimestamp": "2019-10-14T09:03:17.562Z", + "anonymousId": "123456", + "event": "product list viewed", + "userId": "testuserId1", + "properties": { + "index": "products", + "products": [ + { + "objectId": "ecommerce-sample-data-919", + "position": 7 + }, + { + "objectId": "9780439784542", + "position": 8 + } + ], + "queryId": "" + }, + "integrations": { + "All": true + }, + "sentAt": "2019-10-14T09:03:22.563Z" + }, + "destination": { + "Config": { + "apiKey": "34d8efa09c5b048bbacc6af157f2e687", + "applicationId": "O2YARRI15I", + "eventTypeSettings": [ + { + "from": "product list viewed", + "to": "click" + } + ] + }, + "DestinationDefinition": { + "Config": { + "cdkV2TestThreshold": 1 + } + } + } + }, + { + "message": { + "channel": "web", + "context": { + "app": { + "build": "1.0.0", + "name": "RudderLabs JavaScript SDK", + "namespace": "com.rudderlabs.javascript", + "version": "1.0.0" + }, + "traits": { + "email": "testone@gmail.com", + "firstName": "test", + "lastName": "one" + }, + "library": { + "name": "RudderLabs JavaScript SDK", + "version": "1.0.0" + }, + "userAgent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.90 Safari/537.36", + "locale": "en-US", + "ip": "0.0.0.0", + "os": { + "name": "", + "version": "" + }, + "screen": { + "density": 2 + }, + "page": { + "path": "/destinations/ometria", + "referrer": "", + "search": "", + "title": "", + "url": "https://docs.rudderstack.com/destinations/ometria", + "category": "destination", + "initial_referrer": "https://docs.rudderstack.com", + "initial_referring_domain": "docs.rudderstack.com" + } + }, + "type": "track", + "messageId": "84e26acc-56a5-4835-8233-591137fca468", + "session_id": "3049dc4c-5a95-4ccd-a3e7-d74a7e411f22", + "originalTimestamp": "2019-10-14T09:03:17.562Z", + "anonymousId": "123456", + "event": "product list viewed", + "userId": "testuserId1", + "properties": { + "index": "products", + "products": [ + { + "objectId": "ecommerce-sample-data-919", + "position": 7 + }, + { + "objectId": "9780439784542", + "position": 8 + } + ], + "queryId": "43b15df305339e827f0ac0bdc5ebcaa7" + }, + "integrations": { + "All": true + }, + "sentAt": "2019-10-14T09:03:22.563Z" + }, + "destination": { + "Config": { + "apiKey": "34d8efa09c5b048bbacc6af157f2e687", + "applicationId": "O2YARRI15I", + "eventTypeSettings": [ + { + "from": "product list viewed", + "to": "view" + } + ] + }, + "DestinationDefinition": { + "Config": { + "cdkV2TestThreshold": 1 + } + } + } + }, + { + "message": { + "channel": "web", + "context": { + "app": { + "build": "1.0.0", + "name": "RudderLabs JavaScript SDK", + "namespace": "com.rudderlabs.javascript", + "version": "1.0.0" + }, + "traits": { + "email": "testone@gmail.com", + "firstName": "test", + "lastName": "one" + }, + "library": { + "name": "RudderLabs JavaScript SDK", + "version": "1.0.0" + }, + "userAgent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.90 Safari/537.36", + "locale": "en-US", + "ip": "0.0.0.0", + "os": { + "name": "", + "version": "" + }, + "screen": { + "density": 2 + }, + "page": { + "path": "/destinations/ometria", + "referrer": "", + "search": "", + "title": "", + "url": "https://docs.rudderstack.com/destinations/ometria", + "category": "destination", + "initial_referrer": "https://docs.rudderstack.com", + "initial_referring_domain": "docs.rudderstack.com" + } + }, + "type": "track", + "messageId": "84e26acc-56a5-4835-8233-591137fca468", + "session_id": "3049dc4c-5a95-4ccd-a3e7-d74a7e411f22", + "originalTimestamp": "2019-10-14T09:03:17.562Z", + "anonymousId": "123456", + "event": "product list viewed", + "userId": "testuserId1", + "properties": { + "index": "products", + "products": [ + { + "objectId": "ecommerce-sample-data-919", + "position": "7" + }, + { + "objectId": "9780439784542", + "position": "a" + } + ], + "queryId": "43b15df305339e827f0ac0bdc5ebcaa7" + }, + "integrations": { + "All": true + }, + "sentAt": "2019-10-14T09:03:22.563Z" + }, + "destination": { + "Config": { + "apiKey": "34d8efa09c5b048bbacc6af157f2e687", + "applicationId": "O2YARRI15I", + "eventTypeSettings": [ + { + "from": "product list viewed", + "to": "click" + } + ] + }, + "DestinationDefinition": { + "Config": { + "cdkV2TestThreshold": 1 + } + } + } + }, + { + "message": { + "channel": "web", + "context": { + "app": { + "build": "1.0.0", + "name": "RudderLabs JavaScript SDK", + "namespace": "com.rudderlabs.javascript", + "version": "1.0.0" + }, + "traits": { + "email": "testone@gmail.com", + "firstName": "test", + "lastName": "one" + }, + "library": { + "name": "RudderLabs JavaScript SDK", + "version": "1.0.0" + }, + "userAgent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.90 Safari/537.36", + "locale": "en-US", + "ip": "0.0.0.0", + "os": { + "name": "", + "version": "" + }, + "screen": { + "density": 2 + }, + "page": { + "path": "/destinations/ometria", + "referrer": "", + "search": "", + "title": "", + "url": "https://docs.rudderstack.com/destinations/ometria", + "category": "destination", + "initial_referrer": "https://docs.rudderstack.com", + "initial_referring_domain": "docs.rudderstack.com" + } + }, + "type": "track", + "messageId": "84e26acc-56a5-4835-8233-591137fca468", + "session_id": "3049dc4c-5a95-4ccd-a3e7-d74a7e411f22", + "originalTimestamp": "2019-10-14T09:03:17.562Z", + "anonymousId": "123456", + "event": "product list viewed", + "userId": "testuserId1", + "properties": { + "index": "products", + "products": [ + { + "objectId": "ecommerce-sample-data-919", + "position": "7" + }, + { + "objectId": "9780439784542", + "position": "a" + } + ], + "queryId": "43b15df305339e827f0ac0bdc5ebcaa7" + }, + "integrations": { + "All": true + }, + "sentAt": "2019-10-14T09:03:22.563Z" + }, + "destination": { + "Config": { + "apiKey": "34d8efa09c5b048bbacc6af157f2e687", + "applicationId": "O2YARRI15I", + "eventTypeSettings": [ + { + "from": "product list viewed", + "to": "check" + } + ] + }, + "DestinationDefinition": { + "Config": { + "cdkV2TestThreshold": 1 + } + } + } + }, + { + "message": { + "channel": "web", + "context": { + "app": { + "build": "1.0.0", + "name": "RudderLabs JavaScript SDK", + "namespace": "com.rudderlabs.javascript", + "version": "1.0.0" + }, + "traits": { + "email": "testone@gmail.com", + "firstName": "test", + "lastName": "one" + }, + "library": { + "name": "RudderLabs JavaScript SDK", + "version": "1.0.0" + }, + "userAgent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.90 Safari/537.36", + "locale": "en-US", + "ip": "0.0.0.0", + "os": { + "name": "", + "version": "" + }, + "screen": { + "density": 2 + }, + "page": { + "path": "/destinations/ometria", + "referrer": "", + "search": "", + "title": "", + "url": "https://docs.rudderstack.com/destinations/ometria", + "category": "destination", + "initial_referrer": "https://docs.rudderstack.com", + "initial_referring_domain": "docs.rudderstack.com" + } + }, + "type": "track", + "messageId": "84e26acc-56a5-4835-8233-591137fca468", + "session_id": "3049dc4c-5a95-4ccd-a3e7-d74a7e411f22", + "originalTimestamp": "2019-10-14T09:03:17.562Z", + "anonymousId": "123456", + "event": "product list viewed", + "userId": "testuserId1", + "properties": { + "index": "products", + "products": [ + { + "objectId": "ecommerce-sample-data-919", + "position": "7" + }, + { + "objectId": "9780439784542", + "position": "a" + } + ], + "queryId": "43b15df305339e827f0ac0bdc5ebcaa7" + }, + "integrations": { + "All": true + }, + "sentAt": "2019-10-14T09:03:22.563Z" + }, + "destination": { + "Config": { + "apiKey": "34d8efa09c5b048bbacc6af157f2e687", + "applicationId": "O2YARRI15I" + }, + "DestinationDefinition": { + "Config": { + "cdkV2TestThreshold": 1 + } + } + } + } + ] + } +} \ No newline at end of file diff --git a/__tests__/data/versioned_processor_algolia_output.json b/__tests__/data/versioned_processor_algolia_output.json new file mode 100644 index 0000000000..1381f11649 --- /dev/null +++ b/__tests__/data/versioned_processor_algolia_output.json @@ -0,0 +1,161 @@ +[ + { + "output": { + "version": "1", + "type": "REST", + "method": "POST", + "endpoint": "https://insights.algolia.io/1/events", + "headers": { + "X-Algolia-Application-Id": "O2YARRI15I", + "X-Algolia-API-Key": "34d8efa09c5b048bbacc6af157f2e687" + }, + "params": {}, + "body": { + "JSON": { + "events": [ + { + "index": "products", + "userToken": "testuserId1", + "filters": ["field1:hello", "val1:val2"], + "eventName": "product clicked", + "eventType": "click" + } + ] + }, + "JSON_ARRAY": {}, + "XML": {}, + "FORM": {} + }, + "files": {}, + "userId": "" + }, + "statusCode": 200 + }, + { + "output": { + "version": "1", + "type": "REST", + "method": "POST", + "endpoint": "https://insights.algolia.io/1/events", + "headers": { + "X-Algolia-Application-Id": "O2YARRI15I", + "X-Algolia-API-Key": "34d8efa09c5b048bbacc6af157f2e687" + }, + "params": {}, + "body": { + "JSON": { + "events": [ + { + "index": "products", + "userToken": "testuserId1", + "queryID": "43b15df305339e827f0ac0bdc5ebcaa7", + "eventName": "product list viewed", + "eventType": "click", + "objectIDs": ["ecommerce-sample-data-919", "9780439784542"], + "positions": [7, 8] + } + ] + }, + "JSON_ARRAY": {}, + "XML": {}, + "FORM": {} + }, + "files": {}, + "userId": "" + }, + "statusCode": 200 + }, + { + "statusCode": 400, + "error": "Either filters or objectIds is required.", + "statTags": { + "errorAt": "proc", + "destType": "ALGOLIA", + "stage": "transform", + "scope": "exception" + } + }, + { + "statusCode": 400, + "error": "eventType is mandatory for track call", + "statTags": { + "errorAt": "proc", + "destType": "ALGOLIA", + "stage": "transform", + "scope": "exception" + } + }, + { + "statusCode": 400, + "error": "for click eventType either both positions and queryId should be present or none", + "statTags": { + "errorAt": "proc", + "destType": "ALGOLIA", + "stage": "transform", + "scope": "exception" + } + }, + { + "output": { + "version": "1", + "type": "REST", + "method": "POST", + "endpoint": "https://insights.algolia.io/1/events", + "headers": { + "X-Algolia-Application-Id": "O2YARRI15I", + "X-Algolia-API-Key": "34d8efa09c5b048bbacc6af157f2e687" + }, + "params": {}, + "body": { + "JSON": { + "events": [ + { + "index": "products", + "userToken": "testuserId1", + "queryID": "43b15df305339e827f0ac0bdc5ebcaa7", + "eventName": "product list viewed", + "eventType": "view", + "objectIDs": ["ecommerce-sample-data-919", "9780439784542"] + } + ] + }, + "JSON_ARRAY": {}, + "XML": {}, + "FORM": {} + }, + "files": {}, + "userId": "" + }, + "statusCode": 200 + }, + { + "statusCode": 400, + "error": "for click eventType either both positions and queryId should be present or none", + "statTags": { + "errorAt": "proc", + "destType": "ALGOLIA", + "stage": "transform", + "scope": "exception" + } + }, + { + "statusCode": 400, + "error": "eventType can be either click, view or conversion", + "statTags": { + "errorAt": "proc", + "destType": "ALGOLIA", + "stage": "transform", + "scope": "exception" + } + }, + { + "statusCode": 400, + "error": "eventType is mandatory for track call", + "statTags": { + "errorAt": "proc", + "destType": "ALGOLIA", + "stage": "transform", + "scope": "exception" + } + } +] diff --git a/__tests__/data/versioned_router_algolia_input.json b/__tests__/data/versioned_router_algolia_input.json new file mode 100644 index 0000000000..f729054978 --- /dev/null +++ b/__tests__/data/versioned_router_algolia_input.json @@ -0,0 +1,288 @@ +{ + "request": { + "body": { + "destType": "algolia", + "input": [ + { + "message": { + "type": "track", + "event": "product list viewed", + "sentAt": "2021-10-25T09:40:08.880Z", + "userId": "test-user-id1", + "channel": "web", + "context": { + "os": { + "name": "", + "version": "" + }, + "app": { + "name": "RudderLabs JavaScript SDK", + "build": "1.0.0", + "version": "1.2.1", + "namespace": "com.rudderlabs.javascript" + }, + "page": { + "url": "http://127.0.0.1:5500/index.html", + "path": "/index.html", + "title": "Test", + "search": "", + "tab_url": "http://127.0.0.1:5500/index.html", + "referrer": "http://127.0.0.1:5500/index.html", + "initial_referrer": "http://127.0.0.1:5500/index.html", + "referring_domain": "127.0.0.1:5500", + "initial_referring_domain": "127.0.0.1:5500" + }, + "locale": "en-GB", + "screen": { + "width": 1440, + "height": 900, + "density": 2, + "innerWidth": 1440, + "innerHeight": 335 + }, + "traits": { + "city": "Brussels", + "email": "testemail@email.com", + "phone": "1234567890", + "country": "Belgium", + "firstName": "Tintin", + "custom_date": 1574769933368, + "custom_date1": "2019-10-14T11:15:53.296Z", + "custom_flavor": "chocolate" + }, + "library": { + "name": "RudderLabs JavaScript SDK", + "version": "1.2.1" + }, + "campaign": {}, + "userAgent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/94.0.4606.71 Safari/537.36" + }, + "rudderId": "e3e907f1-f79a-444b-b91d-da47488f8273", + "messageId": "8cdd3d2e-5e07-42ec-abdc-9b6bd4333840", + "timestamp": "2021-10-25T15:10:08.888+05:30", + "properties": { + "index": "products", + "queryId": "43b15df305339e827f0ac0bdc5ebcaa7", + "products": [ + { + "objectId": "ecommerce-sample-data-919", + "position": 7 + }, + { + "objectId": "9780439784542", + "position": 8 + } + ] + }, + "receivedAt": "2021-10-25T15:10:08.889+05:30", + "request_ip": "[::1]", + "anonymousId": "7138f7d9-5dd2-4337-805d-ca17be59dc8e", + "integrations": { + "All": true + }, + "originalTimestamp": "2021-10-25T09:40:08.879Z" + }, + "metadata": { + "jobId": 1 + }, + "destination": { + "ID": "1zzHtStW2ZPlullmz6L7DGnmk9V", + "Name": "algolia-dev", + "DestinationDefinition": { + "ID": "1zgVZhcj1Tij4qlKg7B1Jp16IrH", + "Name": "ALGOLIA", + "DisplayName": "Algolia", + "Config": { + "destConfig": { + "defaultConfig": [ + "apiKey", + "applicationId", + "eventTypeSettings" + ] + }, + "excludeKeys": [], + "includeKeys": [], + "saveDestinationResponse": true, + "secretKeys": [ + "apiKey", + "applicationId" + ], + "supportedMessageTypes": [ + "track" + ], + "supportedSourceTypes": [ + "android", + "ios", + "web", + "unity", + "amp", + "cloud", + "reactnative", + "flutter", + "cordova" + ], + "transformAt": "router", + "transformAtV1": "router", + "cdkV2TestThreshold": 1 + }, + "ResponseRules": {} + }, + "Config": { + "apiKey": "apiKey", + "applicationId": "appId", + "eventTypeSettings": [ + { + "from": "product clicked", + "to": "click" + }, + { + "from": "product list viewed", + "to": "view" + } + ] + }, + "Enabled": true, + "Transformations": [], + "IsProcessorEnabled": true + } + }, + { + "message": { + "type": "track", + "event": "product clicked", + "sentAt": "2021-10-25T09:40:08.886Z", + "userId": "test-user-id1", + "channel": "web", + "context": { + "os": { + "name": "", + "version": "" + }, + "app": { + "name": "RudderLabs JavaScript SDK", + "build": "1.0.0", + "version": "1.2.1", + "namespace": "com.rudderlabs.javascript" + }, + "page": { + "url": "http://127.0.0.1:5500/index.html", + "path": "/index.html", + "title": "Test", + "search": "", + "tab_url": "http://127.0.0.1:5500/index.html", + "referrer": "http://127.0.0.1:5500/index.html", + "initial_referrer": "http://127.0.0.1:5500/index.html", + "referring_domain": "127.0.0.1:5500", + "initial_referring_domain": "127.0.0.1:5500" + }, + "locale": "en-GB", + "screen": { + "width": 1440, + "height": 900, + "density": 2, + "innerWidth": 1440, + "innerHeight": 335 + }, + "traits": { + "city": "Brussels", + "email": "testemail@email.com", + "phone": "1234567890", + "country": "Belgium", + "firstName": "Tintin", + "custom_date": 1574769933368, + "custom_date1": "2019-10-14T11:15:53.296Z", + "custom_flavor": "chocolate" + }, + "library": { + "name": "RudderLabs JavaScript SDK", + "version": "1.2.1" + }, + "campaign": {}, + "userAgent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/94.0.4606.71 Safari/537.36" + }, + "rudderId": "e3e907f1-f79a-444b-b91d-da47488f8273", + "messageId": "7b58e140-b66b-4e88-a5ec-bd6811fc3836", + "timestamp": "2021-10-25T15:10:08.943+05:30", + "properties": { + "index": "products", + "filters": [ + "field1:hello", + "val1:val2" + ] + }, + "receivedAt": "2021-10-25T15:10:08.943+05:30", + "request_ip": "[::1]", + "anonymousId": "7138f7d9-5dd2-4337-805d-ca17be59dc8e", + "integrations": { + "All": true + }, + "originalTimestamp": "2021-10-25T09:40:08.886Z" + }, + "metadata": { + "jobId": 2 + }, + "destination": { + "ID": "1zzHtStW2ZPlullmz6L7DGnmk9V", + "Name": "algolia-dev", + "DestinationDefinition": { + "ID": "1zgVZhcj1Tij4qlKg7B1Jp16IrH", + "Name": "ALGOLIA", + "DisplayName": "Algolia", + "Config": { + "destConfig": { + "defaultConfig": [ + "apiKey", + "applicationId", + "eventTypeSettings" + ] + }, + "excludeKeys": [], + "includeKeys": [], + "saveDestinationResponse": true, + "secretKeys": [ + "apiKey", + "applicationId" + ], + "supportedMessageTypes": [ + "track" + ], + "supportedSourceTypes": [ + "android", + "ios", + "web", + "unity", + "amp", + "cloud", + "reactnative", + "flutter", + "cordova" + ], + "transformAt": "router", + "transformAtV1": "router", + "cdkV2TestThreshold": 1 + }, + "ResponseRules": {} + }, + "Config": { + "apiKey": "apiKey", + "applicationId": "appId", + "eventTypeSettings": [ + { + "from": "product clicked", + "to": "click" + }, + { + "from": "product list viewed", + "to": "view" + } + ] + }, + "Enabled": true, + "Transformations": [], + "IsProcessorEnabled": true + } + } + ] + } + } +} \ No newline at end of file diff --git a/__tests__/data/versioned_router_algolia_output.json b/__tests__/data/versioned_router_algolia_output.json new file mode 100644 index 0000000000..11346e84a4 --- /dev/null +++ b/__tests__/data/versioned_router_algolia_output.json @@ -0,0 +1,90 @@ +{ + "output": [ + { + "batchedRequest": { + "version": "1", + "type": "REST", + "method": "POST", + "endpoint": "https://insights.algolia.io/1/events", + "headers": { + "X-Algolia-Application-Id": "appId", + "X-Algolia-API-Key": "apiKey" + }, + "params": {}, + "body": { + "JSON": { + "events": [ + { + "index": "products", + "userToken": "test-user-id1", + "queryID": "43b15df305339e827f0ac0bdc5ebcaa7", + "eventName": "product list viewed", + "eventType": "view", + "objectIDs": ["ecommerce-sample-data-919", "9780439784542"] + }, + { + "index": "products", + "userToken": "test-user-id1", + "filters": ["field1:hello", "val1:val2"], + "eventName": "product clicked", + "eventType": "click" + } + ] + }, + "JSON_ARRAY": {}, + "XML": {}, + "FORM": {} + }, + "files": {} + }, + "metadata": [{ "jobId": 1 }, { "jobId": 2 }], + "batched": true, + "statusCode": 200, + "destination": { + "ID": "1zzHtStW2ZPlullmz6L7DGnmk9V", + "Name": "algolia-dev", + "DestinationDefinition": { + "ID": "1zgVZhcj1Tij4qlKg7B1Jp16IrH", + "Name": "ALGOLIA", + "DisplayName": "Algolia", + "Config": { + "destConfig": { + "defaultConfig": ["apiKey", "applicationId", "eventTypeSettings"] + }, + "excludeKeys": [], + "includeKeys": [], + "saveDestinationResponse": true, + "secretKeys": ["apiKey", "applicationId"], + "supportedMessageTypes": ["track"], + "supportedSourceTypes": [ + "android", + "ios", + "web", + "unity", + "amp", + "cloud", + "reactnative", + "flutter", + "cordova" + ], + "transformAt": "router", + "transformAtV1": "router", + "cdkV2TestThreshold": 1 + }, + "ResponseRules": {} + }, + "Config": { + "apiKey": "apiKey", + "applicationId": "appId", + "eventTypeSettings": [ + { "from": "product clicked", "to": "click" }, + { "from": "product list viewed", "to": "view" } + ] + }, + "Enabled": true, + "Transformations": [], + "IsProcessorEnabled": true + } + } + ] +} diff --git a/__tests__/pinterestConversion-cdk.test.js b/__tests__/pinterestConversion-cdk.test.js index 3348d5cf05..10a1347f26 100644 --- a/__tests__/pinterestConversion-cdk.test.js +++ b/__tests__/pinterestConversion-cdk.test.js @@ -1,20 +1,11 @@ const fs = require("fs"); const path = require("path"); const { TRANSFORMER_METRIC } = require("../v0/util/constant"); -const { getWorkflowEngine } = require("../cdk/v2/handler"); +const { processCdkV2Workflow } = require("../cdk/v2/handler"); const integration = "pinterest_tag"; const name = "Pinterest Conversion API"; -const procWorkflowEnginePromise = getWorkflowEngine( - integration, - TRANSFORMER_METRIC.ERROR_AT.PROC -); -const rtWorkflowEnginePromise = getWorkflowEngine( - integration, - TRANSFORMER_METRIC.ERROR_AT.RT -); - describe(`${name} Tests`, () => { describe("Processor Tests", () => { const inputDataFile = fs.readFileSync( @@ -29,9 +20,12 @@ describe(`${name} Tests`, () => { it(`${name} - payload: ${index}`, async () => { const expected = expectedData[index]; try { - const procWorkflowEngine = await procWorkflowEnginePromise; - const result = await procWorkflowEngine.execute(input); - expect(result.output).toEqual(expected); + const output = await processCdkV2Workflow( + integration, + input, + TRANSFORMER_METRIC.ERROR_AT.PROC + ); + expect(output).toEqual(expected); } catch (error) { expect(error.message).toEqual(expected.error); } @@ -66,22 +60,32 @@ describe(`${name} Tests`, () => { const expectedRouterErrorData = JSON.parse(outputRouterErrorDataFile); it("Payload with error input", async () => { - const rtWorkflowEngine = await rtWorkflowEnginePromise; - const result = await rtWorkflowEngine.execute(inputRouterErrorData); - expect(result.output).toEqual(expectedRouterErrorData); + const output = await processCdkV2Workflow( + integration, + inputRouterErrorData, + TRANSFORMER_METRIC.ERROR_AT.RT + ); + expect(output).toEqual(expectedRouterErrorData); }); it("Payload with Default Batch size", async () => { - const rtWorkflowEngine = await rtWorkflowEnginePromise; - const result = await rtWorkflowEngine.execute(inputRouterData); - expect(result.output).toEqual(expectedRouterData); + const output = await processCdkV2Workflow( + integration, + inputRouterData, + TRANSFORMER_METRIC.ERROR_AT.RT + ); + expect(output).toEqual(expectedRouterData); }); it("Payload with Batch size 3", async () => { - const rtWorkflowEngine = await rtWorkflowEnginePromise; - const result = await rtWorkflowEngine.execute(inputRouterData, { - MAX_BATCH_SIZE: 3 - }); - expect(result.output).toEqual(expectedRouterBatchData); + const output = await processCdkV2Workflow( + integration, + inputRouterData, + TRANSFORMER_METRIC.ERROR_AT.RT, + { + MAX_BATCH_SIZE: 3 + } + ); + expect(output).toEqual(expectedRouterBatchData); }); }); }); diff --git a/__tests__/versionedRouter.test.js b/__tests__/versionedRouter.test.js index 7cf215231c..4e42720139 100644 --- a/__tests__/versionedRouter.test.js +++ b/__tests__/versionedRouter.test.js @@ -11,6 +11,9 @@ const typeArg = process.argv.filter(x => x.startsWith("--type="))[0]; // send ar // eg: jest versionedRouter --destName=heap --type=router // eg: jest versionedRouter --destName=heap --type=response // eg: jest versionedRouter --type=all +// To invoke CDK live compare: +// router: CDK_LIVE_TEST=1 npx jest versionedRouter --destName=algolia --type=router +// processor: CDK_LIVE_TEST=1 npx jest versionedRouter --destName=algolia --type=processor let destination; if (typeArg) { diff --git a/cdk/v2/destinations/algolia/procWorkflow.yaml b/cdk/v2/destinations/algolia/procWorkflow.yaml new file mode 100644 index 0000000000..873099e1b3 --- /dev/null +++ b/cdk/v2/destinations/algolia/procWorkflow.yaml @@ -0,0 +1,19 @@ +bindings: +- name: EventType + path: ../../../../constants + +steps: + - name: validateInput + template: | + ( + $assert($exists(message.type), "message Type is not present. Aborting message."); + $assert($exists(destination.Config.apiKey), "Invalid Api Key"); + $assert($exists(destination.Config.applicationId), "Invalid Application Id"); + $assert(message.type in [$EventType.TRACK], + "message type " & message.type & " not supported") + ) + - name: trackEventWorkflow + condition: message.type = $EventType.TRACK + externalWorkflow: + path: ./trackEventWorkflow.yaml + diff --git a/cdk/v2/destinations/algolia/rtWorkflow.yaml b/cdk/v2/destinations/algolia/rtWorkflow.yaml new file mode 100644 index 0000000000..5ac809afff --- /dev/null +++ b/cdk/v2/destinations/algolia/rtWorkflow.yaml @@ -0,0 +1,68 @@ +bindings: +- path: ../../../../v0/destinations/algolia/config + +steps: + - name: validateInput + template: | + ( + $assert($type($) = "array" and $count($) > 0, "Invalid event array") + ) + + - name: prepareContext + template: + $setContext("batchMode", true) + + - name: transform + externalWorkflow: + path: ./procWorkflow.yaml + loopOverInput: true + + - name: successfulEvents + template: | + $outputs.transform#$i.output.{ + "output": $, + /* $$ refers to input (root of the document) */ + "destination": $$[$i].destination, + "metadata": $$[$i].metadata + } ~> $toArray + - name: failedEvents + template: | + $outputs.transform#$i.error.{ + "metadata": $toArray($$[$i].metadata), + "batched": false, + "statusCode": status, + "error": message + } ~> $toArray + - name: batchSuccessfulEvents + description: Batches the successfulEvents + template: | + ( + $batches := $chunk($outputs.successfulEvents, $MAX_BATCH_SIZE); + $batches.{ + "batchedRequest": { + "body": { + "JSON": {"events": output}, + "JSON_ARRAY": {}, + "XML": {}, + "FORM": {} + }, + "version": "1", + "type": "REST", + "method": "POST", + "endpoint": $ENDPOINT, + "headers": $[0].destination.Config.{ + "X-Algolia-Application-Id": applicationId, + "X-Algolia-API-Key": apiKey + }, + "params": {}, + "files": {} + }, + "metadata": metadata, + "batched": true, + "statusCode": 200, + "destination": $[0].destination + } ~> $toArray; + ) + - name: finalPayload + template: | + $toArray($outputs.($append(batchSuccessfulEvents, failedEvents))) diff --git a/cdk/v2/destinations/algolia/trackEventWorkflow.yaml b/cdk/v2/destinations/algolia/trackEventWorkflow.yaml new file mode 100644 index 0000000000..7e8bae196f --- /dev/null +++ b/cdk/v2/destinations/algolia/trackEventWorkflow.yaml @@ -0,0 +1,86 @@ +bindings: + - path: ../../../../v0/destinations/algolia/util + - path: ../../../../v0/destinations/algolia/config + - path: ../../../../v0/util + +steps: + - name: validateInput + template: | + ( + $assert($exists(message.event), "event is required for track call"); + ) + + - name: preparePayload + template: | + ( + $eventTypeMap := $eventTypeMapping(destination.Config); + $event := message.event.$trim().$lowercase(); + $eventType := $getValueFromMessage(message, "properties.eventType") ? $getValueFromMessage(message, "properties.eventType") : $lookup($eventTypeMap, $event); + $assert($exists($eventType), "eventType is mandatory for track call"); + $setContext("payload", $constructPayload(message, $trackMapping)); + $newPayload := $merge([$context.payload, { "eventName": $event, "eventType": $eventType }]); + $setContext("payload", $newPayload); + $setContext("payload", $genericpayloadValidator($context.payload)); + ) + + - name: populateProductsData + condition: $context.payload.eventName in ["product list viewed", "order completed"] and $exists(message.properties.products) + steps: + - name: populateForClickEvent + condition: $context.payload.eventType = "click" + template: | + ( + $validProducts := message.properties.products[$exists(objectId) and $exists(position)]#$i[$i<20]; + $objectAndPositionIds := { + "objectIDs": $validProducts.objectId, + "positions": $validProducts.position + }; + $objLen := $count($validProducts.objectId); + $posLen := $count($validProducts.position); + $newPayload := $merge([$context.payload, $objectAndPositionIds]); + $setContext("payload", $clickPayloadValidator($newPayload)) + ) + - name: populateForOtherEvents + condition: $context.payload.eventType != "click" + template: | + ( + $objectIDs := { + "objectIDs": message.properties.products.objectId#$i[$i<20] + }; + $setContext("payload", $merge([$context.payload, $objectIDs])); + ) + + - name: validateDestPayload + template: | + ( + $assert($context.payload.($not(filters and objectIDs)), "event can't have both objectIds and filters at the same time."); + $assert($context.payload.(filters or objectIDs), "Either filters or objectIds is required."); + ) + + - name: prepareResponsePayloadForBatch + condition: $exists($context.batchMode) + template: | + $context.payload + onComplete: return + + - name: prepareResponsePayload + condition: $not($exists($context.batchMode)) + template: | + { + "body": { + "JSON": {"events": [$context.payload]}, + "JSON_ARRAY": {}, + "XML": {}, + "FORM": {} + }, + "version": "1", + "type": "REST", + "method": "POST", + "endpoint": $ENDPOINT, + "headers": { + "X-Algolia-Application-Id": destination.Config.applicationId, + "X-Algolia-API-Key": destination.Config.apiKey + }, + "params": {}, + "files": {} + } diff --git a/cdk/v2/pinterest_tag/procWorkflow.yaml b/cdk/v2/destinations/pinterest_tag/procWorkflow.yaml similarity index 97% rename from cdk/v2/pinterest_tag/procWorkflow.yaml rename to cdk/v2/destinations/pinterest_tag/procWorkflow.yaml index 333a6e072d..187041ab43 100644 --- a/cdk/v2/pinterest_tag/procWorkflow.yaml +++ b/cdk/v2/destinations/pinterest_tag/procWorkflow.yaml @@ -1,12 +1,12 @@ bindings: - name: EventType - path: ../../../constants + path: ../../../../constants - name: SourceKeys - path: ../../../v0/util/data/GenericFieldMapping.json + path: ../../../../v0/util/data/GenericFieldMapping.json exportAll: true - - path: ../../../v0/destinations/pinterest_tag/utils + - path: ../../../../v0/destinations/pinterest_tag/utils - name: ENDPOINT - path: ../../../v0/destinations/pinterest_tag/config + path: ../../../../v0/destinations/pinterest_tag/config steps: - name: checkIfProcessed condition: $exists(message.statusCode) diff --git a/cdk/v2/pinterest_tag/rtWorkflow.yaml b/cdk/v2/destinations/pinterest_tag/rtWorkflow.yaml similarity index 94% rename from cdk/v2/pinterest_tag/rtWorkflow.yaml rename to cdk/v2/destinations/pinterest_tag/rtWorkflow.yaml index 87e5f4b84d..714ca0f924 100644 --- a/cdk/v2/pinterest_tag/rtWorkflow.yaml +++ b/cdk/v2/destinations/pinterest_tag/rtWorkflow.yaml @@ -1,5 +1,5 @@ bindings: - - path: ../../../v0/destinations/pinterest_tag/config + - path: ../../../../v0/destinations/pinterest_tag/config steps: - name: validateInput template: | @@ -19,7 +19,7 @@ steps: /* $$ refers to input (root of the document) */ "destination": $$[$i].destination, "metadata": $$[$i].metadata - } + } ~> $toArray - name: failedEvents template: | $outputs.transform#$i.error.{ @@ -27,7 +27,7 @@ steps: "batched": false, "statusCode": status, "error": message - } + } ~> $toArray - name: batchSuccessfulEvents description: Batches the successfulEvents template: | diff --git a/cdk/v2/handler.js b/cdk/v2/handler.js index 34a8813fcb..452e57e8a2 100644 --- a/cdk/v2/handler.js +++ b/cdk/v2/handler.js @@ -1,9 +1,11 @@ const { WorkflowEngineFactory } = require("rudder-workflow-engine"); const { + getErrorInfo, getRootPathForDestination, getWorkflowPath, - getPlatformBindingsPaths + getPlatformBindingsPaths, + isCdkV2Destination } = require("./utils"); async function getWorkflowEngineInternal(destName, flowType) { @@ -33,6 +35,24 @@ function getWorkflowEngine(destName, flowType) { return workflowEnginePromiseMap[destName][flowType]; } +async function processCdkV2Workflow( + destType, + parsedEvent, + flowType, + bindings = {} +) { + try { + const workflowEngine = await getWorkflowEngine(destType, flowType); + + const result = await workflowEngine.execute(parsedEvent, bindings); + // TODO: Handle remaining output scenarios + return result.output; + } catch (error) { + throw getErrorInfo(error, isCdkV2Destination(parsedEvent)); + } +} + module.exports = { - getWorkflowEngine + getWorkflowEngine, + processCdkV2Workflow }; diff --git a/cdk/v2/utils.js b/cdk/v2/utils.js index b9e9401526..483d176564 100644 --- a/cdk/v2/utils.js +++ b/cdk/v2/utils.js @@ -49,7 +49,7 @@ async function getWorkflowPath( function getRootPathForDestination(destName) { // TODO: Resolve the CDK v2 destination directory // path from the root directory - return path.join(CDK_V2_ROOT_DIR, destName); + return path.join(CDK_V2_ROOT_DIR, "destinations", destName); } async function getPlatformBindingsPaths() { diff --git a/package-lock.json b/package-lock.json index d4df8fc64c..46215d66b0 100644 --- a/package-lock.json +++ b/package-lock.json @@ -49,7 +49,7 @@ "pprof": "^3.2.0", "prom-client": "^14.0.1", "rudder-transformer-cdk": "^1.4.3", - "rudder-workflow-engine": "^0.2.3", + "rudder-workflow-engine": "^0.2.4", "set-value": "^4.0.1", "sha256": "^0.2.0", "statsd-client": "^0.4.4", @@ -12300,9 +12300,9 @@ } }, "node_modules/pino": { - "version": "8.6.1", - "resolved": "https://registry.npmjs.org/pino/-/pino-8.6.1.tgz", - "integrity": "sha512-fi+V2K98eMZjQ/uEHHSiMALNrz7HaFdKNYuyA3ZUrbH0f1e8sPFDmeRGzg7ZH2q4QDxGnJPOswmqlEaTAZeDPA==", + "version": "8.7.0", + "resolved": "https://registry.npmjs.org/pino/-/pino-8.7.0.tgz", + "integrity": "sha512-l9sA5uPxmZzwydhMWUcm1gI0YxNnYl8MfSr2h8cwLvOAzQLBLewzF247h/vqHe3/tt6fgtXeG9wdjjoetdI/vA==", "dependencies": { "atomic-sleep": "^1.0.0", "fast-redact": "^3.1.1", @@ -14713,9 +14713,9 @@ } }, "node_modules/rudder-workflow-engine": { - "version": "0.2.3", - "resolved": "https://registry.npmjs.org/rudder-workflow-engine/-/rudder-workflow-engine-0.2.3.tgz", - "integrity": "sha512-5heQ5OuGPL/GeB0kFAr9kPD/PP+HvuSmUGQwXSovZVBm0sQfddEdSYlMc/gbMjuqCixBtZtAJoLXam4lVA1+mA==", + "version": "0.2.4", + "resolved": "https://registry.npmjs.org/rudder-workflow-engine/-/rudder-workflow-engine-0.2.4.tgz", + "integrity": "sha512-349KohoykhHbnjiJXakwvvZpYBjsWNTy4v3IkCPcyNSYOJJtmNYy9lK+EWNcbjsYoPnr0ik+q3on6fStuvo4og==", "dependencies": { "js-yaml": "^4.1.0", "jsonata": "^1.8.6", @@ -26690,9 +26690,9 @@ "dev": true }, "pino": { - "version": "8.6.1", - "resolved": "https://registry.npmjs.org/pino/-/pino-8.6.1.tgz", - "integrity": "sha512-fi+V2K98eMZjQ/uEHHSiMALNrz7HaFdKNYuyA3ZUrbH0f1e8sPFDmeRGzg7ZH2q4QDxGnJPOswmqlEaTAZeDPA==", + "version": "8.7.0", + "resolved": "https://registry.npmjs.org/pino/-/pino-8.7.0.tgz", + "integrity": "sha512-l9sA5uPxmZzwydhMWUcm1gI0YxNnYl8MfSr2h8cwLvOAzQLBLewzF247h/vqHe3/tt6fgtXeG9wdjjoetdI/vA==", "requires": { "atomic-sleep": "^1.0.0", "fast-redact": "^3.1.1", @@ -28519,9 +28519,9 @@ } }, "rudder-workflow-engine": { - "version": "0.2.3", - "resolved": "https://registry.npmjs.org/rudder-workflow-engine/-/rudder-workflow-engine-0.2.3.tgz", - "integrity": "sha512-5heQ5OuGPL/GeB0kFAr9kPD/PP+HvuSmUGQwXSovZVBm0sQfddEdSYlMc/gbMjuqCixBtZtAJoLXam4lVA1+mA==", + "version": "0.2.4", + "resolved": "https://registry.npmjs.org/rudder-workflow-engine/-/rudder-workflow-engine-0.2.4.tgz", + "integrity": "sha512-349KohoykhHbnjiJXakwvvZpYBjsWNTy4v3IkCPcyNSYOJJtmNYy9lK+EWNcbjsYoPnr0ik+q3on6fStuvo4og==", "requires": { "js-yaml": "^4.1.0", "jsonata": "^1.8.6", diff --git a/package.json b/package.json index 1b3ee3ec6e..10f26bdd61 100644 --- a/package.json +++ b/package.json @@ -30,8 +30,8 @@ "dependencies": { "@amplitude/ua-parser-js": "^0.7.24", "@aws-sdk/client-lambda": "^3.186.0", - "@aws-sdk/lib-storage": "^3.142.0", "@aws-sdk/client-s3": "^3.180.0", + "@aws-sdk/lib-storage": "^3.142.0", "@bugsnag/js": "^7.18.0", "@ndhoule/extend": "^2.0.0", "ajv": "^8.11.0", @@ -68,7 +68,7 @@ "pprof": "^3.2.0", "prom-client": "^14.0.1", "rudder-transformer-cdk": "^1.4.3", - "rudder-workflow-engine": "^0.2.3", + "rudder-workflow-engine": "^0.2.4", "set-value": "^4.0.1", "sha256": "^0.2.0", "statsd-client": "^0.4.4", diff --git a/v0/destinations/algolia/transform.js b/v0/destinations/algolia/transform.js index 4050a82add..6b595f34a4 100644 --- a/v0/destinations/algolia/transform.js +++ b/v0/destinations/algolia/transform.js @@ -118,9 +118,8 @@ const process = event => { }; const processRouterDest = async inputs => { - const errorRespEvents = checkInvalidRtTfEvents(inputs, "ALGOLIA"); - if (errorRespEvents.length > 0) { - return errorRespEvents; + if (!Array.isArray(inputs) || inputs.length === 0) { + throw new CustomError("Invalid event array", 400); } const inputChunks = returnArrayOfSubarrays(inputs, MAX_BATCH_SIZE); diff --git a/versionedRouter.js b/versionedRouter.js index c03a99a3ef..c3f6065d2a 100644 --- a/versionedRouter.js +++ b/versionedRouter.js @@ -37,12 +37,8 @@ const { getIntegrations } = require("./routes/utils"); const { setupUserTransformHandler } = require("./util/customTransformer"); const { CommonUtils } = require("./util/common"); const { RespStatusError, RetryRequestError } = require("./util/utils"); -const { getWorkflowEngine } = require("./cdk/v2/handler"); -const { - getErrorInfo, - isCdkV2Destination, - getCdkV2TestThreshold -} = require("./cdk/v2/utils"); +const { isCdkV2Destination, getCdkV2TestThreshold } = require("./cdk/v2/utils"); +const { getWorkflowEngine, processCdkV2Workflow } = require("./cdk/v2/handler"); const CDK_DEST_PATH = "cdk"; const basePath = path.resolve(__dirname, `./${CDK_DEST_PATH}`); @@ -111,23 +107,11 @@ function getCommonMetadata(ctx) { }; } -async function handleCdkV2(destType, parsedEvent, flowType) { - try { - const workflowEngine = await getWorkflowEngine(destType, flowType); - - const result = await workflowEngine.execute(parsedEvent); - // TODO: Handle remaining output scenarios - return result.output; - } catch (error) { - throw getErrorInfo(error, isCdkV2Destination(parsedEvent)); - } -} - async function getCdkV2Result(destName, event, flowType) { const cdkResult = {}; try { cdkResult.output = JSON.parse( - JSON.stringify(await handleCdkV2(destName, event, flowType)) + JSON.stringify(await processCdkV2Workflow(destName, event, flowType)) ); } catch (error) { cdkResult.error = { @@ -138,22 +122,6 @@ async function getCdkV2Result(destName, event, flowType) { return cdkResult; } -function removeSensitiveData(result) { - const newResult = {}; - Object.keys(result).forEach(key => { - if ( - key.includes("metadata") || - key.includes("error") || - key.includes("statusCode") - ) { - newResult[key] = result[key]; - } else { - newResult[key] = "***"; - } - }); - return newResult; -} - async function compareWithCdkV2(destType, input, flowType, v0Result) { try { const envThreshold = parseFloat(process.env.CDK_LIVE_TEST || "0", 10); @@ -256,7 +224,7 @@ async function handleDest(ctx, version, destination) { parsedEvent = processDynamicConfig(parsedEvent); let respEvents; if (isCdkV2Destination(parsedEvent)) { - respEvents = await handleCdkV2( + respEvents = await processCdkV2Workflow( destination, parsedEvent, TRANSFORMER_METRIC.ERROR_AT.PROC @@ -428,47 +396,90 @@ async function handleValidation(ctx) { }); } -async function routerHandleDest(ctx) { - const { destType, input } = ctx.request.body; - const routerDestHandler = getDestHandler("v0", destType); - if (!routerDestHandler || !routerDestHandler.processRouterDest) { - ctx.status = 404; - ctx.body = `${destType} doesn't support router transform`; - return null; +async function isValidRouterDest(event, destType) { + const isCdkV2Dest = isCdkV2Destination(event); + if (isCdkV2Dest) { + try { + await getWorkflowEngine(destType, TRANSFORMER_METRIC.ERROR_AT.RT); + return true; + } catch (error) { + return false; + } + } + try { + const routerDestHandler = getDestHandler("v0", destType); + return routerDestHandler?.processRouterDest !== undefined; + } catch (error) { + return false; } +} +async function routerHandleDest(ctx) { const respEvents = []; - const allDestEvents = _.groupBy(input, event => event.destination.ID); - await Promise.all( - Object.values(allDestEvents).map(async destInput => { - const newDestInput = processDynamicConfig(destInput, "router"); - let listOutput; - if (isCdkV2Destination(newDestInput[0])) { - listOutput = await handleCdkV2( - destType, - newDestInput, - TRANSFORMER_METRIC.ERROR_AT.RT - ); - } else { - listOutput = await handleV0Destination( - routerDestHandler.processRouterDest, - destType, - newDestInput, - TRANSFORMER_METRIC.ERROR_AT.RT + let destType; + try { + const { input } = ctx.request.body; + destType = ctx.request.body.destType; + const isValidRTDest = await isValidRouterDest(input[0], destType); + if (!isValidRTDest) { + ctx.status = 404; + ctx.body = `${destType} doesn't support router transform`; + return null; + } + const allDestEvents = _.groupBy(input, event => event.destination.ID); + await Promise.all( + Object.values(allDestEvents).map(async destInputArray => { + const newDestInputArray = processDynamicConfig( + destInputArray, + "router" ); + let listOutput; + if (isCdkV2Destination(newDestInputArray[0])) { + listOutput = await processCdkV2Workflow( + destType, + newDestInputArray, + TRANSFORMER_METRIC.ERROR_AT.RT + ); + } else { + const routerDestHandler = getDestHandler("v0", destType); + listOutput = await handleV0Destination( + routerDestHandler.processRouterDest, + destType, + newDestInputArray, + TRANSFORMER_METRIC.ERROR_AT.RT + ); + } + respEvents.push(...listOutput); + }) + ); + respEvents + .filter( + resp => + "error" in resp && + _.isObject(resp.statTags) && + !_.isEmpty(resp.statTags) + ) + .forEach(resp => { + set(resp, "statTags.errorAt", TRANSFORMER_METRIC.ERROR_AT.RT); + }); + } catch (error) { + logger.error(error); + const errObj = generateErrorObject( + error, + destType, + TRANSFORMER_METRIC.TRANSFORMER_STAGE.TRANSFORM + ); + + const resp = { + statusCode: errObj.status, + error: errObj.message || "Error occurred while processing the payload.", + statTags: { + ...errObj.statTags, + errorAt: TRANSFORMER_METRIC.ERROR_AT.RT } - respEvents.push(...listOutput); - }) - ); - respEvents - .filter( - resp => - "error" in resp && - _.isObject(resp.statTags) && - !_.isEmpty(resp.statTags) - ) - .forEach(resp => { - set(resp, "statTags.errorAt", TRANSFORMER_METRIC.ERROR_AT.RT); - }); + }; + + respEvents.push(resp); + } ctx.body = { output: respEvents }; return ctx.body; }