From 4b5ea130df4a17887190ff403e5294be47b30876 Mon Sep 17 00:00:00 2001 From: Sai Sankeerth Date: Tue, 20 Sep 2022 22:27:43 +0530 Subject: [PATCH 1/9] fix(marketo): unhandled exception status code fix --- v0/destinations/marketo/util.js | 29 +++++++++++++++++++++++++---- 1 file changed, 25 insertions(+), 4 deletions(-) diff --git a/v0/destinations/marketo/util.js b/v0/destinations/marketo/util.js index 330b989740..dc20969f20 100644 --- a/v0/destinations/marketo/util.js +++ b/v0/destinations/marketo/util.js @@ -13,7 +13,15 @@ const { TRANSFORMER_METRIC } = require("../../util/constant"); const ErrorBuilder = require("../../util/error"); const MARKETO_RETRYABLE_CODES = ["601", "602", "604", "611"]; -const MARKETO_ABORTABLE_CODES = ["600", "603", "605", "609", "610", "612"]; +const MARKETO_ABORTABLE_CODES = [ + "600", + "603", + "605", + "609", + "610", + "612", + "1006" +]; const MARKETO_THROTTLED_CODES = ["502", "606", "607", "608", "615"]; const { DESTINATION } = require("./config"); @@ -91,14 +99,13 @@ const marketoResponseHandler = (destResponse, sourceMessage, stage) => { } // non 2xx failure throw new ErrorBuilder() - .setStatus(status) + .setStatus(400) .setMessage(`Error occured ${sourceMessage}`) .setDestinationResponse(destResponse) .setStatTags({ destType: DESTINATION, stage, - scope: TRANSFORMER_METRIC.MEASUREMENT_TYPE.API.SCOPE, - meta: getDynamicMeta(status) + scope: TRANSFORMER_METRIC.MEASUREMENT_TYPE.EXCEPTION.SCOPE }) .build(); }; @@ -154,6 +161,20 @@ const responseHandler = (destinationResponse, _dest) => { "during Marketo Response Handling", TRANSFORMER_METRIC.TRANSFORMER_STAGE.RESPONSE_TRANSFORM ); + const { response } = destinationResponse; + if (response && !response.success) { + // non 2xx failure + throw new ErrorBuilder() + .setStatus(400) + .setMessage(`Error occured during Marketo Response Handling`) + .setDestinationResponse(destinationResponse.response) + .setStatTags({ + destType: DESTINATION, + stage: TRANSFORMER_METRIC.TRANSFORMER_STAGE.RESPONSE_TRANSFORM, + scope: TRANSFORMER_METRIC.MEASUREMENT_TYPE.EXCEPTION.SCOPE + }) + .build(); + } // else successfully return status, message and original destination response return { status, From 0b2fb8d715486c48a175819f0d229034cf1a3e0d Mon Sep 17 00:00:00 2001 From: Sai Sankeerth Date: Tue, 20 Sep 2022 22:28:10 +0530 Subject: [PATCH 2/9] test-cases for marketo unhandled error fix --- __mocks__/data/marketo/response.json | 21 ++ __tests__/data/marketo_router_input.json | 401 ++++++++++++++++++++-- __tests__/data/marketo_router_output.json | 16 + 3 files changed, 402 insertions(+), 36 deletions(-) diff --git a/__mocks__/data/marketo/response.json b/__mocks__/data/marketo/response.json index 1d207a6349..ff68d1a9e2 100644 --- a/__mocks__/data/marketo/response.json +++ b/__mocks__/data/marketo/response.json @@ -274,5 +274,26 @@ "message": "Any other error" } ] + }, + "https://520-BRK-122.mktorest.com/identity/oauth/token": { + "access_token": "access_token_success", + "expires_in": 3599, + "scope": "integrations@rudderstack.com", + "token_type": "bearer" + }, + "https://520-BRK-122.mktorest.com/rest/v1/leads.json?filterType=email&filterValues=0c7b8b80-9c43-4f8e-b2d2-5e2448a25040@j.mail": { + "requestId": "12093#17672aeaee6", + "result": [], + "success": true + }, + "https://520-BRK-122.mktorest.com/rest/v1/leads.json": { + "requestId": "142e4#1835b117b76", + "success": false, + "errors": [ + { + "code": "1006", + "message": "Lookup field 'userId' not found" + } + ] } } diff --git a/__tests__/data/marketo_router_input.json b/__tests__/data/marketo_router_input.json index 91d2ed9191..3879087a0f 100644 --- a/__tests__/data/marketo_router_input.json +++ b/__tests__/data/marketo_router_input.json @@ -65,13 +65,29 @@ "clientSecret": "marketo_client_secret_success", "trackAnonymousEvents": true, "customActivityEventMap": [ - { "from": "Product Clicked", "to": "100001" } + { + "from": "Product Clicked", + "to": "100001" + } + ], + "customActivityPropertyMap": [ + { + "from": "name", + "to": "productName" + } ], - "customActivityPropertyMap": [{ "from": "name", "to": "productName" }], "customActivityPrimaryKeyMap": [ - { "from": "Product Clicked", "to": "name" } + { + "from": "Product Clicked", + "to": "name" + } ], - "leadTraitMapping": [{ "from": "leadScore", "to": "customLeadScore" }] + "leadTraitMapping": [ + { + "from": "leadScore", + "to": "customLeadScore" + } + ] }, "secretConfig": {}, "ID": "1mMy5cqbtfuaKZv1IhVQKnBdVwe", @@ -95,7 +111,9 @@ "leadTraitMapping" ] }, - "secretKeys": ["clientSecret"], + "secretKeys": [ + "clientSecret" + ], "excludeKeys": [], "includeKeys": [], "routerTransform": true, @@ -113,20 +131,56 @@ "responseType": "JSON", "rules": { "retryable": [ - { "success": "false", "errors.0.code": 600 }, - { "success": "false", "errors.0.code": 601 }, - { "success": "false", "errors.0.code": 602 }, - { "success": "false", "errors.0.code": 604 }, - { "success": "false", "errors.0.code": 606 }, - { "success": "false", "errors.0.code": 607 }, - { "success": "false", "errors.0.code": 608 }, - { "success": "false", "errors.0.code": 611 } + { + "success": "false", + "errors.0.code": 600 + }, + { + "success": "false", + "errors.0.code": 601 + }, + { + "success": "false", + "errors.0.code": 602 + }, + { + "success": "false", + "errors.0.code": 604 + }, + { + "success": "false", + "errors.0.code": 606 + }, + { + "success": "false", + "errors.0.code": 607 + }, + { + "success": "false", + "errors.0.code": 608 + }, + { + "success": "false", + "errors.0.code": 611 + } ], "abortable": [ - { "success": "false", "errors.0.code": 603 }, - { "success": "false", "errors.0.code": 605 }, - { "success": "false", "errors.0.code": 609 }, - { "success": "false", "errors.0.code": 610 } + { + "success": "false", + "errors.0.code": 603 + }, + { + "success": "false", + "errors.0.code": 605 + }, + { + "success": "false", + "errors.0.code": 609 + }, + { + "success": "false", + "errors.0.code": 610 + } ] } }, @@ -208,13 +262,29 @@ "clientSecret": "marketo_client_secret_success", "trackAnonymousEvents": true, "customActivityEventMap": [ - { "from": "Product Clicked", "to": "100001" } + { + "from": "Product Clicked", + "to": "100001" + } + ], + "customActivityPropertyMap": [ + { + "from": "name", + "to": "productName" + } ], - "customActivityPropertyMap": [{ "from": "name", "to": "productName" }], "customActivityPrimaryKeyMap": [ - { "from": "Product Clicked", "to": "name" } + { + "from": "Product Clicked", + "to": "name" + } ], - "leadTraitMapping": [{ "from": "leadScore", "to": "customLeadScore" }] + "leadTraitMapping": [ + { + "from": "leadScore", + "to": "customLeadScore" + } + ] }, "secretConfig": {}, "ID": "1mMy5cqbtfuaKZv1IhVQKnBdVwe", @@ -238,7 +308,220 @@ "leadTraitMapping" ] }, - "secretKeys": ["clientSecret"], + "secretKeys": [ + "clientSecret" + ], + "excludeKeys": [], + "includeKeys": [], + "routerTransform": true, + "supportedSourceTypes": [ + "android", + "ios", + "web", + "unity", + "amp", + "cloud", + "reactnative" + ] + }, + "responseRules": { + "responseType": "JSON", + "rules": { + "retryable": [ + { + "success": "false", + "errors.0.code": 600 + }, + { + "success": "false", + "errors.0.code": 601 + }, + { + "success": "false", + "errors.0.code": 602 + }, + { + "success": "false", + "errors.0.code": 604 + }, + { + "success": "false", + "errors.0.code": 606 + }, + { + "success": "false", + "errors.0.code": 607 + }, + { + "success": "false", + "errors.0.code": 608 + }, + { + "success": "false", + "errors.0.code": 611 + } + ], + "abortable": [ + { + "success": "false", + "errors.0.code": 603 + }, + { + "success": "false", + "errors.0.code": 605 + }, + { + "success": "false", + "errors.0.code": 609 + }, + { + "success": "false", + "errors.0.code": 610 + } + ] + } + }, + "id": "1aIXqM806xAVm92nx07YwKbRrO9", + "name": "MARKETO", + "displayName": "Marketo", + "createdAt": "2020-04-09T09:24:31.794Z", + "updatedAt": "2021-01-11T11:03:28.103Z" + }, + "transformations": [], + "isConnectionEnabled": true, + "isProcessorEnabled": true + } + }, + { + "message": { + "type": "identify", + "sentAt": "2022-09-19T10:34:02.002Z", + "userId": "e17c5a5e-5e2f-430b-b497-fe3f1ea3a704", + "channel": "web", + "context": { + "os": { + "name": "", + "version": "" + }, + "app": { + "name": "RudderLabs JavaScript SDK", + "build": "1.0.0", + "version": "2.12.1", + "namespace": "com.rudderlabs.javascript" + }, + "page": { + "url": "https://accounts.app.t2.broken.com/home", + "path": "/home", + "title": "Home", + "search": "", + "tab_url": "https://accounts.app.t2.broken.com/home", + "referrer": "https://ts50-cvii.core.broken.org/", + "initial_referrer": "https://ts50-cvii.core.broken.org/", + "referring_domain": "ts50-cvii.core.broken.org", + "initial_referring_domain": "ts50-cvii.core.broken.org" + }, + "locale": "en-IN", + "screen": { + "width": 1728, + "height": 1117, + "density": 2, + "innerWidth": 1728, + "innerHeight": 969 + }, + "traits": { + "name": "AM", + "email": "0c7b8b80-9c43-4f8e-b2d2-5e2448a25040@j.mail", + "lastName": "M", + "firstName": "A", + "accountName": "MACDEV", + "billingName": "g g", + "companyName": "macDev", + "currentTier": "Webinar Pro 250", + "billingCycle": "Annually", + "lastBillingDate": "2022-06-29T09:40:42.000Z", + "nextBillingDate": "2023-09-09T04:00:00.000Z", + "subscriptionType": "Webinar Pro 250", + "subscriptionStatus": "ACTIVE", + "lastWebinarEventDate": "2022-09-15T20:00:00.000Z", + "nextWebinarEventDate": "2022-09-16T06:15:00.000Z" + }, + "library": { + "name": "RudderLabs JavaScript SDK", + "version": "2.12.1" + }, + "campaign": {}, + "userAgent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/105.0.0.0 Safari/537.36" + }, + "rudderId": "73dc83ef-587f-4077-90f3-c36083e64019", + "messageId": "1663583642000200-e3f31c4c-6361-4f99-b643-7755303a6007", + "timestamp": "2022-09-19T10:34:02.954Z", + "receivedAt": "2022-09-19T10:34:02.956Z", + "request_ip": "11.105.44.120", + "anonymousId": "0c7b8b80-9c43-4f8e-b2d2-5e2448a25040", + "integrations": { + "All": true + }, + "originalTimestamp": "2022-09-19T10:34:02.000Z" + }, + "destination": { + "Config": { + "accountId": "520-BRK-122", + "clientId": "504300cd-76b2-a7l4-bhle-90a07420nx73", + "clientSecret": "3l3gJpzRsZagD6gu7tnTeKXz0bomLGnd", + "trackAnonymousEvents": false, + "createIfNotExist": true, + "customActivityEventMap": [ + { + "from": "acq_signup_completed", + "to": "100026" + }, + { + "from": "act_createwebinarform_submit", + "to": "100025" + }, + { + "from": "act_presentation_style", + "to": "100025" + }, + { + "from": "act_webinar_view", + "to": "100025" + }, + { + "from": "act_webinar_join", + "to": "100025" + }, + { + "from": "act_presentation_addteammember", + "to": "100025" + }, + { + "from": "act_engagement_discussions_savediscussion", + "to": "100025" + }, + { + "to": "100025", + "from": "act_engagement_networking_savetime" + } + ] + }, + "destinationDefinition": { + "config": { + "destConfig": { + "defaultConfig": [ + "accountId", + "clientId", + "clientSecret", + "trackAnonymousEvents", + "customActivityEventMap", + "customActivityPropertyMap", + "customActivityPrimaryKeyMap", + "leadTraitMapping" + ] + }, + "secretKeys": [ + "clientSecret" + ], "excludeKeys": [], "includeKeys": [], "routerTransform": true, @@ -256,20 +539,56 @@ "responseType": "JSON", "rules": { "retryable": [ - { "success": "false", "errors.0.code": 600 }, - { "success": "false", "errors.0.code": 601 }, - { "success": "false", "errors.0.code": 602 }, - { "success": "false", "errors.0.code": 604 }, - { "success": "false", "errors.0.code": 606 }, - { "success": "false", "errors.0.code": 607 }, - { "success": "false", "errors.0.code": 608 }, - { "success": "false", "errors.0.code": 611 } + { + "success": "false", + "errors.0.code": 600 + }, + { + "success": "false", + "errors.0.code": 601 + }, + { + "success": "false", + "errors.0.code": 602 + }, + { + "success": "false", + "errors.0.code": 604 + }, + { + "success": "false", + "errors.0.code": 606 + }, + { + "success": "false", + "errors.0.code": 607 + }, + { + "success": "false", + "errors.0.code": 608 + }, + { + "success": "false", + "errors.0.code": 611 + } ], "abortable": [ - { "success": "false", "errors.0.code": 603 }, - { "success": "false", "errors.0.code": 605 }, - { "success": "false", "errors.0.code": 609 }, - { "success": "false", "errors.0.code": 610 } + { + "success": "false", + "errors.0.code": 603 + }, + { + "success": "false", + "errors.0.code": 605 + }, + { + "success": "false", + "errors.0.code": 609 + }, + { + "success": "false", + "errors.0.code": 610 + } ] } }, @@ -279,10 +598,20 @@ "createdAt": "2020-04-09T09:24:31.794Z", "updatedAt": "2021-01-11T11:03:28.103Z" }, + "secretConfig": {}, + "ID": "1mMy5cqbtfuaKZv1IhVQKnBdVke", + "name": "Marketo", + "enabled": true, + "workspaceId": "1TSN08muJTZwH8iCDmnnRt1pmMd", + "deleted": false, + "createdAt": "2022-02-10T08:39:32.005Z", + "updatedAt": "2022-09-03T16:22:31.374Z", "transformations": [], "isConnectionEnabled": true, "isProcessorEnabled": true + }, + "metadata": { + "jobId": 3 } } -] - +] \ No newline at end of file diff --git a/__tests__/data/marketo_router_output.json b/__tests__/data/marketo_router_output.json index a7aa9dbdc4..bab875005c 100644 --- a/__tests__/data/marketo_router_output.json +++ b/__tests__/data/marketo_router_output.json @@ -236,5 +236,21 @@ "isConnectionEnabled": true, "isProcessorEnabled": true } + }, + { + "batched": false, + "statusCode": 400, + "error": "Request Failed for Marketo, Lookup field 'userId' not found (Aborted).[Marketo Transformer]: During lead look up using email", + "statTags": { + "destType": "MARKETO", + "stage": "transform", + "scope": "api", + "meta": "abortable" + }, + "metadata": [ + { + "jobId": 3 + } + ] } ] From aaf6d5a203add4dc05fa34a52e09859739914b98 Mon Sep 17 00:00:00 2001 From: Sai Sankeerth Date: Wed, 21 Sep 2022 13:32:25 +0530 Subject: [PATCH 3/9] adding error doc ref and changing account-id to a more random one in tests --- __mocks__/data/marketo/response.json | 6 +++--- __tests__/data/marketo_router_input.json | 2 +- v0/destinations/marketo/util.js | 5 +++++ 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/__mocks__/data/marketo/response.json b/__mocks__/data/marketo/response.json index ff68d1a9e2..c357d52fb4 100644 --- a/__mocks__/data/marketo/response.json +++ b/__mocks__/data/marketo/response.json @@ -275,18 +275,18 @@ } ] }, - "https://520-BRK-122.mktorest.com/identity/oauth/token": { + "https://valid_account_broken_event.mktorest.com/identity/oauth/token": { "access_token": "access_token_success", "expires_in": 3599, "scope": "integrations@rudderstack.com", "token_type": "bearer" }, - "https://520-BRK-122.mktorest.com/rest/v1/leads.json?filterType=email&filterValues=0c7b8b80-9c43-4f8e-b2d2-5e2448a25040@j.mail": { + "https://valid_account_broken_event.mktorest.com/rest/v1/leads.json?filterType=email&filterValues=0c7b8b80-9c43-4f8e-b2d2-5e2448a25040@j.mail": { "requestId": "12093#17672aeaee6", "result": [], "success": true }, - "https://520-BRK-122.mktorest.com/rest/v1/leads.json": { + "https://valid_account_broken_event.mktorest.com/rest/v1/leads.json": { "requestId": "142e4#1835b117b76", "success": false, "errors": [ diff --git a/__tests__/data/marketo_router_input.json b/__tests__/data/marketo_router_input.json index 3879087a0f..8e1de5a329 100644 --- a/__tests__/data/marketo_router_input.json +++ b/__tests__/data/marketo_router_input.json @@ -465,7 +465,7 @@ }, "destination": { "Config": { - "accountId": "520-BRK-122", + "accountId": "valid_account_broken_event", "clientId": "504300cd-76b2-a7l4-bhle-90a07420nx73", "clientSecret": "3l3gJpzRsZagD6gu7tnTeKXz0bomLGnd", "trackAnonymousEvents": false, diff --git a/v0/destinations/marketo/util.js b/v0/destinations/marketo/util.js index dc20969f20..4f98f09063 100644 --- a/v0/destinations/marketo/util.js +++ b/v0/destinations/marketo/util.js @@ -12,6 +12,11 @@ const { isHttpStatusSuccess } = require("../../util/index"); const { TRANSFORMER_METRIC } = require("../../util/constant"); const ErrorBuilder = require("../../util/error"); +/** + * The error codes that are generated by Marketo are present in the mentioned link: + * https://developers.facebook.com/docs/marketing-api/error-reference/ + */ + const MARKETO_RETRYABLE_CODES = ["601", "602", "604", "611"]; const MARKETO_ABORTABLE_CODES = [ "600", From 32e736c01712451926fd7d654a7045375598d160 Mon Sep 17 00:00:00 2001 From: Sai Sankeerth Date: Wed, 21 Sep 2022 13:55:44 +0530 Subject: [PATCH 4/9] corrected ref link for error codes --- v0/destinations/marketo/util.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/v0/destinations/marketo/util.js b/v0/destinations/marketo/util.js index 4f98f09063..b3dbe47d46 100644 --- a/v0/destinations/marketo/util.js +++ b/v0/destinations/marketo/util.js @@ -14,7 +14,7 @@ const ErrorBuilder = require("../../util/error"); /** * The error codes that are generated by Marketo are present in the mentioned link: - * https://developers.facebook.com/docs/marketing-api/error-reference/ + * https://developers.marketo.com/rest-api/error-codes/ */ const MARKETO_RETRYABLE_CODES = ["601", "602", "604", "611"]; From a5b4211cbbe7ca6ec60402e5c3db8ff41e7f47fa Mon Sep 17 00:00:00 2001 From: Sai Sankeerth Date: Wed, 21 Sep 2022 20:12:56 +0530 Subject: [PATCH 5/9] refactor into a single method --- v0/destinations/marketo/util.js | 16 +--------------- 1 file changed, 1 insertion(+), 15 deletions(-) diff --git a/v0/destinations/marketo/util.js b/v0/destinations/marketo/util.js index b3dbe47d46..70be09ae5e 100644 --- a/v0/destinations/marketo/util.js +++ b/v0/destinations/marketo/util.js @@ -161,25 +161,11 @@ const responseHandler = (destinationResponse, _dest) => { .build(); } // check for marketo application level failures - marketoApplicationErrorHandler( + marketoResponseHandler( destinationResponse, "during Marketo Response Handling", TRANSFORMER_METRIC.TRANSFORMER_STAGE.RESPONSE_TRANSFORM ); - const { response } = destinationResponse; - if (response && !response.success) { - // non 2xx failure - throw new ErrorBuilder() - .setStatus(400) - .setMessage(`Error occured during Marketo Response Handling`) - .setDestinationResponse(destinationResponse.response) - .setStatTags({ - destType: DESTINATION, - stage: TRANSFORMER_METRIC.TRANSFORMER_STAGE.RESPONSE_TRANSFORM, - scope: TRANSFORMER_METRIC.MEASUREMENT_TYPE.EXCEPTION.SCOPE - }) - .build(); - } // else successfully return status, message and original destination response return { status, From 8d1a0126c75baff629d698bf6df5c8bfe7a5a129 Mon Sep 17 00:00:00 2001 From: Sai Sankeerth Date: Thu, 22 Sep 2022 06:50:40 +0530 Subject: [PATCH 6/9] make message more readable, add cases for unhandled exception in transformation, proxy and add successful case for identify --- __mocks__/data/marketo/proxy_response.json | 14 + __mocks__/data/marketo/response.json | 52 +++ __tests__/data/marketo_proxy_input.json | 34 ++ __tests__/data/marketo_proxy_output.json | 25 ++ __tests__/data/marketo_router_input.json | 444 +++++++++++++++++++++ __tests__/data/marketo_router_output.json | 201 ++++++++++ v0/destinations/marketo/util.js | 7 +- 7 files changed, 776 insertions(+), 1 deletion(-) diff --git a/__mocks__/data/marketo/proxy_response.json b/__mocks__/data/marketo/proxy_response.json index acd9bd462e..62dab15757 100644 --- a/__mocks__/data/marketo/proxy_response.json +++ b/__mocks__/data/marketo/proxy_response.json @@ -264,5 +264,19 @@ }, "https://mktId.mktorest.com/rest/v1/leads.json/test7": { "code": "[ENOTFOUND] :: DNS lookup failed" + }, + "https://unhandled_exception_in_proxy_req.mktorest.com/rest/v1/leads.json": { + "data": { + "requestId": "142e4#1835b117b76", + "success": false, + "errors": [ + { + "code": "random_marketo_code", + "message": "problem" + } + ] + }, + "status": 200, + "statusText": "OK" } } \ No newline at end of file diff --git a/__mocks__/data/marketo/response.json b/__mocks__/data/marketo/response.json index c357d52fb4..fd407a1aed 100644 --- a/__mocks__/data/marketo/response.json +++ b/__mocks__/data/marketo/response.json @@ -295,5 +295,57 @@ "message": "Lookup field 'userId' not found" } ] + }, + "https://unhandled_status_code.mktorest.com/identity/oauth/token": { + "access_token": "access_token_success", + "expires_in": 3599, + "scope": "integrations@rudderstack.com", + "token_type": "bearer" + }, + "https://unhandled_status_code.mktorest.com/rest/v1/leads.json?filterType=email&filterValues=0c7b8b80-9c43-4f8e-b2d2-5e2448a25040@j.mail": { + "requestId": "12093#17672aeaee6", + "result": [], + "success": true + }, + "https://unhandled_status_code.mktorest.com/rest/v1/leads.json": { + "requestId": "142e4#1835b117b76", + "success": false, + "errors": [ + { + "code": "random_marketo_code", + "message": "some other problem" + } + ] + }, + "https://successful_identify_transformation.mktorest.com/identity/oauth/token": { + "access_token": "access_token_success", + "expires_in": 3599, + "scope": "integrations@rudderstack.com", + "token_type": "bearer" + }, + "https://successful_identify_transformation.mktorest.com/rest/v1/leads.json": { + "requestId": "7ab2#17672a46a99", + "result": [ + { + "id": 4, + "status": "created" + } + ], + "success": true + }, + "https://successful_identify_transformation.mktorest.com/rest/v1/leads.json?filterType=email&filterValues=0c7b8b80-9c43-4f8e-b2d2-5e2448a25040@j.mail": { + "requestId": "12093#17672aeaee6", + "result": [ + { + "createdAt": "2022-09-17T21:39:07Z", + "email": "0c7b8b80-9c43-4f8e-b2d2-5e2448a25040@j.mail", + "firstName": "random_first", + "id": 4, + "lastName": "random_last", + "updatedAt": "2022-09-20T21:48:07Z", + "userId": "test-user-957ue" + } + ], + "success": true } } diff --git a/__tests__/data/marketo_proxy_input.json b/__tests__/data/marketo_proxy_input.json index 35d6a4f829..8962543db3 100644 --- a/__tests__/data/marketo_proxy_input.json +++ b/__tests__/data/marketo_proxy_input.json @@ -250,5 +250,39 @@ "files": {} } } + }, + { + "request": { + "body": { + "version": "1", + "type": "REST", + "method": "POST", + "endpoint": "https://unhandled_exception_in_proxy_req.mktorest.com/rest/v1/leads.json", + "headers": { + "Content-Type": "application/json", + "Authorization": "Bearer access_token_success" + }, + "params": {}, + "body": { + "JSON": { + "action": "createOrUpdate", + "input": [ + { + "Email": "0c7b8b80-9c43-4f8e-b2d2-5e2448a25040@j.mail", + "FirstName": "A", + "LastName": "M", + "id": 4, + "userId": "e17c5a5e-5e2f-430b-b497-fe3f1ea3a704" + } + ], + "lookupField": "id" + }, + "XML": {}, + "JSON_ARRAY": {}, + "FORM": {} + }, + "files": {} + } + } } ] \ No newline at end of file diff --git a/__tests__/data/marketo_proxy_output.json b/__tests__/data/marketo_proxy_output.json index c42af639a7..2101fcd47f 100644 --- a/__tests__/data/marketo_proxy_output.json +++ b/__tests__/data/marketo_proxy_output.json @@ -141,5 +141,30 @@ "errorAt": "proxy" } } + }, + { + "output": { + "status": 400, + "message": "[Error occurred while processing response for destination marketo]: Error occurred during Marketo Response Handling -> problem", + "destinationResponse": { + "response": { + "requestId": "142e4#1835b117b76", + "success": false, + "errors": [ + { + "code": "random_marketo_code", + "message": "problem" + } + ] + }, + "status": 200 + }, + "statTags": { + "destType": "MARKETO", + "stage": "responseTransform", + "scope": "exception", + "errorAt": "proxy" + } + } } ] \ No newline at end of file diff --git a/__tests__/data/marketo_router_input.json b/__tests__/data/marketo_router_input.json index 8e1de5a329..68691708ca 100644 --- a/__tests__/data/marketo_router_input.json +++ b/__tests__/data/marketo_router_input.json @@ -613,5 +613,449 @@ "metadata": { "jobId": 3 } + }, + { + "message": { + "type": "identify", + "sentAt": "2022-09-19T10:34:02.002Z", + "userId": "e17c5a5e-5e2f-430b-b497-fe3f1ea3a704", + "channel": "web", + "context": { + "os": { + "name": "", + "version": "" + }, + "app": { + "name": "RudderLabs JavaScript SDK", + "build": "1.0.0", + "version": "2.12.1", + "namespace": "com.rudderlabs.javascript" + }, + "page": { + "url": "https://accounts.app.t2.broken.com/home", + "path": "/home", + "title": "Home", + "search": "", + "tab_url": "https://accounts.app.t2.broken.com/home", + "referrer": "https://ts50-cvii.core.broken.org/", + "initial_referrer": "https://ts50-cvii.core.broken.org/", + "referring_domain": "ts50-cvii.core.broken.org", + "initial_referring_domain": "ts50-cvii.core.broken.org" + }, + "locale": "en-IN", + "screen": { + "width": 1728, + "height": 1117, + "density": 2, + "innerWidth": 1728, + "innerHeight": 969 + }, + "traits": { + "name": "AM", + "email": "0c7b8b80-9c43-4f8e-b2d2-5e2448a25040@j.mail", + "lastName": "M", + "firstName": "A", + "accountName": "MACDEV", + "billingName": "g g", + "companyName": "macDev", + "currentTier": "Webinar Pro 250", + "billingCycle": "Annually", + "lastBillingDate": "2022-06-29T09:40:42.000Z", + "nextBillingDate": "2023-09-09T04:00:00.000Z", + "subscriptionType": "Webinar Pro 250", + "subscriptionStatus": "ACTIVE", + "lastWebinarEventDate": "2022-09-15T20:00:00.000Z", + "nextWebinarEventDate": "2022-09-16T06:15:00.000Z" + }, + "library": { + "name": "RudderLabs JavaScript SDK", + "version": "2.12.1" + }, + "campaign": {}, + "userAgent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/105.0.0.0 Safari/537.36" + }, + "rudderId": "73dc83ef-587f-4077-90f3-c36083e64019", + "messageId": "1663583642000200-e3f31c4c-6361-4f99-b643-7755303a6007", + "timestamp": "2022-09-19T10:34:02.954Z", + "receivedAt": "2022-09-19T10:34:02.956Z", + "request_ip": "11.105.44.120", + "anonymousId": "0c7b8b80-9c43-4f8e-b2d2-5e2448a25040", + "integrations": { + "All": true + }, + "originalTimestamp": "2022-09-19T10:34:02.000Z" + }, + "destination": { + "Config": { + "accountId": "unhandled_status_code", + "clientId": "504300cd-76b2-a7l4-bhle-90a07420nx73", + "clientSecret": "3l3gJpzRsZagD6gu7tnTeKXz0bomLGnd", + "trackAnonymousEvents": false, + "createIfNotExist": true, + "customActivityEventMap": [ + { + "from": "acq_signup_completed", + "to": "100026" + }, + { + "from": "act_createwebinarform_submit", + "to": "100025" + }, + { + "from": "act_presentation_style", + "to": "100025" + }, + { + "from": "act_webinar_view", + "to": "100025" + }, + { + "from": "act_webinar_join", + "to": "100025" + }, + { + "from": "act_presentation_addteammember", + "to": "100025" + }, + { + "from": "act_engagement_discussions_savediscussion", + "to": "100025" + }, + { + "to": "100025", + "from": "act_engagement_networking_savetime" + } + ] + }, + "destinationDefinition": { + "config": { + "destConfig": { + "defaultConfig": [ + "accountId", + "clientId", + "clientSecret", + "trackAnonymousEvents", + "customActivityEventMap", + "customActivityPropertyMap", + "customActivityPrimaryKeyMap", + "leadTraitMapping" + ] + }, + "secretKeys": [ + "clientSecret" + ], + "excludeKeys": [], + "includeKeys": [], + "routerTransform": true, + "supportedSourceTypes": [ + "android", + "ios", + "web", + "unity", + "amp", + "cloud", + "reactnative" + ] + }, + "responseRules": { + "responseType": "JSON", + "rules": { + "retryable": [ + { + "success": "false", + "errors.0.code": 600 + }, + { + "success": "false", + "errors.0.code": 601 + }, + { + "success": "false", + "errors.0.code": 602 + }, + { + "success": "false", + "errors.0.code": 604 + }, + { + "success": "false", + "errors.0.code": 606 + }, + { + "success": "false", + "errors.0.code": 607 + }, + { + "success": "false", + "errors.0.code": 608 + }, + { + "success": "false", + "errors.0.code": 611 + } + ], + "abortable": [ + { + "success": "false", + "errors.0.code": 603 + }, + { + "success": "false", + "errors.0.code": 605 + }, + { + "success": "false", + "errors.0.code": 609 + }, + { + "success": "false", + "errors.0.code": 610 + } + ] + } + }, + "id": "1aIXqM806xAVm92nx07YwKbRrO9", + "name": "MARKETO", + "displayName": "Marketo", + "createdAt": "2020-04-09T09:24:31.794Z", + "updatedAt": "2021-01-11T11:03:28.103Z" + }, + "secretConfig": {}, + "ID": "1mMy5cqbtfuaKZv1IhVQKnBdVke", + "name": "Marketo", + "enabled": true, + "workspaceId": "1TSN08muJTZwH8iCDmnnRt1pmMd", + "deleted": false, + "createdAt": "2022-02-10T08:39:32.005Z", + "updatedAt": "2022-09-03T16:22:31.374Z", + "transformations": [], + "isConnectionEnabled": true, + "isProcessorEnabled": true + }, + "metadata": { + "jobId": 4 + } + }, + { + "message": { + "type": "identify", + "sentAt": "2022-09-19T10:34:02.002Z", + "userId": "e17c5a5e-5e2f-430b-b497-fe3f1ea3a704", + "channel": "web", + "context": { + "os": { + "name": "", + "version": "" + }, + "app": { + "name": "RudderLabs JavaScript SDK", + "build": "1.0.0", + "version": "2.12.1", + "namespace": "com.rudderlabs.javascript" + }, + "page": { + "url": "https://accounts.app.t2.broken.com/home", + "path": "/home", + "title": "Home", + "search": "", + "tab_url": "https://accounts.app.t2.broken.com/home", + "referrer": "https://ts50-cvii.core.broken.org/", + "initial_referrer": "https://ts50-cvii.core.broken.org/", + "referring_domain": "ts50-cvii.core.broken.org", + "initial_referring_domain": "ts50-cvii.core.broken.org" + }, + "locale": "en-IN", + "screen": { + "width": 1728, + "height": 1117, + "density": 2, + "innerWidth": 1728, + "innerHeight": 969 + }, + "traits": { + "name": "AM", + "email": "0c7b8b80-9c43-4f8e-b2d2-5e2448a25040@j.mail", + "lastName": "M", + "firstName": "A", + "accountName": "MACDEV", + "billingName": "g g", + "companyName": "macDev", + "currentTier": "Webinar Pro 250", + "billingCycle": "Annually", + "lastBillingDate": "2022-06-29T09:40:42.000Z", + "nextBillingDate": "2023-09-09T04:00:00.000Z", + "subscriptionType": "Webinar Pro 250", + "subscriptionStatus": "ACTIVE", + "lastWebinarEventDate": "2022-09-15T20:00:00.000Z", + "nextWebinarEventDate": "2022-09-16T06:15:00.000Z" + }, + "library": { + "name": "RudderLabs JavaScript SDK", + "version": "2.12.1" + }, + "campaign": {}, + "userAgent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/105.0.0.0 Safari/537.36" + }, + "rudderId": "73dc83ef-587f-4077-90f3-c36083e64019", + "messageId": "1663583642000200-e3f31c4c-6361-4f99-b643-7755303a6007", + "timestamp": "2022-09-19T10:34:02.954Z", + "receivedAt": "2022-09-19T10:34:02.956Z", + "request_ip": "11.105.44.120", + "anonymousId": "0c7b8b80-9c43-4f8e-b2d2-5e2448a25040", + "integrations": { + "All": true + }, + "originalTimestamp": "2022-09-19T10:34:02.000Z" + }, + "destination": { + "Config": { + "accountId": "successful_identify_transformation", + "clientId": "504300cd-76b2-a7l4-bhle-90a07420nx73", + "clientSecret": "3l3gJpzRsZagD6gu7tnTeKXz0bomLGnd", + "trackAnonymousEvents": false, + "createIfNotExist": true, + "customActivityEventMap": [ + { + "from": "acq_signup_completed", + "to": "100026" + }, + { + "from": "act_createwebinarform_submit", + "to": "100025" + }, + { + "from": "act_presentation_style", + "to": "100025" + }, + { + "from": "act_webinar_view", + "to": "100025" + }, + { + "from": "act_webinar_join", + "to": "100025" + }, + { + "from": "act_presentation_addteammember", + "to": "100025" + }, + { + "from": "act_engagement_discussions_savediscussion", + "to": "100025" + }, + { + "to": "100025", + "from": "act_engagement_networking_savetime" + } + ] + }, + "destinationDefinition": { + "config": { + "destConfig": { + "defaultConfig": [ + "accountId", + "clientId", + "clientSecret", + "trackAnonymousEvents", + "customActivityEventMap", + "customActivityPropertyMap", + "customActivityPrimaryKeyMap", + "leadTraitMapping" + ] + }, + "secretKeys": [ + "clientSecret" + ], + "excludeKeys": [], + "includeKeys": [], + "routerTransform": true, + "supportedSourceTypes": [ + "android", + "ios", + "web", + "unity", + "amp", + "cloud", + "reactnative" + ] + }, + "responseRules": { + "responseType": "JSON", + "rules": { + "retryable": [ + { + "success": "false", + "errors.0.code": 600 + }, + { + "success": "false", + "errors.0.code": 601 + }, + { + "success": "false", + "errors.0.code": 602 + }, + { + "success": "false", + "errors.0.code": 604 + }, + { + "success": "false", + "errors.0.code": 606 + }, + { + "success": "false", + "errors.0.code": 607 + }, + { + "success": "false", + "errors.0.code": 608 + }, + { + "success": "false", + "errors.0.code": 611 + } + ], + "abortable": [ + { + "success": "false", + "errors.0.code": 603 + }, + { + "success": "false", + "errors.0.code": 605 + }, + { + "success": "false", + "errors.0.code": 609 + }, + { + "success": "false", + "errors.0.code": 610 + } + ] + } + }, + "id": "1aIXqM806xAVm92nx07YwKbRrO9", + "name": "MARKETO", + "displayName": "Marketo", + "createdAt": "2020-04-09T09:24:31.794Z", + "updatedAt": "2021-01-11T11:03:28.103Z" + }, + "secretConfig": {}, + "ID": "1mMy5cqbtfuaKZv1IhVQKnBdVke", + "name": "Marketo", + "enabled": true, + "workspaceId": "1TSN08muJTZwH8iCDmnnRt1pmMd", + "deleted": false, + "createdAt": "2022-02-10T08:39:32.005Z", + "updatedAt": "2022-09-03T16:22:31.374Z", + "transformations": [], + "isConnectionEnabled": true, + "isProcessorEnabled": true + }, + "metadata": { + "jobId": 5 + } } ] \ No newline at end of file diff --git a/__tests__/data/marketo_router_output.json b/__tests__/data/marketo_router_output.json index bab875005c..13d482164e 100644 --- a/__tests__/data/marketo_router_output.json +++ b/__tests__/data/marketo_router_output.json @@ -252,5 +252,206 @@ "jobId": 3 } ] + }, + { + "batched": false, + "statusCode": 400, + "error": "Error occurred [Marketo Transformer]: During lead look up using email -> some other problem", + "statTags": { + "destType": "MARKETO", + "stage": "transform", + "scope": "exception" + }, + "metadata": [ + { + "jobId": 4 + } + ] + }, + { + "batchedRequest": { + "version": "1", + "type": "REST", + "method": "POST", + "endpoint": "https://successful_identify_transformation.mktorest.com/rest/v1/leads.json", + "headers": { + "Content-Type": "application/json", + "Authorization": "Bearer access_token_success" + }, + "params": {}, + "body": { + "JSON": { + "action": "createOrUpdate", + "input": [ + { + "Email": "0c7b8b80-9c43-4f8e-b2d2-5e2448a25040@j.mail", + "FirstName": "A", + "LastName": "M", + "id": 4, + "userId": "e17c5a5e-5e2f-430b-b497-fe3f1ea3a704" + } + ], + "lookupField": "id" + }, + "XML": {}, + "JSON_ARRAY": {}, + "FORM": {} + }, + "files": {} + }, + "metadata": [ + { + "jobId": 5 + } + ], + "batched": false, + "statusCode": 200, + "destination": { + "Config": { + "accountId": "successful_identify_transformation", + "clientId": "504300cd-76b2-a7l4-bhle-90a07420nx73", + "clientSecret": "3l3gJpzRsZagD6gu7tnTeKXz0bomLGnd", + "trackAnonymousEvents": false, + "createIfNotExist": true, + "customActivityEventMap": [ + { + "from": "acq_signup_completed", + "to": "100026" + }, + { + "from": "act_createwebinarform_submit", + "to": "100025" + }, + { + "from": "act_presentation_style", + "to": "100025" + }, + { + "from": "act_webinar_view", + "to": "100025" + }, + { + "from": "act_webinar_join", + "to": "100025" + }, + { + "from": "act_presentation_addteammember", + "to": "100025" + }, + { + "from": "act_engagement_discussions_savediscussion", + "to": "100025" + }, + { + "to": "100025", + "from": "act_engagement_networking_savetime" + } + ] + }, + "destinationDefinition": { + "config": { + "destConfig": { + "defaultConfig": [ + "accountId", + "clientId", + "clientSecret", + "trackAnonymousEvents", + "customActivityEventMap", + "customActivityPropertyMap", + "customActivityPrimaryKeyMap", + "leadTraitMapping" + ] + }, + "secretKeys": [ + "clientSecret" + ], + "excludeKeys": [], + "includeKeys": [], + "routerTransform": true, + "supportedSourceTypes": [ + "android", + "ios", + "web", + "unity", + "amp", + "cloud", + "reactnative" + ] + }, + "responseRules": { + "responseType": "JSON", + "rules": { + "retryable": [ + { + "success": "false", + "errors.0.code": 600 + }, + { + "success": "false", + "errors.0.code": 601 + }, + { + "success": "false", + "errors.0.code": 602 + }, + { + "success": "false", + "errors.0.code": 604 + }, + { + "success": "false", + "errors.0.code": 606 + }, + { + "success": "false", + "errors.0.code": 607 + }, + { + "success": "false", + "errors.0.code": 608 + }, + { + "success": "false", + "errors.0.code": 611 + } + ], + "abortable": [ + { + "success": "false", + "errors.0.code": 603 + }, + { + "success": "false", + "errors.0.code": 605 + }, + { + "success": "false", + "errors.0.code": 609 + }, + { + "success": "false", + "errors.0.code": 610 + } + ] + } + }, + "id": "1aIXqM806xAVm92nx07YwKbRrO9", + "name": "MARKETO", + "displayName": "Marketo", + "createdAt": "2020-04-09T09:24:31.794Z", + "updatedAt": "2021-01-11T11:03:28.103Z" + }, + "secretConfig": {}, + "ID": "1mMy5cqbtfuaKZv1IhVQKnBdVke", + "name": "Marketo", + "enabled": true, + "workspaceId": "1TSN08muJTZwH8iCDmnnRt1pmMd", + "deleted": false, + "createdAt": "2022-02-10T08:39:32.005Z", + "updatedAt": "2022-09-03T16:22:31.374Z", + "transformations": [], + "isConnectionEnabled": true, + "isProcessorEnabled": true + } } ] diff --git a/v0/destinations/marketo/util.js b/v0/destinations/marketo/util.js index 70be09ae5e..402e27da60 100644 --- a/v0/destinations/marketo/util.js +++ b/v0/destinations/marketo/util.js @@ -102,10 +102,15 @@ const marketoResponseHandler = (destResponse, sourceMessage, stage) => { marketoApplicationErrorHandler(destResponse, sourceMessage, stage); } } + // More readable error message + let message = `Error occurred ${sourceMessage}`; + if (response.errors.length > 0 && response.errors[0].message) { + message += ` -> ${response.errors[0].message}`; + } // non 2xx failure throw new ErrorBuilder() .setStatus(400) - .setMessage(`Error occured ${sourceMessage}`) + .setMessage(message) .setDestinationResponse(destResponse) .setStatTags({ destType: DESTINATION, From b216b7d8035d75d78fa5173872463fd3e905add8 Mon Sep 17 00:00:00 2001 From: Sai Sankeerth Date: Thu, 22 Sep 2022 07:04:47 +0530 Subject: [PATCH 7/9] refactor the response handling and change comments --- v0/destinations/marketo/util.js | 36 ++++++++++++++++----------------- 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/v0/destinations/marketo/util.js b/v0/destinations/marketo/util.js index 402e27da60..ba625d9e48 100644 --- a/v0/destinations/marketo/util.js +++ b/v0/destinations/marketo/util.js @@ -88,6 +88,23 @@ const marketoApplicationErrorHandler = ( const marketoResponseHandler = (destResponse, sourceMessage, stage) => { const { status, response } = destResponse; + // if the responsee from destination is not a success case build an explicit error + if (!isHttpStatusSuccess(status)) { + throw new ErrorBuilder() + .setStatus(status) + .setMessage( + `[Marketo Response Handler] - Request failed with status: ${status}` + ) + .setDestinationResponse(destResponse) + .isTransformResponseFailure(true) + .setStatTags({ + destType: DESTINATION, + stage: TRANSFORMER_METRIC.TRANSFORMER_STAGE.RESPONSE_TRANSFORM, + scope: TRANSFORMER_METRIC.MEASUREMENT_TYPE.API.SCOPE, + meta: getDynamicMeta(status) + }) + .build(); + } if (isHttpStatusSuccess(status)) { // for authentication requests if (response && response.access_token) { @@ -107,7 +124,7 @@ const marketoResponseHandler = (destResponse, sourceMessage, stage) => { if (response.errors.length > 0 && response.errors[0].message) { message += ` -> ${response.errors[0].message}`; } - // non 2xx failure + // Marketo sent us some failure which is not handled throw new ErrorBuilder() .setStatus(400) .setMessage(message) @@ -148,23 +165,6 @@ const sendPostRequest = async (url, data, options) => { const responseHandler = (destinationResponse, _dest) => { const message = `[Marketo Response Handler] - Request Processed Successfully`; const { status } = destinationResponse; - // if the responsee from destination is not a success case build an explicit error - if (!isHttpStatusSuccess(status)) { - throw new ErrorBuilder() - .setStatus(status) - .setMessage( - `[Marketo Response Handler] - Request failed with status: ${status}` - ) - .setDestinationResponse(destinationResponse) - .isTransformResponseFailure(true) - .setStatTags({ - destType: DESTINATION, - stage: TRANSFORMER_METRIC.TRANSFORMER_STAGE.RESPONSE_TRANSFORM, - scope: TRANSFORMER_METRIC.MEASUREMENT_TYPE.API.SCOPE, - meta: getDynamicMeta(status) - }) - .build(); - } // check for marketo application level failures marketoResponseHandler( destinationResponse, From 1b29076593d6da9714c33048437010c3a03256a9 Mon Sep 17 00:00:00 2001 From: Sai Sankeerth Date: Thu, 22 Sep 2022 07:44:13 +0530 Subject: [PATCH 8/9] change the unhandled status-code scope to api --- __tests__/data/marketo_proxy_output.json | 3 ++- __tests__/data/marketo_router_output.json | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/__tests__/data/marketo_proxy_output.json b/__tests__/data/marketo_proxy_output.json index 2101fcd47f..23f1ba35fe 100644 --- a/__tests__/data/marketo_proxy_output.json +++ b/__tests__/data/marketo_proxy_output.json @@ -162,7 +162,8 @@ "statTags": { "destType": "MARKETO", "stage": "responseTransform", - "scope": "exception", + "scope": "api", + "meta": "abortable", "errorAt": "proxy" } } diff --git a/__tests__/data/marketo_router_output.json b/__tests__/data/marketo_router_output.json index 13d482164e..ccb2fdcda8 100644 --- a/__tests__/data/marketo_router_output.json +++ b/__tests__/data/marketo_router_output.json @@ -260,7 +260,8 @@ "statTags": { "destType": "MARKETO", "stage": "transform", - "scope": "exception" + "scope": "api", + "meta": "abortable" }, "metadata": [ { From a3728527a11b6ffe0aff7f85fd36b80c6924c6ad Mon Sep 17 00:00:00 2001 From: Sai Sankeerth Date: Thu, 22 Sep 2022 07:48:22 +0530 Subject: [PATCH 9/9] missed code-commit --- v0/destinations/marketo/util.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/v0/destinations/marketo/util.js b/v0/destinations/marketo/util.js index ba625d9e48..19d98dd01f 100644 --- a/v0/destinations/marketo/util.js +++ b/v0/destinations/marketo/util.js @@ -132,7 +132,8 @@ const marketoResponseHandler = (destResponse, sourceMessage, stage) => { .setStatTags({ destType: DESTINATION, stage, - scope: TRANSFORMER_METRIC.MEASUREMENT_TYPE.EXCEPTION.SCOPE + scope: TRANSFORMER_METRIC.MEASUREMENT_TYPE.API.SCOPE, + meta: getDynamicMeta(400) }) .build(); };