diff --git a/src/util/prometheus.js b/src/util/prometheus.js index 7750fef153..52b5fb8d37 100644 --- a/src/util/prometheus.js +++ b/src/util/prometheus.js @@ -91,7 +91,9 @@ class Prometheus { try { let metric = this.prometheusRegistry.getSingleMetric(appendPrefix(name)); if (!metric) { - logger.warn(`Prometheus: Summary metric ${name} not found in the registry. Creating a new one`); + logger.warn( + `Prometheus: Summary metric ${name} not found in the registry. Creating a new one`, + ); metric = this.newSummaryStat(name, '', Object.keys(tags)); } metric.observe(tags, value); @@ -104,7 +106,9 @@ class Prometheus { try { let metric = this.prometheusRegistry.getSingleMetric(appendPrefix(name)); if (!metric) { - logger.warn(`Prometheus: Timing metric ${name} not found in the registry. Creating a new one`); + logger.warn( + `Prometheus: Timing metric ${name} not found in the registry. Creating a new one`, + ); metric = this.newHistogramStat(name, '', Object.keys(tags)); } metric.observe(tags, (new Date() - start) / 1000); @@ -117,7 +121,9 @@ class Prometheus { try { let metric = this.prometheusRegistry.getSingleMetric(appendPrefix(name)); if (!metric) { - logger.warn(`Prometheus: Histogram metric ${name} not found in the registry. Creating a new one`); + logger.warn( + `Prometheus: Histogram metric ${name} not found in the registry. Creating a new one`, + ); metric = this.newHistogramStat(name, '', Object.keys(tags)); } metric.observe(tags, value); @@ -134,7 +140,9 @@ class Prometheus { try { let metric = this.prometheusRegistry.getSingleMetric(appendPrefix(name)); if (!metric) { - logger.warn(`Prometheus: Counter metric ${name} not found in the registry. Creating a new one`); + logger.warn( + `Prometheus: Counter metric ${name} not found in the registry. Creating a new one`, + ); metric = this.newCounterStat(name, '', Object.keys(tags)); } metric.inc(tags, delta); @@ -147,8 +155,10 @@ class Prometheus { try { let metric = this.prometheusRegistry.getSingleMetric(appendPrefix(name)); if (!metric) { - logger.warn(`Prometheus: Gauge metric ${name} not found in the registry. Creating a new one`); - metric = this.newGaugeStat(name, '', Object.keys(tags)); + logger.warn( + `Prometheus: Gauge metric ${name} not found in the registry. Creating a new one`, + ); + metric = this.newGaugeStat(name, '', Object.keys(tags)); } metric.set(tags, value); } catch (e) { @@ -758,6 +768,18 @@ class Prometheus { 500, 600, 700, 800, 900, 1000, ], }, + { + name: 'fb_custom_audience_event_having_all_null_field_values_for_a_user', + help: 'fbcustomaudience event having all null field values for a user', + type: 'counter', + labelNames: ['destinationId', 'nullFields'], + }, + { + name: 'fb_custom_audience_event_having_all_null_field_values_for_all_users', + help: 'fbcustomaudience event having all null field values for all users', + type: 'counter', + labelNames: ['destinationId'], + }, { name: 'http_request_size', help: 'http_request_size', diff --git a/src/v0/destinations/fb_custom_audience/transform.js b/src/v0/destinations/fb_custom_audience/transform.js index 38c2064576..e2ea6d8f89 100644 --- a/src/v0/destinations/fb_custom_audience/transform.js +++ b/src/v0/destinations/fb_custom_audience/transform.js @@ -60,6 +60,7 @@ const preparePayload = ( paramsPayload, isHashRequired, disableFormat, + destinationId, ) => { const prepareFinalPayload = _.cloneDeep(paramsPayload); if (Array.isArray(userSchema)) { @@ -73,6 +74,7 @@ const preparePayload = ( userUpdateList, isHashRequired, disableFormat, + destinationId, ); return batchingWithPayloadSize(prepareFinalPayload); }; @@ -125,6 +127,7 @@ const prepareResponse = ( paramsPayload, isHashRequired, disableFormat, + destination.ID, ); // paramsPayload.schema = userSchema; const respList = []; diff --git a/src/v0/destinations/fb_custom_audience/util.js b/src/v0/destinations/fb_custom_audience/util.js index a7a71866bb..e8aaf334fc 100644 --- a/src/v0/destinations/fb_custom_audience/util.js +++ b/src/v0/destinations/fb_custom_audience/util.js @@ -2,6 +2,7 @@ const _ = require('lodash'); const sha256 = require('sha256'); const get = require('get-value'); const jsonSize = require('json-size'); +const stats = require('../../../util/stats'); const { isDefinedAndNotNull } = require('../../util'); const { maxPayloadSize } = require('./config'); @@ -126,43 +127,84 @@ const ensureApplicableFormat = (userProperty, userInformation) => { return updatedProperty; }; +const getUpdatedDataElement = (dataElement, isHashRequired, eachProperty, updatedProperty) => { + let tmpUpdatedProperty = updatedProperty; + /** + * hash the original value for the properties apart from 'MADID' && 'EXTERN_ID as hashing is not required for them + * ref: https://developers.facebook.com/docs/marketing-api/audiences/guides/custom-audiences#hash + * sending null values for the properties for which user hasn't provided any value + */ + if (isHashRequired && eachProperty !== 'MADID' && eachProperty !== 'EXTERN_ID') { + if (tmpUpdatedProperty) { + tmpUpdatedProperty = `${tmpUpdatedProperty}`; + dataElement.push(sha256(tmpUpdatedProperty)); + } else { + dataElement.push(null); + } + } + // if property name is MADID or EXTERN_ID if the value is undefined send null + else if (!tmpUpdatedProperty && (eachProperty === 'MADID' || eachProperty === 'EXTERN_ID')) { + dataElement.push(null); + } else { + dataElement.push(tmpUpdatedProperty); + } + return dataElement; +}; + // Function responsible for making the data field without payload object // Based on the "isHashRequired" value hashing is explicitly enabled or disabled -const prepareDataField = (userSchema, userUpdateList, isHashRequired, disableFormat) => { +const prepareDataField = ( + userSchema, + userUpdateList, + isHashRequired, + disableFormat, + destinationId, +) => { const data = []; - let updatedProperty; - let dataElement; + let nullEvent = true; // flag to check for bad events (all user properties are null) + userUpdateList.forEach((eachUser) => { - dataElement = []; + let dataElement = []; + let nullUserData = true; // flag to check for bad event (all properties are null for a user) + userSchema.forEach((eachProperty) => { const userProperty = eachUser[eachProperty]; - if (isHashRequired) { - if (!disableFormat) { - // when user requires formatting - updatedProperty = ensureApplicableFormat(eachProperty, userProperty); - } else { - // when user requires hashing but does not require formatting - updatedProperty = userProperty; - } - } else { - // when hashing is not required - updatedProperty = userProperty; + let updatedProperty = userProperty; + + if (isHashRequired && !disableFormat) { + updatedProperty = ensureApplicableFormat(eachProperty, userProperty); } - if (isHashRequired && eachProperty !== 'MADID' && eachProperty !== 'EXTERN_ID') { - // for MOBILE_ADVERTISER_ID, MADID,EXTERN_ID hashing is not required ref: https://developers.facebook.com/docs/marketing-api/audiences/guides/custom-audiences#hash - if (updatedProperty) { - updatedProperty = `${updatedProperty}`; - dataElement.push(sha256(updatedProperty)); - } else { - dataElement.push(null); - } - } else { - dataElement.push(updatedProperty); + + dataElement = getUpdatedDataElement( + dataElement, + isHashRequired, + eachProperty, + updatedProperty, + ); + + if (dataElement[dataElement.length - 1]) { + nullUserData = false; + nullEvent = false; } }); + + if (nullUserData) { + stats.increment('fb_custom_audience_event_having_all_null_field_values_for_a_user', { + destinationId, + nullFields: userSchema, + }); + } + data.push(dataElement); }); + if (nullEvent) { + stats.increment('fb_custom_audience_event_having_all_null_field_values_for_all_users', { + destinationId, + }); + } + return data; }; + module.exports = { prepareDataField, getSchemaForEventMappedToDest, batchingWithPayloadSize }; diff --git a/test/__tests__/data/fb_custom_audience.json b/test/__tests__/data/fb_custom_audience.json index 5795bbfc4e..4900ad1a15 100644 --- a/test/__tests__/data/fb_custom_audience.json +++ b/test/__tests__/data/fb_custom_audience.json @@ -1726,6 +1726,115 @@ } ] }, + { + "description": "All the field values are null", + "input": { + "message": { + "userId": "user 1", + "anonymousId": "anon-id-new", + "event": "event1", + "type": "audiencelist", + "properties": { + "listData": { + "add": [ + { + "EMAIL": null, + "DOBM": null, + "DOBD": null, + "DOBY": null, + "PHONE": null, + "GEN": null, + "FI": null, + "MADID": null, + "ZIP": null, + "ST": null, + "COUNTRY": null + } + ] + } + }, + "context": { + "ip": "14.5.67.21", + "library": { + "name": "http" + } + }, + "timestamp": "2020-02-02T00:23:09.544Z" + }, + "destination": { + "Config": { + "accessToken": "ABC", + "userSchema": [ + "EMAIL", + "DOBM", + "DOBD", + "DOBY", + "PHONE", + "GEN", + "FI", + "MADID", + "ZIP", + "ST", + "COUNTRY" + ], + "isHashRequired": true, + "disableFormat": false, + "audienceId": "aud1", + "isRaw": true, + "type": "UNKNOWN", + "subType": "ANYTHING", + "maxUserCount": "50" + }, + "Enabled": true, + "Transformations": [], + "IsProcessorEnabled": true + }, + "libraries": [], + "request": { + "query": {} + } + }, + "output": [ + { + "version": "1", + "type": "REST", + "method": "POST", + "endpoint": "https://graph.facebook.com/v16.0/aud1/users", + "headers": {}, + "params": { + "access_token": "ABC", + "payload": { + "is_raw": true, + "data_source": { + "type": "UNKNOWN", + "sub_type": "ANYTHING" + }, + "schema": [ + "EMAIL", + "DOBM", + "DOBD", + "DOBY", + "PHONE", + "GEN", + "FI", + "MADID", + "ZIP", + "ST", + "COUNTRY" + ], + "data": [[null, null, null, null, null, null, null, null, null, null, null]] + } + }, + "body": { + "JSON": {}, + "JSON_ARRAY": {}, + "XML": {}, + "FORM": {} + }, + "files": {} + } + ] + }, { "description": "Schema Phone Field Missing", "input": {