feat: add stats for bad events in fb_custom_audience

src/util/prometheus.js

@@ -91,7 +91,9 @@ class Prometheus {
     try {
       let metric = this.prometheusRegistry.getSingleMetric(appendPrefix(name));
       if (!metric) {
-        logger.warn(`Prometheus: Summary metric ${name} not found in the registry. Creating a new one`);
+        logger.warn(
+          `Prometheus: Summary metric ${name} not found in the registry. Creating a new one`,
+        );
         metric = this.newSummaryStat(name, '', Object.keys(tags));
       }
       metric.observe(tags, value);
@@ -104,7 +106,9 @@ class Prometheus {
     try {
       let metric = this.prometheusRegistry.getSingleMetric(appendPrefix(name));
       if (!metric) {
-        logger.warn(`Prometheus: Timing metric ${name} not found in the registry. Creating a new one`);
+        logger.warn(
+          `Prometheus: Timing metric ${name} not found in the registry. Creating a new one`,
+        );
         metric = this.newHistogramStat(name, '', Object.keys(tags));
       }
       metric.observe(tags, (new Date() - start) / 1000);
@@ -117,7 +121,9 @@ class Prometheus {
     try {
       let metric = this.prometheusRegistry.getSingleMetric(appendPrefix(name));
       if (!metric) {
-        logger.warn(`Prometheus: Histogram metric ${name} not found in the registry. Creating a new one`);
+        logger.warn(
+          `Prometheus: Histogram metric ${name} not found in the registry. Creating a new one`,
+        );
         metric = this.newHistogramStat(name, '', Object.keys(tags));
       }
       metric.observe(tags, value);
@@ -134,7 +140,9 @@ class Prometheus {
     try {
       let metric = this.prometheusRegistry.getSingleMetric(appendPrefix(name));
       if (!metric) {
-        logger.warn(`Prometheus: Counter metric ${name} not found in the registry. Creating a new one`);
+        logger.warn(
+          `Prometheus: Counter metric ${name} not found in the registry. Creating a new one`,
+        );
         metric = this.newCounterStat(name, '', Object.keys(tags));
       }
       metric.inc(tags, delta);
@@ -147,8 +155,10 @@ class Prometheus {
     try {
       let metric = this.prometheusRegistry.getSingleMetric(appendPrefix(name));
       if (!metric) {
-        logger.warn(`Prometheus: Gauge metric ${name} not found in the registry. Creating a new one`);
-        metric =  this.newGaugeStat(name, '', Object.keys(tags));
+        logger.warn(
+          `Prometheus: Gauge metric ${name} not found in the registry. Creating a new one`,
+        );
+        metric = this.newGaugeStat(name, '', Object.keys(tags));
       }
       metric.set(tags, value);
     } catch (e) {
@@ -758,6 +768,18 @@ class Prometheus {
           500, 600, 700, 800, 900, 1000,
         ],
       },
+      {
+        name: 'fb_custom_audience_event_having_all_null_field_values_for_a_user',
+        help: 'fbcustomaudience event having all null field values for a user',
+        type: 'counter',
+        labelNames: ['destinationId', 'nullFields'],
+      },
+      {
+        name: 'fb_custom_audience_event_having_all_null_field_values_for_all_users',
+        help: 'fbcustomaudience event having all null field values for all users',
+        type: 'counter',
+        labelNames: ['destinationId'],
+      },
       {
         name: 'http_request_size',
         help: 'http_request_size',

src/v0/destinations/fb_custom_audience/transform.js

@@ -60,6 +60,7 @@ const preparePayload = (
   paramsPayload,
   isHashRequired,
   disableFormat,
+  destinationId,
 ) => {
   const prepareFinalPayload = _.cloneDeep(paramsPayload);
   if (Array.isArray(userSchema)) {
@@ -73,6 +74,7 @@ const preparePayload = (
     userUpdateList,
     isHashRequired,
     disableFormat,
+    destinationId,
   );
   return batchingWithPayloadSize(prepareFinalPayload);
 };
@@ -125,6 +127,7 @@ const prepareResponse = (
     paramsPayload,
     isHashRequired,
     disableFormat,
+    destination.ID,
   );
   // paramsPayload.schema = userSchema;
   const respList = [];

src/v0/destinations/fb_custom_audience/util.js

@@ -2,6 +2,7 @@ const _ = require('lodash');
 const sha256 = require('sha256');
 const get = require('get-value');
 const jsonSize = require('json-size');
+const stats = require('../../../util/stats');
 
 const { isDefinedAndNotNull } = require('../../util');
 const { maxPayloadSize } = require('./config');
@@ -126,43 +127,84 @@ const ensureApplicableFormat = (userProperty, userInformation) => {
   return updatedProperty;
 };
 
+const getUpdatedDataElement = (dataElement, isHashRequired, eachProperty, updatedProperty) => {
+  let tmpUpdatedProperty = updatedProperty;
+  /**
+   * hash the original value for the properties apart from 'MADID' && 'EXTERN_ID as hashing is not required for them
+   * ref: https://developers.facebook.com/docs/marketing-api/audiences/guides/custom-audiences#hash
+   * sending null values for the properties for which user hasn't provided any value
+   */
+  if (isHashRequired && eachProperty !== 'MADID' && eachProperty !== 'EXTERN_ID') {
+    if (tmpUpdatedProperty) {
+      tmpUpdatedProperty = `${tmpUpdatedProperty}`;
+      dataElement.push(sha256(tmpUpdatedProperty));
+    } else {
+      dataElement.push(null);
+    }
+  }
+  // if property name is MADID or EXTERN_ID if the value is undefined send null
+  else if (!tmpUpdatedProperty && (eachProperty === 'MADID' || eachProperty === 'EXTERN_ID')) {
+    dataElement.push(null);
+  } else {
+    dataElement.push(tmpUpdatedProperty);
+  }
+  return dataElement;
+};
+
 // Function responsible for making the data field without payload object
 // Based on the "isHashRequired" value hashing is explicitly enabled or disabled
-const prepareDataField = (userSchema, userUpdateList, isHashRequired, disableFormat) => {
+const prepareDataField = (
+  userSchema,
+  userUpdateList,
+  isHashRequired,
+  disableFormat,
+  destinationId,
+) => {
   const data = [];
-  let updatedProperty;
-  let dataElement;
+  let nullEvent = true; // flag to check for bad events (all user properties are null)
+
   userUpdateList.forEach((eachUser) => {
-    dataElement = [];
+    let dataElement = [];
+    let nullUserData = true; // flag to check for bad event (all properties are null for a user)
+
     userSchema.forEach((eachProperty) => {
       const userProperty = eachUser[eachProperty];
-      if (isHashRequired) {
-        if (!disableFormat) {
-          // when user requires formatting
-          updatedProperty = ensureApplicableFormat(eachProperty, userProperty);
-        } else {
-          // when user requires hashing but does not require formatting
-          updatedProperty = userProperty;
-        }
-      } else {
-        // when hashing is not required
-        updatedProperty = userProperty;
+      let updatedProperty = userProperty;
+
+      if (isHashRequired && !disableFormat) {
+        updatedProperty = ensureApplicableFormat(eachProperty, userProperty);
       }
-      if (isHashRequired && eachProperty !== 'MADID' && eachProperty !== 'EXTERN_ID') {
-        // for MOBILE_ADVERTISER_ID, MADID,EXTERN_ID hashing is not required ref: https://developers.facebook.com/docs/marketing-api/audiences/guides/custom-audiences#hash
-        if (updatedProperty) {
-          updatedProperty = `${updatedProperty}`;
-          dataElement.push(sha256(updatedProperty));
-        } else {
-          dataElement.push(null);
-        }
-      } else {
-        dataElement.push(updatedProperty);
+
+      dataElement = getUpdatedDataElement(
+        dataElement,
+        isHashRequired,
+        eachProperty,
+        updatedProperty,
+      );
+
+      if (dataElement[dataElement.length - 1]) {
+        nullUserData = false;
+        nullEvent = false;
       }
     });
+
+    if (nullUserData) {
+      stats.increment('fb_custom_audience_event_having_all_null_field_values_for_a_user', {
+        destinationId,
+        nullFields: userSchema,
+      });
+    }
+
     data.push(dataElement);
   });
 
+  if (nullEvent) {
+    stats.increment('fb_custom_audience_event_having_all_null_field_values_for_all_users', {
+      destinationId,
+    });
+  }
+
   return data;
 };
+
 module.exports = { prepareDataField, getSchemaForEventMappedToDest, batchingWithPayloadSize };

test/__tests__/data/fb_custom_audience.json

@@ -1726,6 +1726,115 @@
       }
     ]
   },
+  {
+    "description": "All the field values are null",
+    "input": {
+      "message": {
+        "userId": "user 1",
+        "anonymousId": "anon-id-new",
+        "event": "event1",
+        "type": "audiencelist",
+        "properties": {
+          "listData": {
+            "add": [
+              {
+                "EMAIL": null,
+                "DOBM": null,
+                "DOBD": null,
+                "DOBY": null,
+                "PHONE": null,
+                "GEN": null,
+                "FI": null,
+                "MADID": null,
+                "ZIP": null,
+                "ST": null,
+                "COUNTRY": null
+              }
+            ]
+          }
+        },
+        "context": {
+          "ip": "14.5.67.21",
+          "library": {
+            "name": "http"
+          }
+        },
+        "timestamp": "2020-02-02T00:23:09.544Z"
+      },
+      "destination": {
+        "Config": {
+          "accessToken": "ABC",
+          "userSchema": [
+            "EMAIL",
+            "DOBM",
+            "DOBD",
+            "DOBY",
+            "PHONE",
+            "GEN",
+            "FI",
+            "MADID",
+            "ZIP",
+            "ST",
+            "COUNTRY"
+          ],
+          "isHashRequired": true,
+          "disableFormat": false,
+          "audienceId": "aud1",
+          "isRaw": true,
+          "type": "UNKNOWN",
+          "subType": "ANYTHING",
+          "maxUserCount": "50"
+        },
+        "Enabled": true,
+        "Transformations": [],
+        "IsProcessorEnabled": true
+      },
+      "libraries": [],
+      "request": {
+        "query": {}
+      }
+    },
+    "output": [
+      {
+        "version": "1",
+        "type": "REST",
+        "method": "POST",
+        "endpoint": "https://graph.facebook.com/v16.0/aud1/users",
+        "headers": {},
+        "params": {
+          "access_token": "ABC",
+          "payload": {
+            "is_raw": true,
+            "data_source": {
+              "type": "UNKNOWN",
+              "sub_type": "ANYTHING"
+            },
+            "schema": [
+              "EMAIL",
+              "DOBM",
+              "DOBD",
+              "DOBY",
+              "PHONE",
+              "GEN",
+              "FI",
+              "MADID",
+              "ZIP",
+              "ST",
+              "COUNTRY"
+            ],
+            "data": [[null, null, null, null, null, null, null, null, null, null, null]]
+          }
+        },
+        "body": {
+          "JSON": {},
+          "JSON_ARRAY": {},
+          "XML": {},
+          "FORM": {}
+        },
+        "files": {}
+      }
+    ]
+  },
   {
     "description": "Schema Phone Field Missing",
     "input": {