Submission/entity wait time

lib/model/query/analytics.js

@@ -527,6 +527,20 @@ const countSubmissionReprocess = () => ({ oneFirst }) => oneFirst(sql`
   WHERE "action" = 'submission.reprocess'
 `);
 
+// Measure how much time entities whose source is a submission.create
+// event take to process to look for a processing lag. We look at the
+// submission create loggedAt timestamp (when sub was created) to when
+// the event was processed, which will be after the entity version was
+// created.
+const measureEntityProcessingTime = () => ({ one }) => one(sql`
+SELECT
+  MAX("processed"-"loggedAt") as max_wait,
+  AVG("processed"-"loggedAt") as avg_wait
+FROM entity_def_sources
+JOIN audits ON audits.id = "auditId"
+WHERE action = 'submission.create'
+`);
+
 // Other
 const getProjectsWithDescriptions = () => ({ all }) => all(sql`
 select id as "projectId", length(trim(description)) as description_length from projects where coalesce(trim(description),'')!=''`);
@@ -803,5 +817,6 @@ module.exports = {
   getDatasetEvents,
   countOfflineBranches,
   countInterruptedBranches,
-  countSubmissionReprocess
+  countSubmissionReprocess,
+  measureEntityProcessingTime
 };

test/integration/other/analytics-queries.js

@@ -1671,6 +1671,44 @@ describe('analytics task queries', function () {
       countReprocess = await container.Analytics.countSubmissionReprocess();
       countReprocess.should.equal(2);
     }));
+
+    it('should measure time from submission creation to entity version finished processing', testService(async (service, container) => {
+      await createTestForm(service, container, testData.forms.offlineEntity, 1);
+
+      const asAlice = await service.login('alice');
+
+      // Make an entity that doesn't have a source
+      await asAlice.post('/v1/projects/1/datasets/people/entities')
+        .send({
+          uuid: '12345678-1234-4123-8234-123456789abc',
+          label: 'Johnny Doe',
+          data: { first_name: 'Johnny', age: '22' }
+        })
+        .expect(200);
+
+      // times may be null if no submissions have been processed
+      let waitTime = await container.Analytics.measureEntityProcessingTime();
+      waitTime.should.eql({ max_wait: null, avg_wait: null });
+
+      // Make an entity update
+      await asAlice.post('/v1/projects/1/forms/offlineEntity/submissions')
+        .send(testData.instances.offlineEntity.one
+          .replace('branchId=""', `branchId="${uuid()}"`)
+        )
+        .set('Content-Type', 'application/xml')
+        .expect(200);
+
+      // Make another entity create
+      await asAlice.post('/v1/projects/1/forms/offlineEntity/submissions')
+        .send(testData.instances.offlineEntity.two)
+        .set('Content-Type', 'application/xml')
+        .expect(200);
+
+      await exhaust(container);
+      waitTime = await container.Analytics.measureEntityProcessingTime();
+      waitTime.max_wait.should.be.greaterThan(0);
+      waitTime.avg_wait.should.be.greaterThan(0);
+    }));
   });
 
   describe('other project metrics', () => {