From 7e8a2e8c2e25d251a62a1411ae0bb0665a2ece85 Mon Sep 17 00:00:00 2001 From: Kathleen Tuite Date: Tue, 24 Sep 2024 13:40:22 -0700 Subject: [PATCH 1/8] Start of offline entity analytics queries --- lib/model/query/analytics.js | 50 ++++- test/integration/other/analytics-queries.js | 217 ++++++++++++++++++++ 2 files changed, 264 insertions(+), 3 deletions(-) diff --git a/lib/model/query/analytics.js b/lib/model/query/analytics.js index 40a8baba0..3edcb10e5 100644 --- a/lib/model/query/analytics.js +++ b/lib/model/query/analytics.js @@ -394,7 +394,6 @@ group by f."projectId"`); // Datasets -/* eslint-disable no-tabs */ const getDatasets = () => ({ all }) => all(sql` SELECT ds.id, ds."projectId", COUNT(DISTINCT p.id) num_properties, COUNT(DISTINCT e.id) num_entities_total, @@ -462,7 +461,6 @@ FROM datasets ds WHERE ds."publishedAt" IS NOT NULL GROUP BY ds.id, ds."projectId" `); -/* eslint-enable no-tabs */ const getDatasetEvents = () => ({ all }) => all(sql` SELECT @@ -477,6 +475,50 @@ WHERE audits.action = 'entity.bulk.create' GROUP BY ds.id, ds."projectId" `); + +// Offline entities + +// Number of offline branches involving more than one update +// Updates from offline-enabled Collect will include branchId so it is not enough +// to count that but we can look at trunkVersion and branchBaseVersion to find +// versions that had a true (multi-step) offlne operation. +const countOfflineBranches = () => ({ oneFirst }) => oneFirst(sql` +SELECT COUNT(DISTINCT "branchId") +FROM entity_defs +WHERE "branchId" IS NOT NULL AND ("trunkVersion" IS NULL OR "branchBaseVersion" > "trunkVersion") +`); + +// Look up offline branches that have another branchId +// interrupting them, E.g. abc, abc, xyz, abc +const countInterruptedBranches = () => ({ oneFirst }) => oneFirst(sql` +WITH sortedRows AS ( + SELECT + "entityId", + "version", + "branchId", + LAG("branchId") OVER (PARTITION BY "entityId" ORDER BY "version") AS prevBranchId + FROM entity_defs +), +distinctRuns AS ( + SELECT + "entityId", + "branchId" + FROM sortedRows + WHERE "branchId" != prevBranchId OR prevBranchId IS NULL -- Keep first row and changes +), +duplicateRuns AS ( + SELECT + "entityId", + "branchId", + COUNT(*) AS runCount + FROM distinctRuns + GROUP BY "entityId", "branchId" + HAVING COUNT(*) > 1 -- Selects branchIds that occur more than once +) +SELECT COUNT(*) +FROM duplicateRuns; +`); + // Other const getProjectsWithDescriptions = () => ({ all }) => all(sql` select id as "projectId", length(trim(description)) as description_length from projects where coalesce(trim(description),'')!=''`); @@ -750,5 +792,7 @@ module.exports = { projectMetrics, getLatestAudit, getDatasets, - getDatasetEvents + getDatasetEvents, + countOfflineBranches, + countInterruptedBranches }; diff --git a/test/integration/other/analytics-queries.js b/test/integration/other/analytics-queries.js index 1288134e6..4d3b6d961 100644 --- a/test/integration/other/analytics-queries.js +++ b/test/integration/other/analytics-queries.js @@ -2,6 +2,7 @@ const appRoot = require('app-root-path'); const { sql } = require('slonik'); const { testService, testContainer } = require('../setup'); const { createReadStream, readFileSync } = require('fs'); +const uuid = require('uuid').v4; const { promisify } = require('util'); const testData = require('../../data/xml'); @@ -1331,6 +1332,222 @@ describe('analytics task queries', function () { })); }); + describe('offline entity metrics', () => { + it('should count number of offline entity branches (nontrivial branches with >1 update)', testService(async (service, container) => { + await createTestForm(service, container, testData.forms.offlineEntity, 1); + + const asAlice = await service.login('alice'); + + // Entity version from API doesn't have a branchId so doesn't count + await asAlice.post('/v1/projects/1/datasets/people/entities') + .send({ + uuid: '12345678-1234-4123-8234-123456789abc', + label: 'Johnny Doe', + data: { first_name: 'Johnny', age: '22' } + }) + .expect(200); + + // Branch with only update (trunkVersion = baseVersion, doesn't count) + await asAlice.post('/v1/projects/1/forms/offlineEntity/submissions') + .send(testData.instances.offlineEntity.one + .replace('branchId=""', `branchId="${uuid()}"`) + ) + .set('Content-Type', 'application/xml') + .expect(200); + + // Branch with two updates (does count) + const branchIdOne = uuid(); + await asAlice.post('/v1/projects/1/forms/offlineEntity/submissions') + .send(testData.instances.offlineEntity.one + .replace('one', 'one-update1') + .replace('branchId=""', `branchId="${branchIdOne}"`) + ) + .set('Content-Type', 'application/xml') + .expect(200); + + await asAlice.post('/v1/projects/1/forms/offlineEntity/submissions') + .send(testData.instances.offlineEntity.one + .replace('one', 'one-update2') + .replace('baseVersion="1"', 'baseVersion="2"') + .replace('branchId=""', `branchId="${branchIdOne}"`) + ) + .set('Content-Type', 'application/xml') + .expect(200); + + // Branch with create, then update (does count) + await asAlice.post('/v1/projects/1/forms/offlineEntity/submissions') + .send(testData.instances.offlineEntity.two + ) + .set('Content-Type', 'application/xml') + .expect(200); + await asAlice.post('/v1/projects/1/forms/offlineEntity/submissions') + .send(testData.instances.offlineEntity.two + .replace('create="1"', 'update="1"') + .replace('branchId=""', `branchId="${uuid()}"`) + .replace('two', 'two-update') + .replace('baseVersion=""', 'baseVersion="1"') + .replace('new', 'checked in') + ) + .set('Content-Type', 'application/xml') + .expect(200); + + await exhaust(container); + + const countOfflineBranches = await container.Analytics.countOfflineBranches(); + countOfflineBranches.should.equal(2); + + // Sanity check this count, should be 0 + const countInterruptedBranches = await container.Analytics.countInterruptedBranches(); + countInterruptedBranches.should.equal(0); + })); + + it('should count number of interrupted branches', testService(async (service, container) => { + await createTestForm(service, container, testData.forms.offlineEntity, 1); + + const asAlice = await service.login('alice'); + + // make some entities to update + await asAlice.post('/v1/projects/1/datasets/people/entities') + .send({ + entities: [ + { uuid: '12345678-1234-4123-8234-123456789aaa', label: 'aaa' }, + { uuid: '12345678-1234-4123-8234-123456789bbb', label: 'bbb' }, + { uuid: '12345678-1234-4123-8234-123456789ccc', label: 'ccc' }, + ], + source: { name: 'api', size: 3 } + }) + .expect(200); + + // aaa entity, branches: A, A, B, A (counts as interrupted) + const aaaBranchA = uuid(); + const aaaBranchB = uuid(); + await asAlice.post('/v1/projects/1/forms/offlineEntity/submissions') + .send(testData.instances.offlineEntity.one + .replace('id="12345678-1234-4123-8234-123456789abc', 'id="12345678-1234-4123-8234-123456789aaa') + .replace('one', 'aaa-v1') + .replace('branchId=""', `branchId="${aaaBranchA}"`) + ) + .set('Content-Type', 'application/xml') + .expect(200); + + await asAlice.post('/v1/projects/1/forms/offlineEntity/submissions') + .send(testData.instances.offlineEntity.one + .replace('id="12345678-1234-4123-8234-123456789abc', 'id="12345678-1234-4123-8234-123456789aaa') + .replace('one', 'aaa-v2') + .replace('baseVersion="1"', 'baseVersion="2"') + .replace('branchId=""', `branchId="${aaaBranchA}"`) + ) + .set('Content-Type', 'application/xml') + .expect(200); + + await asAlice.post('/v1/projects/1/forms/offlineEntity/submissions') + .send(testData.instances.offlineEntity.one + .replace('id="12345678-1234-4123-8234-123456789abc', 'id="12345678-1234-4123-8234-123456789aaa') + .replace('one', 'aaa-interrupt') + .replace('branchId=""', `branchId="${aaaBranchB}"`) + ) + .set('Content-Type', 'application/xml') + .expect(200); + + await asAlice.post('/v1/projects/1/forms/offlineEntity/submissions') + .send(testData.instances.offlineEntity.one + .replace('id="12345678-1234-4123-8234-123456789abc', 'id="12345678-1234-4123-8234-123456789aaa') + .replace('one', 'aaa-v3') + .replace('baseVersion="1"', 'baseVersion="3"') + .replace('branchId=""', `branchId="${aaaBranchA}"`) + ) + .set('Content-Type', 'application/xml') + .expect(200); + + // bbb entity, branches: A, A, B, B (not counted) + const bbbBranchA = uuid(); + const bbbBranchB = uuid(); + await asAlice.post('/v1/projects/1/forms/offlineEntity/submissions') + .send(testData.instances.offlineEntity.one + .replace('id="12345678-1234-4123-8234-123456789abc', 'id="12345678-1234-4123-8234-123456789bbb') + .replace('one', 'bbb-v1') + .replace('branchId=""', `branchId="${bbbBranchA}"`) + ) + .set('Content-Type', 'application/xml') + .expect(200); + + await asAlice.post('/v1/projects/1/forms/offlineEntity/submissions') + .send(testData.instances.offlineEntity.one + .replace('id="12345678-1234-4123-8234-123456789abc', 'id="12345678-1234-4123-8234-123456789bbb') + .replace('one', 'bbb-v2') + .replace('baseVersion="1"', 'baseVersion="2"') + .replace('branchId=""', `branchId="${bbbBranchA}"`) + ) + .set('Content-Type', 'application/xml') + .expect(200); + + await asAlice.post('/v1/projects/1/forms/offlineEntity/submissions') + .send(testData.instances.offlineEntity.one + .replace('id="12345678-1234-4123-8234-123456789abc', 'id="12345678-1234-4123-8234-123456789bbb') + .replace('one', 'bbb-newbranch-v1') + .replace('branchId=""', `branchId="${bbbBranchB}"`) + ) + .set('Content-Type', 'application/xml') + .expect(200); + + await asAlice.post('/v1/projects/1/forms/offlineEntity/submissions') + .send(testData.instances.offlineEntity.one + .replace('id="12345678-1234-4123-8234-123456789abc', 'id="12345678-1234-4123-8234-123456789bbb') + .replace('one', 'bbb-newbranch-v2') + .replace('baseVersion="1"', 'baseVersion="2"') + .replace('branchId=""', `branchId="${bbbBranchB}"`) + ) + .set('Content-Type', 'application/xml') + .expect(200); + + // ccc entity, branches: A, B, A, B (counted twice) + const cccBranchA = uuid(); + const cccBranchB = uuid(); + await asAlice.post('/v1/projects/1/forms/offlineEntity/submissions') + .send(testData.instances.offlineEntity.one + .replace('id="12345678-1234-4123-8234-123456789abc', 'id="12345678-1234-4123-8234-123456789ccc') + .replace('one', 'ccc-v1') + .replace('branchId=""', `branchId="${cccBranchA}"`) + ) + .set('Content-Type', 'application/xml') + .expect(200); + + await asAlice.post('/v1/projects/1/forms/offlineEntity/submissions') + .send(testData.instances.offlineEntity.one + .replace('id="12345678-1234-4123-8234-123456789abc', 'id="12345678-1234-4123-8234-123456789ccc') + .replace('one', 'ccc-newbranch-v1') + .replace('branchId=""', `branchId="${cccBranchB}"`) + ) + .set('Content-Type', 'application/xml') + .expect(200); + + await asAlice.post('/v1/projects/1/forms/offlineEntity/submissions') + .send(testData.instances.offlineEntity.one + .replace('id="12345678-1234-4123-8234-123456789abc', 'id="12345678-1234-4123-8234-123456789ccc') + .replace('one', 'ccc-v2') + .replace('baseVersion="1"', 'baseVersion="2"') + .replace('branchId=""', `branchId="${cccBranchA}"`) + ) + .set('Content-Type', 'application/xml') + .expect(200); + + await asAlice.post('/v1/projects/1/forms/offlineEntity/submissions') + .send(testData.instances.offlineEntity.one + .replace('id="12345678-1234-4123-8234-123456789abc', 'id="12345678-1234-4123-8234-123456789ccc') + .replace('one', 'ccc-newbranch-v2') + .replace('baseVersion="1"', 'baseVersion="2"') + .replace('branchId=""', `branchId="${cccBranchB}"`) + ) + .set('Content-Type', 'application/xml') + .expect(200); + + await exhaust(container); + + const countInterruptedBranches = await container.Analytics.countInterruptedBranches(); + countInterruptedBranches.should.equal(3); + })); + }); + describe('other project metrics', () => { it('should calculate projects with descriptions', testService(async (service, container) => { await service.login('alice', (asAlice) => From f7af3e21b7cd8299f0441146e2b25d0d5b845044 Mon Sep 17 00:00:00 2001 From: Kathleen Tuite Date: Tue, 24 Sep 2024 14:12:25 -0700 Subject: [PATCH 2/8] Counting submission.reprocess events --- lib/model/query/analytics.js | 11 +- test/integration/other/analytics-queries.js | 113 ++++++++++++++++++++ 2 files changed, 123 insertions(+), 1 deletion(-) diff --git a/lib/model/query/analytics.js b/lib/model/query/analytics.js index 3edcb10e5..f345b6201 100644 --- a/lib/model/query/analytics.js +++ b/lib/model/query/analytics.js @@ -519,6 +519,14 @@ SELECT COUNT(*) FROM duplicateRuns; `); +// Number of submissions temporarily held in backlog but were automatically +// removed from backlog when preceeding submission came in +const countSubmissionReprocess = () => ({ oneFirst }) => oneFirst(sql` + SELECT COUNT(*) + FROM audits + WHERE "action" = 'submission.reprocess' +`); + // Other const getProjectsWithDescriptions = () => ({ all }) => all(sql` select id as "projectId", length(trim(description)) as description_length from projects where coalesce(trim(description),'')!=''`); @@ -794,5 +802,6 @@ module.exports = { getDatasets, getDatasetEvents, countOfflineBranches, - countInterruptedBranches + countInterruptedBranches, + countSubmissionReprocess }; diff --git a/test/integration/other/analytics-queries.js b/test/integration/other/analytics-queries.js index 4d3b6d961..45c9814fb 100644 --- a/test/integration/other/analytics-queries.js +++ b/test/integration/other/analytics-queries.js @@ -1546,6 +1546,119 @@ describe('analytics task queries', function () { const countInterruptedBranches = await container.Analytics.countInterruptedBranches(); countInterruptedBranches.should.equal(3); })); + + it('should count number of submission.reprocess events (submissions temporarily in the backlog)', testService(async (service, container) => { + await createTestForm(service, container, testData.forms.offlineEntity, 1); + + const asAlice = await service.login('alice'); + + // Create entity to update + await asAlice.post('/v1/projects/1/datasets/people/entities') + .send({ + uuid: '12345678-1234-4123-8234-123456789abc', + label: 'label' + }) + .expect(200); + + const branchId = uuid(); + + // Send second update in branch first + await asAlice.post('/v1/projects/1/forms/offlineEntity/submissions') + .send(testData.instances.offlineEntity.one + .replace('one', 'one-update1') + .replace('baseVersion="1"', 'baseVersion="2"') + .replace('branchId=""', `branchId="${branchId}"`) + ) + .set('Content-Type', 'application/xml') + .expect(200); + + await exhaust(container); + + // Should be in backlog + let backlogCount = await container.oneFirst(sql`select count(*) from entity_submission_backlog`); + backlogCount.should.equal(1); + + // Send first update in + await asAlice.post('/v1/projects/1/forms/offlineEntity/submissions') + .send(testData.instances.offlineEntity.one + .replace('branchId=""', `branchId="${branchId}"`) + ) + .set('Content-Type', 'application/xml') + .expect(200); + + await exhaust(container); + + // backlog should be empty now + backlogCount = await container.oneFirst(sql`select count(*) from entity_submission_backlog`); + backlogCount.should.equal(0); + + let countReprocess = await container.Analytics.countSubmissionReprocess(); + countReprocess.should.equal(1); + + // Send a future update that will get held in backlog + await asAlice.post('/v1/projects/1/forms/offlineEntity/submissions') + .send(testData.instances.offlineEntity.one + .replace('one', 'one-update10') + .replace('baseVersion="1"', 'baseVersion="10"') + .replace('branchId=""', `branchId="${branchId}"`) + ) + .set('Content-Type', 'application/xml') + .expect(200); + + await exhaust(container); + + backlogCount = await container.oneFirst(sql`select count(*) from entity_submission_backlog`); + backlogCount.should.equal(1); + + // A submission being put in the backlog is not what is counted so this is still 1 + countReprocess = await container.Analytics.countSubmissionReprocess(); + countReprocess.should.equal(1); + + // force processing the backlog + await container.Entities.processBacklog(true); + + backlogCount = await container.oneFirst(sql`select count(*) from entity_submission_backlog`); + backlogCount.should.equal(0); + + // Force processing also doesn't change this count so it is still 1 + countReprocess = await container.Analytics.countSubmissionReprocess(); + countReprocess.should.equal(1); + + //---------- + + // Trigger another reprocess by sending an update before a create + const branchId2 = uuid(); + + // Send the second submission that updates an entity (before the entity has been created) + await asAlice.post('/v1/projects/1/forms/offlineEntity/submissions') + .send(testData.instances.offlineEntity.two + .replace('create="1"', 'update="1"') + .replace('branchId=""', `branchId="${branchId2}"`) + .replace('two', 'two-update') + .replace('baseVersion=""', 'baseVersion="1"') + .replace('new', 'checked in') + ) + .set('Content-Type', 'application/xml') + .expect(200); + + await exhaust(container); + + backlogCount = await container.oneFirst(sql`select count(*) from entity_submission_backlog`); + backlogCount.should.equal(1); + + // Send the second submission to create the entity, which should trigger + // the processing of the next submission in the branch. + await asAlice.post('/v1/projects/1/forms/offlineEntity/submissions') + .send(testData.instances.offlineEntity.two) + .set('Content-Type', 'application/xml') + .expect(200); + + await exhaust(container); + + // Two reprocessing events logged now + countReprocess = await container.Analytics.countSubmissionReprocess(); + countReprocess.should.equal(2); + })); }); describe('other project metrics', () => { From df96ea4fb0500d423256115fae23711169eafe70 Mon Sep 17 00:00:00 2001 From: Kathleen Tuite Date: Tue, 24 Sep 2024 15:13:37 -0700 Subject: [PATCH 3/8] Submission/entity wait time --- lib/model/query/analytics.js | 17 ++++++++- test/integration/other/analytics-queries.js | 38 +++++++++++++++++++++ 2 files changed, 54 insertions(+), 1 deletion(-) diff --git a/lib/model/query/analytics.js b/lib/model/query/analytics.js index f345b6201..306c8430b 100644 --- a/lib/model/query/analytics.js +++ b/lib/model/query/analytics.js @@ -527,6 +527,20 @@ const countSubmissionReprocess = () => ({ oneFirst }) => oneFirst(sql` WHERE "action" = 'submission.reprocess' `); +// Measure how much time entities whose source is a submission.create +// event take to process to look for a processing lag. We look at the +// submission create loggedAt timestamp (when sub was created) to when +// the event was processed, which will be after the entity version was +// created. +const measureEntityProcessingTime = () => ({ one }) => one(sql` +SELECT + MAX("processed"-"loggedAt") as max_wait, + AVG("processed"-"loggedAt") as avg_wait +FROM entity_def_sources +JOIN audits ON audits.id = "auditId" +WHERE action = 'submission.create' +`); + // Other const getProjectsWithDescriptions = () => ({ all }) => all(sql` select id as "projectId", length(trim(description)) as description_length from projects where coalesce(trim(description),'')!=''`); @@ -803,5 +817,6 @@ module.exports = { getDatasetEvents, countOfflineBranches, countInterruptedBranches, - countSubmissionReprocess + countSubmissionReprocess, + measureEntityProcessingTime }; diff --git a/test/integration/other/analytics-queries.js b/test/integration/other/analytics-queries.js index 45c9814fb..054cd6559 100644 --- a/test/integration/other/analytics-queries.js +++ b/test/integration/other/analytics-queries.js @@ -1659,6 +1659,44 @@ describe('analytics task queries', function () { countReprocess = await container.Analytics.countSubmissionReprocess(); countReprocess.should.equal(2); })); + + it('should measure time from submission creation to entity version finished processing', testService(async (service, container) => { + await createTestForm(service, container, testData.forms.offlineEntity, 1); + + const asAlice = await service.login('alice'); + + // Make an entity that doesn't have a source + await asAlice.post('/v1/projects/1/datasets/people/entities') + .send({ + uuid: '12345678-1234-4123-8234-123456789abc', + label: 'Johnny Doe', + data: { first_name: 'Johnny', age: '22' } + }) + .expect(200); + + // times may be null if no submissions have been processed + let waitTime = await container.Analytics.measureEntityProcessingTime(); + waitTime.should.eql({ max_wait: null, avg_wait: null }); + + // Make an entity update + await asAlice.post('/v1/projects/1/forms/offlineEntity/submissions') + .send(testData.instances.offlineEntity.one + .replace('branchId=""', `branchId="${uuid()}"`) + ) + .set('Content-Type', 'application/xml') + .expect(200); + + // Make another entity create + await asAlice.post('/v1/projects/1/forms/offlineEntity/submissions') + .send(testData.instances.offlineEntity.two) + .set('Content-Type', 'application/xml') + .expect(200); + + await exhaust(container); + waitTime = await container.Analytics.measureEntityProcessingTime(); + waitTime.max_wait.should.be.greaterThan(0); + waitTime.avg_wait.should.be.greaterThan(0); + })); }); describe('other project metrics', () => { From cb543d61762210eb3ac9972b5614eb4a2670a8cf Mon Sep 17 00:00:00 2001 From: Kathleen Tuite Date: Wed, 25 Sep 2024 10:46:13 -0700 Subject: [PATCH 4/8] Updating interrupted branch count query --- lib/model/query/analytics.js | 5 +-- test/integration/other/analytics-queries.js | 36 ++++++++++++++++++++- 2 files changed, 38 insertions(+), 3 deletions(-) diff --git a/lib/model/query/analytics.js b/lib/model/query/analytics.js index 306c8430b..1088ac1df 100644 --- a/lib/model/query/analytics.js +++ b/lib/model/query/analytics.js @@ -496,7 +496,7 @@ WITH sortedRows AS ( "entityId", "version", "branchId", - LAG("branchId") OVER (PARTITION BY "entityId" ORDER BY "version") AS prevBranchId + LAG("branchId") OVER (PARTITION BY "entityId" ORDER BY "version") AS "prevBranchId" FROM entity_defs ), distinctRuns AS ( @@ -504,7 +504,7 @@ distinctRuns AS ( "entityId", "branchId" FROM sortedRows - WHERE "branchId" != prevBranchId OR prevBranchId IS NULL -- Keep first row and changes + WHERE "version" = 1 OR "branchId" IS DISTINCT FROM "prevBranchId" -- Keep first row and changes ), duplicateRuns AS ( SELECT @@ -512,6 +512,7 @@ duplicateRuns AS ( "branchId", COUNT(*) AS runCount FROM distinctRuns + WHERE "branchId" IS NOT NULL GROUP BY "entityId", "branchId" HAVING COUNT(*) > 1 -- Selects branchIds that occur more than once ) diff --git a/test/integration/other/analytics-queries.js b/test/integration/other/analytics-queries.js index 054cd6559..518d8be11 100644 --- a/test/integration/other/analytics-queries.js +++ b/test/integration/other/analytics-queries.js @@ -1413,6 +1413,7 @@ describe('analytics task queries', function () { { uuid: '12345678-1234-4123-8234-123456789aaa', label: 'aaa' }, { uuid: '12345678-1234-4123-8234-123456789bbb', label: 'bbb' }, { uuid: '12345678-1234-4123-8234-123456789ccc', label: 'ccc' }, + { uuid: '12345678-1234-4123-8234-123456789ddd', label: 'ddd' }, ], source: { name: 'api', size: 3 } }) @@ -1543,8 +1544,41 @@ describe('analytics task queries', function () { await exhaust(container); + // ddd entity, branches A, (api update), A + const dddBranchA = uuid(); + await asAlice.post('/v1/projects/1/forms/offlineEntity/submissions') + .send(testData.instances.offlineEntity.one + .replace('id="12345678-1234-4123-8234-123456789abc', 'id="12345678-1234-4123-8234-123456789ddd') + .replace('one', 'ddd-v1') + .replace('branchId=""', `branchId="${dddBranchA}"`) + ) + .set('Content-Type', 'application/xml') + .expect(200); + + await exhaust(container); + + await asAlice.patch('/v1/projects/1/datasets/people/entities/12345678-1234-4123-8234-123456789ddd?baseVersion=2') + .send({ label: 'ddd update' }) + .expect(200); + + await asAlice.patch('/v1/projects/1/datasets/people/entities/12345678-1234-4123-8234-123456789ddd?baseVersion=3') + .send({ label: 'ddd update2' }) + .expect(200); + + await asAlice.post('/v1/projects/1/forms/offlineEntity/submissions') + .send(testData.instances.offlineEntity.one + .replace('id="12345678-1234-4123-8234-123456789abc', 'id="12345678-1234-4123-8234-123456789ddd') + .replace('one', 'ddd-v2') + .replace('baseVersion="1"', 'baseVersion="2"') + .replace('branchId=""', `branchId="${dddBranchA}"`) + ) + .set('Content-Type', 'application/xml') + .expect(200); + + await exhaust(container); + const countInterruptedBranches = await container.Analytics.countInterruptedBranches(); - countInterruptedBranches.should.equal(3); + countInterruptedBranches.should.equal(4); })); it('should count number of submission.reprocess events (submissions temporarily in the backlog)', testService(async (service, container) => { From 2f3b1c918eb8bb3da9719f5628f1a312683676d7 Mon Sep 17 00:00:00 2001 From: Kathleen Tuite Date: Wed, 25 Sep 2024 11:10:51 -0700 Subject: [PATCH 5/8] Tests trying to show submission processing delays --- test/integration/other/analytics-queries.js | 80 +++++++++++++++++++++ 1 file changed, 80 insertions(+) diff --git a/test/integration/other/analytics-queries.js b/test/integration/other/analytics-queries.js index 518d8be11..365f90a27 100644 --- a/test/integration/other/analytics-queries.js +++ b/test/integration/other/analytics-queries.js @@ -1731,6 +1731,86 @@ describe('analytics task queries', function () { waitTime.max_wait.should.be.greaterThan(0); waitTime.avg_wait.should.be.greaterThan(0); })); + + it('should not see a delay for submissions processed with approvalRequired flag toggled', testService(async (service, container) => { + const asAlice = await service.login('alice'); + + // Create form, set dataset people to approvalRequired = true + await createTestForm(service, container, testData.forms.simpleEntity, 1); + await approvalRequired(service, 1, 'people'); + + // Send submission + await submitToForm(service, 'alice', 1, 'simpleEntity', testData.instances.simpleEntity.one); + + // Process submission + await exhaust(container); + + const processTimes1 = await container.one(sql`select "claimed", "processed", "loggedAt" from audits where action = 'submission.create'`); + + // Wait times shouldn't be set yet because submission hasn't been processed into an entity + let waitTime = await container.Analytics.measureEntityProcessingTime(); + waitTime.should.eql({ max_wait: null, avg_wait: null }); + + // Toggle approvalRequired flag + await asAlice.patch('/v1/projects/1/datasets/people?convert=true') + .send({ approvalRequired: false }) + .expect(200); + + await exhaust(container); + + // Wait times are now measured but they are from the initial processing of the submission.create event + // (when it was skipped because the dataset approval was required) + waitTime = await container.Analytics.measureEntityProcessingTime(); + waitTime.max_wait.should.be.greaterThan(0); + waitTime.avg_wait.should.be.greaterThan(0); + + const processTimes2 = await container.one(sql`select "claimed", "processed", "loggedAt" from audits where action = 'submission.create'`); + processTimes1.should.eql(processTimes2); + })); + + it('should not see a delay for submissions held in backlog and then force-processed', testService(async (service, container) => { + const asAlice = await service.login('alice'); + await createTestForm(service, container, testData.forms.offlineEntity, 1); + + // Send an update without the preceeding create + await asAlice.post('/v1/projects/1/forms/offlineEntity/submissions') + .send(testData.instances.offlineEntity.two + .replace('create="1"', 'update="1"') + .replace('branchId=""', `branchId="${uuid()}"`) + .replace('two', 'two-update') + .replace('baseVersion=""', 'baseVersion="1"') + .replace('new', 'checked in') + ) + .set('Content-Type', 'application/xml') + .expect(200); + + await exhaust(container); + + let backlogCount = await container.oneFirst(sql`select count(*) from entity_submission_backlog`); + backlogCount.should.equal(1); + + // The submission.create event does have timestamps on it that we will compare later + const processTimes1 = await container.one(sql`select "claimed", "processed", "loggedAt" from audits where action = 'submission.create'`); + + // Wait times shouldn't be set yet because submission hasn't been processed into an entity + let waitTime = await container.Analytics.measureEntityProcessingTime(); + waitTime.should.eql({ max_wait: null, avg_wait: null }); + + // force processing the backlog + await container.Entities.processBacklog(true); + + backlogCount = await container.oneFirst(sql`select count(*) from entity_submission_backlog`); + backlogCount.should.equal(0); + + // Wait times are now measured but they are from the initial processing of the submission.create event + // (when it was skipped because it went into backlog) + waitTime = await container.Analytics.measureEntityProcessingTime(); + waitTime.max_wait.should.be.greaterThan(0); + waitTime.avg_wait.should.be.greaterThan(0); + + const processTimes2 = await container.one(sql`select "claimed", "processed", "loggedAt" from audits where action = 'submission.create'`); + processTimes1.should.eql(processTimes2); + })); }); describe('other project metrics', () => { From fa25adaf6d9dfdb959b592f2d0527d3d83842551 Mon Sep 17 00:00:00 2001 From: Kathleen Tuite Date: Wed, 25 Sep 2024 11:49:48 -0700 Subject: [PATCH 6/8] New overall entity time query --- lib/model/query/analytics.js | 14 +++++++++++++- test/integration/other/analytics-queries.js | 20 ++++++++++++++++++++ 2 files changed, 33 insertions(+), 1 deletion(-) diff --git a/lib/model/query/analytics.js b/lib/model/query/analytics.js index 1088ac1df..6d10f69af 100644 --- a/lib/model/query/analytics.js +++ b/lib/model/query/analytics.js @@ -542,6 +542,17 @@ JOIN audits ON audits.id = "auditId" WHERE action = 'submission.create' `); +const measureElapsedEntityTime = () => ({ one }) => one(sql` +SELECT + MAX(ed."createdAt" - sd."createdAt") as max_wait, + AVG(ed."createdAt" - sd."createdAt") as avg_wait +FROM entity_defs as ed +JOIN entity_def_sources as eds + ON ed."sourceId" = eds."id" +JOIN submission_defs as sd + ON eds."submissionDefId" = sd.id; +`); + // Other const getProjectsWithDescriptions = () => ({ all }) => all(sql` select id as "projectId", length(trim(description)) as description_length from projects where coalesce(trim(description),'')!=''`); @@ -819,5 +830,6 @@ module.exports = { countOfflineBranches, countInterruptedBranches, countSubmissionReprocess, - measureEntityProcessingTime + measureEntityProcessingTime, + measureElapsedEntityTime }; diff --git a/test/integration/other/analytics-queries.js b/test/integration/other/analytics-queries.js index 365f90a27..bd7fc5368 100644 --- a/test/integration/other/analytics-queries.js +++ b/test/integration/other/analytics-queries.js @@ -1751,6 +1751,10 @@ describe('analytics task queries', function () { let waitTime = await container.Analytics.measureEntityProcessingTime(); waitTime.should.eql({ max_wait: null, avg_wait: null }); + // wait 100ms + // eslint-disable-next-line no-promise-executor-return + await new Promise(resolve => setTimeout(resolve, 100)); + // Toggle approvalRequired flag await asAlice.patch('/v1/projects/1/datasets/people?convert=true') .send({ approvalRequired: false }) @@ -1766,6 +1770,12 @@ describe('analytics task queries', function () { const processTimes2 = await container.one(sql`select "claimed", "processed", "loggedAt" from audits where action = 'submission.create'`); processTimes1.should.eql(processTimes2); + + // Overall time from submission creation to entity version creation should be higher + // because it includes delay from dataset approval flag toggle + const entityTime = await container.Analytics.measureElapsedEntityTime(); + entityTime.max_wait.should.be.greaterThan(waitTime.max_wait + 0.1); + entityTime.avg_wait.should.be.greaterThan(waitTime.avg_wait + 0.1); })); it('should not see a delay for submissions held in backlog and then force-processed', testService(async (service, container) => { @@ -1796,6 +1806,10 @@ describe('analytics task queries', function () { let waitTime = await container.Analytics.measureEntityProcessingTime(); waitTime.should.eql({ max_wait: null, avg_wait: null }); + // wait 100ms + // eslint-disable-next-line no-promise-executor-return + await new Promise(resolve => setTimeout(resolve, 100)); + // force processing the backlog await container.Entities.processBacklog(true); @@ -1810,6 +1824,12 @@ describe('analytics task queries', function () { const processTimes2 = await container.one(sql`select "claimed", "processed", "loggedAt" from audits where action = 'submission.create'`); processTimes1.should.eql(processTimes2); + + // Overall time for entity creation should be higher because it includes the delay + // from waiting in the backlog before being force-processed + const entityTime = await container.Analytics.measureElapsedEntityTime(); + entityTime.max_wait.should.be.greaterThan(waitTime.max_wait + 0.1); + entityTime.avg_wait.should.be.greaterThan(waitTime.avg_wait + 0.1); })); }); From 0215b7b885a8492c2d80aea1019fcb55cb656fca Mon Sep 17 00:00:00 2001 From: Kathleen Tuite Date: Wed, 25 Sep 2024 13:46:50 -0700 Subject: [PATCH 7/8] Add new values to output and update form version --- config/default.json | 2 +- lib/data/analytics.js | 7 ++- lib/model/query/analytics.js | 12 ++++ test/integration/other/analytics-queries.js | 64 ++++++++++++++++++++- 4 files changed, 82 insertions(+), 3 deletions(-) diff --git a/config/default.json b/config/default.json index 0dea82c44..19f31cfca 100644 --- a/config/default.json +++ b/config/default.json @@ -30,7 +30,7 @@ "analytics": { "url": "https://data.getodk.cloud/v1/key/eOZ7S4bzyUW!g1PF6dIXsnSqktRuewzLTpmc6ipBtRq$LDfIMTUKswCexvE0UwJ9/projects/1/forms/odk-analytics/submissions", "formId": "odk-analytics", - "version": "v2024.1.0_1" + "version": "v2024.2.0_1" }, "s3blobStore": {} } diff --git a/lib/data/analytics.js b/lib/data/analytics.js index 2554fa6e0..83f9cdbf9 100644 --- a/lib/data/analytics.js +++ b/lib/data/analytics.js @@ -37,7 +37,12 @@ const metricsTemplate = { "num_client_audit_rows": 0, "num_audits_failed": 0, "num_audits_failed5": 0, - "num_audits_unprocessed": 0 + "num_audits_unprocessed": 0, + "num_offline_entity_branches": 0, + "num_offline_entity_interrupted_branches": 0, + "num_offline_entity_submissions_reprocessed": 0, + "max_entity_submission_delay": 0, + "avg_entity_submission_delay": 0 }, "projects": [ { diff --git a/lib/model/query/analytics.js b/lib/model/query/analytics.js index 6d10f69af..27ecc53b4 100644 --- a/lib/model/query/analytics.js +++ b/lib/model/query/analytics.js @@ -738,10 +738,15 @@ const previewMetrics = () => (({ Analytics }) => Promise.all([ Analytics.countClientAuditAttachments(), Analytics.countClientAuditProcessingFailed(), Analytics.countClientAuditRows(), + Analytics.countOfflineBranches(), + Analytics.countInterruptedBranches(), + Analytics.countSubmissionReprocess(), + Analytics.measureEntityProcessingTime(), Analytics.projectMetrics() ]).then(([db, encrypt, bigForm, admins, audits, archived, managers, viewers, collectors, caAttachments, caFailures, caRows, + oeBranches, oeInterruptedBranches, oeSubReprocess, oeProcessingTime, projMetrics]) => { const metrics = clone(metricsTemplate); // system @@ -776,6 +781,13 @@ const previewMetrics = () => (({ Analytics }) => Promise.all([ metrics.system.num_audits_unprocessed = audits.unprocessed; metrics.system.sso_enabled = oidc.isEnabled() ? 1 : 0; + // 2024.2.0 offline entity metrics + metrics.system.num_offline_entity_branches = oeBranches; + metrics.system.num_offline_entity_interrupted_branches = oeInterruptedBranches; + metrics.system.num_offline_entity_submissions_reprocessed = oeSubReprocess; + metrics.system.max_entity_submission_delay = oeProcessingTime.max_wait; + metrics.system.avg_entity_submission_delay = oeProcessingTime.avg_wait; + return metrics; })); diff --git a/test/integration/other/analytics-queries.js b/test/integration/other/analytics-queries.js index bd7fc5368..8786fb6da 100644 --- a/test/integration/other/analytics-queries.js +++ b/test/integration/other/analytics-queries.js @@ -1890,6 +1890,69 @@ describe('analytics task queries', function () { const event = (await container.Audits.getLatestByAction('submission.attachment.update')).get(); await container.run(sql`update audits set failures = 5 where id = ${event.id}`); + + // 2024.2 offline entity metrics + + // create the form + await asAlice.post('/v1/projects/1/forms?publish=true') + .set('Content-Type', 'application/xml') + .send(testData.forms.offlineEntity) + .expect(200); + + // Creating an update chain + // API create + // update 2, update 1, (out of order reprocessing) + // API update (interrupt branch) + // update 3 + const branchId = uuid(); + + await asAlice.post('/v1/projects/1/datasets/people/entities') + .send({ + uuid: '12345678-1234-4123-8234-123456789abc', + label: 'abc', + }) + .expect(200); + + // switching the order of these updates triggers the + // submission.reprocess count + await asAlice.post('/v1/projects/1/forms/offlineEntity/submissions') + .send(testData.instances.offlineEntity.one + .replace('one', 'one-update2') + .replace('baseVersion="1"', 'baseVersion="2"') + .replace('branchId=""', `branchId="${branchId}"`) + ) + .set('Content-Type', 'application/xml') + .expect(200); + + await asAlice.post('/v1/projects/1/forms/offlineEntity/submissions') + .send(testData.instances.offlineEntity.one + .replace('one', 'one-update1') + .replace('branchId=""', `branchId="${branchId}"`) + ) + .set('Content-Type', 'application/xml') + .expect(200); + + await exhaust(container); + + // inserting an API update before continuing the branch triggers + // the interrupted branch count + await asAlice.patch('/v1/projects/1/datasets/people/entities/12345678-1234-4123-8234-123456789abc?baseVersion=3') + .send({ label: 'abc update' }) + .expect(200); + + await asAlice.post('/v1/projects/1/forms/offlineEntity/submissions') + .send(testData.instances.offlineEntity.one + .replace('one', 'one-update3') + .replace('baseVersion="1"', 'baseVersion="3"') + .replace('branchId=""', `branchId="${branchId}"`) + ) + .set('Content-Type', 'application/xml') + .expect(200); + + await exhaust(container); + + // After the interesting stuff above, encrypt and archive the project + // encrypting a project await asAlice.post('/v1/projects/1/key') .send({ passphrase: 'supersecret', hint: 'it is a secret' }); @@ -1915,7 +1978,6 @@ describe('analytics task queries', function () { (null, 'dummy.action', null, null, '1999-1-1', 5), (null, 'dummy.action', null, null, '1999-1-1', 0)`); - const res = await container.Analytics.previewMetrics(); // can't easily test this metric From 16e33b5e3b6fc353be72f93182e73b7424ba6fab Mon Sep 17 00:00:00 2001 From: Kathleen Tuite Date: Wed, 25 Sep 2024 16:06:06 -0700 Subject: [PATCH 8/8] replace approval required request --- test/integration/other/analytics-queries.js | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/test/integration/other/analytics-queries.js b/test/integration/other/analytics-queries.js index 8786fb6da..c923a4b65 100644 --- a/test/integration/other/analytics-queries.js +++ b/test/integration/other/analytics-queries.js @@ -1737,7 +1737,9 @@ describe('analytics task queries', function () { // Create form, set dataset people to approvalRequired = true await createTestForm(service, container, testData.forms.simpleEntity, 1); - await approvalRequired(service, 1, 'people'); + await asAlice.patch('/v1/projects/1/datasets/people') + .send({ approvalRequired: true }) + .expect(200); // Send submission await submitToForm(service, 'alice', 1, 'simpleEntity', testData.instances.simpleEntity.one);