test: Honeycomb system-test reporter

circle.yml

@@ -2088,15 +2088,19 @@ linux-workflow: &linux-workflow
         requires:
           - build
     - system-tests-chrome:
+        context: test-runner:performance-tracking
         requires:
           - system-tests-node-modules-install
     - system-tests-electron:
+        context: test-runner:performance-tracking
         requires:
           - system-tests-node-modules-install
     - system-tests-firefox:
+        context: test-runner:performance-tracking
         requires:
           - system-tests-node-modules-install
     - system-tests-non-root:
+        context: test-runner:performance-tracking
         executor: non-root-docker-user
         requires:
           - system-tests-node-modules-install

packages/server/package.json

@@ -147,6 +147,7 @@
     "@types/chai-as-promised": "7.1.2",
     "@types/chrome": "0.0.101",
     "@types/http-proxy": "1.17.4",
+    "@types/node": "14.14.31",
     "awesome-typescript-loader": "5.2.1",
     "babel-loader": "8.1.0",
     "body-parser": "1.19.0",

system-tests/lib/performance-reporter.js

@@ -0,0 +1,64 @@
+const path = require('path')
+const chalk = require('chalk')
+const Libhoney = require('libhoney')
+
+const pkg = require('@packages/root')
+const ciProvider = require('@packages/server/lib/util/ci_provider')
+const { commitInfo } = require('@cypress/commit-info')
+
+class StatsdReporter {
+  constructor (runner) {
+    if (!process.env.HONEYCOMB_API_KEY) {
+      return
+    }
+
+    console.log(chalk.green('Reporting to honeycomb'))
+    this.honey = new Libhoney({
+      dataset: 'systemtest-performance',
+      writeKey: process.env.HONEYCOMB_API_KEY,
+    })
+
+    runner.on('test', (test) => {
+      test.wallclockStart = Date.now()
+    })
+
+    runner.on('test end', (test, done) => {
+      // Skipped tests never get a 'start' event, but they still get 'test end' somehow.
+      if (test.state === 'skipped') {
+        return
+      }
+
+      commitInfo().then((commitInformation) => {
+        const ciInformation = ciProvider.commitParams() || {}
+        const [, testTitle, browser] = test.title.match(/(.+?)(?: \[([a-z]+)\])?$/)
+
+        const honeycombEvent = this.honey.newEvent()
+
+        honeycombEvent.timestamp = test.wallclockStart
+        honeycombEvent.add({
+          test: testTitle,
+          specFile: path.basename(test.file),
+          browser,
+          state: test.state,
+          durationMs: Date.now() - test.wallclockStart,
+          mochaDurationMs: test.duration,
+          branch: commitInformation.branch || ciInformation.branch,
+          commitSha: commitInformation.sha || ciInformation.sha,
+          buildUrl: process.env.CIRCLE_BUILD_URL,
+          platform: process.platform,
+          arch: process.arch,
+          version: pkg.version,
+        })
+
+        honeycombEvent.send()
+      })
+    })
+  }
+
+  // If there is no done callback, then mocha-multi-reporter will kill the process without waiting for our honeycomb post to complete.
+  done (failures, callback) {
+    this.honey.flush().then(callback)
+  }
+}
+
+module.exports = StatsdReporter

system-tests/lib/spec_helper.js

@@ -103,6 +103,7 @@ beforeEach(function () {
 
   nock.disableNetConnect()
   nock.enableNetConnect(/localhost/)
+  nock.enableNetConnect(/api.honeycomb.io/)
 
   // always clean up the cache
   // before each test

system-tests/package.json

@@ -55,6 +55,7 @@
     "human-interval": "1.0.0",
     "image-size": "0.8.3",
     "lazy-ass": "1.6.0",
+    "libhoney": "3.0.0",
     "lodash": "^4.17.21",
     "mocha": "7.1.0",
     "mocha-banner": "1.1.2",

system-tests/scripts/mocha-reporter-config.json

@@ -0,0 +1,6 @@
+{
+  "reporterEnabled": "spec, mocha-junit-reporter, ../../system-tests/lib/performance-reporter.js",
+  "mochaJunitReporterReporterOptions": {
+    "mochaFile": "/tmp/cypress/junit/test-results.[hash].xml"
+  }
+}

system-tests/scripts/run.js

@@ -109,7 +109,7 @@ commandAndArguments.args.push(
   '--reporter',
   'mocha-multi-reporters',
   '--reporter-options',
-  'configFile=../../mocha-reporter-config.json',
+  'configFile=../../system-tests/scripts/mocha-reporter-config.json',
   '--extension=js,ts',
   // restore mocha 2.x behavior to force end process after spec run
   '--exit',