Draft Playwright test cases! (#10)

* Save an image in test server public directory

* Rename e2e tests with consistent naming scheme

* server.js: Support POST requests with JSON body

* Define FUDGE factor for assertions

* Define e2e tests: images

* Define e2e test: static

* Define e2e test: cpu

* Define e2e tests: scripts

* Define e2e test: iframe

* Define e2e tests: react

* Remove accidentally copy-pasted comments :sweat-smile:

* Add "Vary: *" header to express responses to convince Safari not to cache

* Refactor: Don't use module type scripts

* Express server can load files from node_modules

* React tests shouldn't depend on internet connection

* Group e2e test spec and html files

* fixup! Define e2e test: static

* Bugfix: waitForLoadingImages cannot report a time before window.onload

* Update tests with new results

* Define e2e test cases: composition

* Define e2e tests: invisible

* Define e2e tests: error

* fixup! df3aaab

* Define e2e tests: interaction

* fixup! Define e2e test cases: composition

* fixup! Define e2e tests: error

* fixup! Bugfix: waitForLoadingImages cannot report a time before window.onload

* Mark two additional known test failures

* Playwright: Set local default to three workers and one retry

* We don't need to upload any playwright artifacts

* Run express service via playwright config!

Co-authored-by: Andrew Hyndman <ahyndman@dropbox.com>

.github/workflows/playwright.yml

@@ -17,11 +17,7 @@ jobs:
       run: yarn
     - name: Install Playwright
       run: npx playwright install --with-deps
+    - name: Build ttvc
+      run: yarn build
     - name: Run Playwright tests
-      run: yarn ci:e2e
-    - uses: actions/upload-artifact@v2
-      if: always()
-      with:
-        name: playwright-report
-        path: playwright-report/
-        retention-days: 30
+      run: yarn test:e2e

package.json

@@ -15,14 +15,14 @@
     "access": "public"
   },
   "scripts": {
+    "clean": "rm -rf dist && rm -rf lib",
     "build": "npm-run-all --parallel build:*",
     "build:tsc": "tsc --project tsconfig.build.json",
     "build:rollup": "rollup --config rollup.config.js",
-    "test": "npm-run-all build test:unit --parallel --race test:server test:e2e",
+    "test": "npm-run-all build test:*",
     "test:unit": "jest",
-    "test:server": "node test/server/server.mjs",
     "test:e2e": "playwright test",
-    "ci:e2e": "npm-run-all build --parallel --race test:server test:e2e",
+    "express": "node test/server/server.mjs",
     "release": "np"
   },
   "engines": {
@@ -36,11 +36,15 @@
     "@types/express": "^4.17.13",
     "@types/jest": "^27.4.1",
     "@types/node": "^17.0.21",
+    "@types/react": "^17.0.43",
+    "@types/react-dom": "^17.0.14",
     "express": "^4.17.3",
     "jest": "^27.5.1",
     "np": "^7.6.1",
     "npm-run-all": "^4.1.5",
     "prettier": "^2.5.1",
+    "react": "^18.0.0",
+    "react-dom": "^18.0.0",
     "rollup": "^2.70.1",
     "rollup-plugin-terser": "^7.0.2",
     "ts-jest": "^27.1.4",

playwright.config.ts

@@ -24,9 +24,9 @@ const config: PlaywrightTestConfig = {
   /* Fail the build on CI if you accidentally left test.only in the source code. */
   forbidOnly: !!process.env.CI,
   /* Retry on CI only */
-  retries: process.env.CI ? 2 : 0,
+  retries: process.env.CI ? 2 : 1,
   /* Opt out of parallel tests on CI. */
-  workers: process.env.CI ? 1 : undefined,
+  workers: process.env.CI ? 1 : 3,
   /* Reporter to use. See https://playwright.dev/docs/test-reporters */
   reporter: 'list',
   /* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */
@@ -96,10 +96,10 @@ const config: PlaywrightTestConfig = {
   // outputDir: 'test-results/',
 
   /* Run your local dev server before starting the tests */
-  // webServer: {
-  //   command: 'npm run start',
-  //   port: 3000,
-  // },
+  webServer: {
+    command: 'yarn express',
+    port: 3000,
+  },
 };
 
 export default config;

src/in_viewport_mutation_observer.ts

@@ -73,6 +73,7 @@ export class InViewportMutationObserver {
   private intersectionsObservedCount = 0;
   private totalMutationCallbackLatencyOverhead = 0;
   private totalIntersectionCallbackLatencyOverhead = 0;
+  private pageHasImages = false;
   public wasDocumentHiddenAtSomePoint = false;
 
   constructor(params: InViewportMutationObserverParams) {
@@ -108,8 +109,8 @@ export class InViewportMutationObserver {
    * This function will wait for all images to load and will return the time when they finished loading
    */
   public waitForLoadingImages = () => {
-    if (loadingImages.size === 0) {
-      return Promise.resolve(performance.now());
+    if (!this.pageHasImages) {
+      return Promise.resolve(0);
     }
     return this.loadingImagesPromise;
   };
@@ -180,6 +181,7 @@ export class InViewportMutationObserver {
   private imageIntersectionObserverCallback = (entries: IntersectionObserverEntry[]) => {
     entries.forEach((entry) => {
       if (entry.isIntersecting) {
+        this.pageHasImages = true;
         loadingImages.add(entry.target as HTMLElement);
       }
     });

test/server/public/ajax-mutation.html → test/e2e/ajax1/index.html

@@ -7,16 +7,13 @@
   <h1 id="h1">Hello</h1>
 
   <script>
-    fetch('/api/hello?delay=500')
+    fetch('/api?delay=500')
       .then((res) => res.text())
       .then((text) => {
         // append a new node
         const h2 = document.createElement('h2');
-        h2.textContent = `Hello world!`;
+        h2.textContent = text;
         document.body.appendChild(h2);
       });
   </script>
-
-  <!-- this script just delays the window "load" event -->
-  <script src="/stub.js?delay=1000"></script>
 </body>

test/e2e/ajax1/index.spec.ts

@@ -0,0 +1,22 @@
+import {test, expect} from '@playwright/test';
+
+import {FUDGE} from '../../util/constants';
+import {getEntries} from '../../util/entries';
+
+const PAGELOAD_DELAY = 200;
+const AJAX_DELAY = 500; // see text-mutation.html
+
+test.describe('TTVC', () => {
+  test('a mutation triggered after an AJAX request', async ({page}) => {
+    test.fail(); // ttvc should wait until all AJAX requests have resolved before measuring
+    await page.goto(`http://localhost:3000/test/ajax1?delay=${PAGELOAD_DELAY}`, {
+      waitUntil: 'networkidle',
+    });
+
+    const entries = await getEntries(page);
+
+    expect(entries.length).toBe(1);
+    expect(entries[0]).toBeGreaterThanOrEqual(PAGELOAD_DELAY + AJAX_DELAY);
+    expect(entries[0]).toBeLessThanOrEqual(PAGELOAD_DELAY + AJAX_DELAY + FUDGE);
+  });
+});

test/server/public/text-mutation.html → test/e2e/ajax2/index.html

@@ -7,12 +7,12 @@
   <h1 id="h1">Hello!</h1>
 
   <script>
-    fetch('/api/hello?delay=500')
+    fetch('/api?delay=500')
       .then((res) => res.text())
       .then(
         (text) =>
           // mutate some text
-          (document.getElementById('h1').textContent = `Hello ${text}!`)
+          (document.getElementById('h1').textContent = text)
       );
   </script>
 </body>

test/e2e/ajax2/index.spec.ts

@@ -0,0 +1,22 @@
+import {test, expect} from '@playwright/test';
+
+import {FUDGE} from '../../util/constants';
+import {getEntries} from '../../util/entries';
+
+const PAGELOAD_DELAY = 200;
+const AJAX_DELAY = 500; // see text-mutation.html
+
+test.describe('TTVC', () => {
+  test('a text-only mutation triggered after AJAX', async ({page}) => {
+    test.fail(); // ttvc should not require mutations to mark a timestamp
+    await page.goto(`http://localhost:3000/test/ajax2?delay=${PAGELOAD_DELAY}`, {
+      waitUntil: 'networkidle',
+    });
+
+    const entries = await getEntries(page);
+
+    expect(entries.length).toBe(1);
+    expect(entries[0]).toBeGreaterThanOrEqual(PAGELOAD_DELAY + AJAX_DELAY);
+    expect(entries[0]).toBeLessThanOrEqual(PAGELOAD_DELAY + AJAX_DELAY + FUDGE);
+  });
+});

test/e2e/ajax3/index.html

@@ -0,0 +1,20 @@
+<head>
+  <script src="/dist/index.min.js"></script>
+  <script src="/analytics.js"></script>
+</head>
+
+<body>
+  <h1 id="h1">Hello</h1>
+
+  <script>
+    fetch('/api?delay=500', {
+      headers: {'content-type': 'application/json'},
+      method: 'POST',
+      body: JSON.stringify({name: 'jack'}),
+    })
+      .then((res) => res.json())
+      .then((json) => {
+        h1.textContent = `Hello ${json.name}!`;
+      });
+  </script>
+</body>

test/e2e/ajax3/index.spec.ts

@@ -0,0 +1,22 @@
+import {test, expect} from '@playwright/test';
+
+import {FUDGE} from '../../util/constants';
+import {getEntries, entryCountIs} from '../../util/entries';
+
+const PAGELOAD_DELAY = 200;
+const AJAX_DELAY = 500; // see text-mutation.html
+
+test.describe('TTVC', () => {
+  test('a mutation triggered after an AJAX request', async ({page}) => {
+    test.fail(); // ttvc should wait until all AJAX requests have resolved before measuring
+    await page.goto(`http://localhost:3000/test/ajax3?delay=${PAGELOAD_DELAY}`, {
+      waitUntil: 'networkidle',
+    });
+
+    const entries = await getEntries(page);
+
+    expect(entries.length).toBe(1);
+    expect(entries[0]).toBeGreaterThanOrEqual(PAGELOAD_DELAY + AJAX_DELAY);
+    expect(entries[0]).toBeLessThanOrEqual(PAGELOAD_DELAY + AJAX_DELAY + FUDGE);
+  });
+});

test/e2e/composition1/index.html

@@ -0,0 +1,18 @@
+<head>
+  <script src="/dist/index.min.js"></script>
+  <script src="/analytics.js"></script>
+  <script async src="/stub.js?delay=2000"></script>
+</head>
+
+<body>
+  <h1 id="h1">Hello world!</h1>
+
+  <script>
+    const asyncScript = document.createElement('script');
+    asyncScript.src = '/mutation.js?delay=500';
+
+    fetch('/api?delay=500')
+      .then((res) => res.text())
+      .then(() => document.head.appendChild(asyncScript));
+  </script>
+</body>

test/e2e/composition1/index.spec.ts

@@ -0,0 +1,23 @@
+import {test, expect} from '@playwright/test';
+
+import {FUDGE} from '../../util/constants';
+import {getEntries} from '../../util/entries';
+
+const PAGELOAD_DELAY = 200;
+const AJAX_DELAY = 500;
+const SCRIPT_DELAY = 500;
+
+test.describe('TTVC', () => {
+  test('ajax request > script load > mutation + background lazyloaded script', async ({page}) => {
+    await page.goto(`http://localhost:3000/test/composition1?delay=${PAGELOAD_DELAY}`, {
+      waitUntil: 'networkidle',
+    });
+
+    const entries = await getEntries(page);
+    expect(entries.length).toBe(1);
+
+    const expectedTtvc = PAGELOAD_DELAY + AJAX_DELAY + SCRIPT_DELAY;
+    expect(entries[0]).toBeGreaterThanOrEqual(expectedTtvc);
+    expect(entries[0]).toBeLessThanOrEqual(expectedTtvc + FUDGE);
+  });
+});

test/e2e/composition2/index.html

@@ -0,0 +1,22 @@
+<head>
+  <script src="/dist/index.min.js"></script>
+  <script src="/analytics.js"></script>
+  <script async src="/stub.js?delay=3000"></script>
+</head>
+
+<body>
+  <h1 id="h1">Hello world!</h1>
+
+  <script src="/busy.js?delay=500"></script>
+  <script>
+    const asyncScript = document.createElement('script');
+    asyncScript.src = '/mutation.js?delay=500';
+
+    fetch('/api?delay=500')
+      .then((res) => res.text())
+      .then(() => {
+        busy(200);
+        document.head.appendChild(asyncScript);
+      });
+  </script>
+</body>

test/e2e/composition2/index.spec.ts

@@ -0,0 +1,26 @@
+import {test, expect} from '@playwright/test';
+
+import {FUDGE} from '../../util/constants';
+import {getEntries} from '../../util/entries';
+
+const PAGELOAD_DELAY = 200;
+const AJAX_DELAY = 500;
+const CPU_DELAY = 200;
+const SCRIPT_DELAY = 500;
+
+test.describe('TTVC', () => {
+  test('script load > ajax request > CPU work > script load > mutation + background lazyloaded script', async ({
+    page,
+  }) => {
+    await page.goto(`http://localhost:3000/test/composition2?delay=${PAGELOAD_DELAY}`, {
+      waitUntil: 'networkidle',
+    });
+
+    const entries = await getEntries(page);
+    expect(entries.length).toBe(1);
+
+    const expectedTtvc = PAGELOAD_DELAY + SCRIPT_DELAY + AJAX_DELAY + CPU_DELAY + SCRIPT_DELAY;
+    expect(entries[0]).toBeGreaterThanOrEqual(expectedTtvc);
+    expect(entries[0]).toBeLessThanOrEqual(expectedTtvc + FUDGE);
+  });
+});

test/e2e/composition3/index.html

@@ -0,0 +1,21 @@
+<head>
+  <script src="/dist/index.min.js"></script>
+  <script src="/analytics.js"></script>
+</head>
+
+<body>
+  <h1 id="h1">Hello world!</h1>
+
+  <script src="/busy.js?delay=500"></script>
+  <script>
+    const asyncScript = document.createElement('script');
+    asyncScript.src = '/mutation.js?delay=500';
+
+    fetch('/api?delay=500')
+      .then((res) => res.text())
+      .then(() => {
+        busy(1000);
+        document.head.appendChild(asyncScript);
+      });
+  </script>
+</body>

test/e2e/composition3/index.spec.ts

@@ -0,0 +1,25 @@
+import {test, expect} from '@playwright/test';
+
+import {FUDGE} from '../../util/constants';
+import {getEntries} from '../../util/entries';
+
+const PAGELOAD_DELAY = 200;
+const AJAX_DELAY = 500;
+const CPU_DELAY = 1000;
+const SCRIPT_DELAY = 500;
+
+test.describe('TTVC', () => {
+  test('script load > ajax request > CPU work > script load > mutation', async ({page}) => {
+    test.fail(); // ttvc is computed too early
+    await page.goto(`http://localhost:3000/test/composition3?delay=${PAGELOAD_DELAY}`, {
+      waitUntil: 'networkidle',
+    });
+
+    const entries = await getEntries(page);
+    expect(entries.length).toBe(1);
+
+    const expectedTtvc = PAGELOAD_DELAY + SCRIPT_DELAY + AJAX_DELAY + CPU_DELAY + SCRIPT_DELAY;
+    expect(entries[0]).toBeGreaterThanOrEqual(expectedTtvc);
+    expect(entries[0]).toBeLessThanOrEqual(expectedTtvc + FUDGE);
+  });
+});