Merge pull request #257 from storybookjs/shard-coverage

fix: nyc command with shard option

README.md

@@ -24,6 +24,7 @@ Storybook test runner turns all of your stories into executable tests.
     - [Manually configuring istanbul](#manually-configuring-istanbul)
   - [2 - Run tests with --coverage flag](#2---run-tests-with---coverage-flag)
   - [3 - Merging code coverage with coverage from other tools](#3---merging-code-coverage-with-coverage-from-other-tools)
+  - [4 - Run tests with --shard flag](#4---run-tests-with---shard-flag)
 - [Experimental test hook API](#experimental-test-hook-api)
   - [prepare](#prepare)
   - [getHttpHeaders](#gethttpheaders)
@@ -416,6 +417,62 @@ Here's an example on how to achieve that:
 > **Note**
 > If your other tests (e.g. Jest) are using a different coverageProvider than `babel`, you will have issues when merging the coverage files. [More info here](#merging-test-coverage-results-in-wrong-coverage).
 
+### 4 - Run tests with --shard flag
+
+The test-runner collects all coverage in one file `coverage/storybook/coverage-storybook.json`. To split the coverage file you should rename it using the `shard-index`. To report the coverage you should merge the coverage files with the nyc merge command.
+
+Github CI example:
+
+```yml
+test:
+  name: Running Test-storybook (${{ matrix.shard }})
+  strategy:
+    matrix:
+      shard: [1, 2, 3, 4]
+  steps:
+    - name: Testing storybook
+      run: yarn test-storybook --coverage --shard=${{ matrix.shard }}/${{ strategy.job-total }}
+    - name: Renaming coverage file
+      uses: mv coverage/storybook/coverage-storybook.json coverage/storybook/coverage-storybook-${matrix.shard}.json
+report-coverage:
+  name: Reporting storybook coverage
+  steps:
+    - name: Merging coverage
+      uses: yarn nyc merge coverage/storybook merged-output/merged-coverage.json
+    - name: Report coverage
+      uses: yarn nyc report --reporter=text -t merged-output --report-dir merged-output
+```
+
+Circle CI example:
+
+```yml
+test:
+  parallelism: 4
+  steps:
+    - run:
+        command: yarn test-storybook --coverage --shard=$(expr $CIRCLE_NODE_INDEX + 1)/$CIRCLE_NODE_TOTAL
+        command: mv coverage/storybook/coverage-storybook.json coverage/storybook/coverage-storybook-${CIRCLE_NODE_INDEX + 1}.json
+report-coverage:
+  steps:
+    - run:
+        command: yarn nyc merge coverage/storybook merged-output/merged-coverage.json
+        command: yarn nyc report --reporter=text -t merged-output --report-dir merged-output
+```
+
+Gitlab CI example:
+
+```yml
+test:
+  parallel: 4
+  script:
+    - yarn test-storybook --coverage --shard=$CI_NODE_INDEX/$CI_NODE_TOTAL
+    - mv coverage/storybook/coverage-storybook.json coverage/storybook/coverage-storybook-${CI_NODE_INDEX}.json
+report-coverage:
+  script:
+    - yarn nyc merge coverage/storybook merged-output/merged-coverage.json
+    - yarn nyc report --reporter=text -t merged-output --report-dir merged-output
+```
+
 ## Experimental test hook API
 
 The test runner renders a story and executes its [play function](https://storybook.js.org/docs/react/writing-stories/play-function) if one exists. However, there are certain behaviors that are not possible to achieve via the play function, which executes in the browser. For example, if you want the test runner to take visual snapshots for you, this is something that is possible via Playwright/Jest, but must be executed in Node.

src/test-storybook.ts

@@ -50,12 +50,7 @@ const cleanup = () => {
   }
 };
 
-let isWatchMode = false;
 async function reportCoverage() {
-  if (isWatchMode || process.env.STORYBOOK_COLLECT_COVERAGE !== 'true') {
-    return;
-  }
-
   const coverageFolderE2E = path.resolve(process.cwd(), '.nyc_output');
   const coverageFolder = path.resolve(process.cwd(), 'coverage/storybook');
 
@@ -76,14 +71,18 @@ async function reportCoverage() {
   // --skip-full in case we only want to show not fully covered code
   // --check-coverage if we want to break if coverage reaches certain threshold
   // .nycrc will be respected for thresholds etc. https://www.npmjs.com/package/nyc#coverage-thresholds
-  execSync(`npx nyc report --reporter=text -t ${coverageFolder} --report-dir ${coverageFolder}`, {
-    stdio: 'inherit',
-  });
+  if (process.env.JEST_SHARD !== 'true') {
+    execSync(`npx nyc report --reporter=text -t ${coverageFolder} --report-dir ${coverageFolder}`, {
+      stdio: 'inherit',
+    });
+  }
 }
 
 const onProcessEnd = () => {
   cleanup();
-  reportCoverage();
+  if (process.env.STORYBOOK_COLLECT_COVERAGE !== 'true') {
+    reportCoverage();
+  }
 };
 
 process.on('SIGINT', onProcessEnd);
@@ -248,7 +247,7 @@ const main = async () => {
   }
 
   // set this flag to skip reporting coverage in watch mode
-  isWatchMode = jestOptions.watch || jestOptions.watchAll;
+  const isWatchMode = jestOptions.includes('--watch') || jestOptions.includes('--watchAll');
 
   const rawTargetURL = process.env.TARGET_URL || runnerOptions.url || 'http://127.0.0.1:6006';
   await checkStorybook(rawTargetURL);
@@ -257,7 +256,7 @@ const main = async () => {
 
   process.env.TARGET_URL = targetURL;
 
-  if (runnerOptions.coverage) {
+  if (!isWatchMode && runnerOptions.coverage) {
     process.env.STORYBOOK_COLLECT_COVERAGE = 'true';
   }
 
@@ -269,6 +268,10 @@ const main = async () => {
     process.env.REFERENCE_URL = sanitizeURL(process.env.REFERENCE_URL);
   }
 
+  if (jestOptions.includes('--shard')) {
+    process.env.JEST_SHARD = 'true';
+  }
+
   // Use TEST_BROWSERS if set, otherwise get from --browser option
   if (!process.env.TEST_BROWSERS && runnerOptions.browsers) {
     if (Array.isArray(runnerOptions.browsers))