feat(core): add install modes

.yarn/versions/fa5cc541.yml

@@ -0,0 +1,30 @@
+releases:
+  "@yarnpkg/cli": major
+  "@yarnpkg/core": minor
+  "@yarnpkg/plugin-essentials": major
+
+declined:
+  - "@yarnpkg/plugin-compat"
+  - "@yarnpkg/plugin-constraints"
+  - "@yarnpkg/plugin-dlx"
+  - "@yarnpkg/plugin-exec"
+  - "@yarnpkg/plugin-file"
+  - "@yarnpkg/plugin-git"
+  - "@yarnpkg/plugin-github"
+  - "@yarnpkg/plugin-http"
+  - "@yarnpkg/plugin-init"
+  - "@yarnpkg/plugin-interactive-tools"
+  - "@yarnpkg/plugin-link"
+  - "@yarnpkg/plugin-node-modules"
+  - "@yarnpkg/plugin-npm"
+  - "@yarnpkg/plugin-npm-cli"
+  - "@yarnpkg/plugin-pack"
+  - "@yarnpkg/plugin-patch"
+  - "@yarnpkg/plugin-pnp"
+  - "@yarnpkg/plugin-stage"
+  - "@yarnpkg/plugin-typescript"
+  - "@yarnpkg/plugin-version"
+  - "@yarnpkg/plugin-workspace-tools"
+  - "@yarnpkg/builder"
+  - "@yarnpkg/doctor"
+  - "@yarnpkg/pnpify"

packages/acceptance-tests/pkg-tests-specs/sources/commands/__snapshots__/install.test.js.snap

@@ -18,6 +18,23 @@ Object {
 }
 `;
 
+exports[`Commands install it should fetch only required packages when using \`--mode=lockfile-update\` 1`] = `
+Object {
+  "code": 0,
+  "stderr": "",
+  "stdout": "➤ YN0000: ┌ Resolution step
+➤ YN0000: └ Completed
+➤ YN0000: ┌ Fetch step
+➤ YN0013: │ no-deps@npm:2.0.0 can't be found in the cache and will be fetched from the remote registry
+➤ YN0000: └ Completed
+➤ YN0000: ┌ Link step
+➤ YN0073: │ Skipped due to mode=lockfile-update
+➤ YN0000: └ Completed
+➤ YN0000: Done with warnings
+",
+}
+`;
+
 exports[`Commands install it should print the logs to the standard output when using --inline-builds 1`] = `
 "➤ YN0000: ┌ Resolution step
 ➤ YN0000: └ Completed
@@ -37,7 +54,7 @@ exports[`Commands install it should print the logs to the standard output when u
 "
 `;
 
-exports[`Commands install it should skip build scripts when using --skip-builds 1`] = `
+exports[`Commands install it should skip build scripts when using --mode=skipped-builds 1`] = `
 "➤ YN0000: ┌ Resolution step
 ➤ YN0000: └ Completed
 ➤ YN0000: ┌ Fetch step

packages/acceptance-tests/pkg-tests-specs/sources/commands/install.test.js

@@ -20,20 +20,20 @@ describe(`Commands`, () => {
     );
 
     test(
-      `it should skip build scripts when using --skip-builds`,
+      `it should skip build scripts when using --mode=skipped-builds`,
       makeTemporaryEnv({
         dependencies: {
           [`no-deps-scripted`]: `1.0.0`,
         },
       }, async ({path, run, source}) => {
-        const {stdout} = await run(`install`, `--inline-builds`, `--skip-builds`);
+        const {stdout} = await run(`install`, `--inline-builds`, `--mode=skipped-builds`);
 
         await expect(stdout).toMatchSnapshot();
       }),
     );
 
     test(
-      `it shouldn't impact how artifacts are generated when using --skip-builds`,
+      `it shouldn't impact how artifacts are generated when using --mode=skipped-builds`,
       makeTemporaryEnv({
         dependencies: {
           [`no-deps-scripted`]: `1.0.0`,
@@ -46,7 +46,7 @@ describe(`Commands`, () => {
 
         await xfs.removePromise(pnpPath);
 
-        await run(`install`, `--skip-builds`);
+        await run(`install`, `--mode=skipped-builds`);
         const pnpFileWithoutBuilds = await xfs.readFilePromise(pnpPath);
 
         expect(pnpFileWithBuilds).toEqual(pnpFileWithoutBuilds);
@@ -349,5 +349,38 @@ describe(`Commands`, () => {
         expect(stdout).not.toMatch(/YN0004/g);
       }),
     );
+
+    test(
+      `it should fetch only required packages when using \`--mode=lockfile-update\``,
+      makeTemporaryEnv({
+        dependencies: {
+          [`one-fixed-dep`]: `1.0.0`,
+          [`no-deps`]: `1.0.0`,
+        },
+      }, async ({path, run, source}) => {
+        await run(`install`, `--mode=lockfile-update`);
+
+        const cacheBefore = await xfs.readdirPromise(`${path}/.yarn/cache`);
+        expect(cacheBefore.find(entry => entry.includes(`one-fixed-dep-npm-1.0.0`))).toBeDefined();
+        expect(cacheBefore.find(entry => entry.includes(`no-deps-npm-1.0.0`))).toBeDefined();
+
+        await xfs.writeJsonPromise(`${path}/package.json`, {
+          dependencies: {
+            [`one-fixed-dep`]: `1.0.0`,
+            [`no-deps`]: `2.0.0`,
+          },
+        });
+        await xfs.removePromise(`${path}/.yarn/cache`);
+        await xfs.mkdirPromise(`${path}/.yarn/cache`, {recursive: true});
+
+        const {code, stdout, stderr} = await run(`install`, `--mode=lockfile-update`);
+        await expect({code, stdout, stderr}).toMatchSnapshot();
+
+        const cacheAfter = await xfs.readdirPromise(`${path}/.yarn/cache`);
+        expect(cacheAfter.find(entry => entry.includes(`one-fixed-dep-npm-1.0.0`))).toBeUndefined();
+        expect(cacheAfter.find(entry => entry.includes(`no-deps-npm-1.0.0`))).toBeUndefined();
+        expect(cacheAfter.find(entry => entry.includes(`no-deps-npm-2.0.0`))).toBeDefined();
+      }),
+    );
   });
 });

packages/plugin-essentials/sources/commands/install.ts

@@ -1,8 +1,9 @@
-import {BaseCommand, WorkspaceRequiredError}                                                from '@yarnpkg/cli';
-import {Configuration, Cache, MessageName, Project, ReportError, StreamReport, formatUtils} from '@yarnpkg/core';
-import {xfs, ppath, Filename}                                                               from '@yarnpkg/fslib';
-import {parseSyml, stringifySyml}                                                           from '@yarnpkg/parsers';
-import {Command, Option, Usage}                                                             from 'clipanion';
+import {BaseCommand, WorkspaceRequiredError}                                                             from '@yarnpkg/cli';
+import {Configuration, Cache, MessageName, Project, ReportError, StreamReport, formatUtils, InstallMode} from '@yarnpkg/core';
+import {xfs, ppath, Filename}                                                                            from '@yarnpkg/fslib';
+import {parseSyml, stringifySyml}                                                                        from '@yarnpkg/parsers';
+import {Command, Option, Usage}                                                                          from 'clipanion';
+import * as t                                                                                            from 'typanion';
 
 // eslint-disable-next-line arca/no-default-export
 export default class YarnCommand extends BaseCommand {
@@ -34,7 +35,11 @@ export default class YarnCommand extends BaseCommand {
 
       If the \`--inline-builds\` option is set, Yarn will verbosely print the output of the build steps of your dependencies (instead of writing them into individual files). This is likely useful mostly for debug purposes only when using Docker-like environments.
 
-      If the \`--skip-builds\` option is set, Yarn will not run the build scripts at all. Note that this is different from setting \`enableScripts\` to false because the later will disable build scripts, and thus affect the content of the artifacts generated on disk, whereas the former will just disable the build step - but not the scripts themselves, which just won't run.
+      If the \`--mode=<mode>\` option is set, Yarn will change which artifacts are generated. The modes currently supported are:
+
+      - \`skipped-builds\` will not run the build scripts at all. Note that this is different from setting \`enableScripts\` to false because the later will disable build scripts, and thus affect the content of the artifacts generated on disk, whereas the former will just disable the build step - but not the scripts themselves, which just won't run.
+
+      - \`lockfile-update\` will skip the link step altogether, and only fetch packages that are missing from the lockfile (or that have no associated checksums). This mode is typically used by tools like Renovate or Dependabot to keep a lockfile up-to-date without incurring the full install cost.
     `,
     examples: [[
       `Install the project`,
@@ -68,8 +73,9 @@ export default class YarnCommand extends BaseCommand {
     description: `Verbosely print the output of the build steps of dependencies`,
   });
 
-  skipBuilds = Option.Boolean(`--skip-builds`, false, {
-    description: `Skip the build step altogether`,
+  mode = Option.String(`--mode`, {
+    description: `Change what artifacts installs generate`,
+    validator: t.isEnum(InstallMode),
   });
 
   // Legacy flags; will emit errors or warnings when used
@@ -309,7 +315,7 @@ export default class YarnCommand extends BaseCommand {
       stdout: this.context.stdout,
       includeLogs: true,
     }, async (report: StreamReport) => {
-      await project.install({cache, report, immutable, skipBuild: this.skipBuilds});
+      await project.install({cache, report, immutable, mode: this.mode});
     });
 
     return report.exitCode();

packages/yarnpkg-core/sources/MessageName.ts

@@ -75,6 +75,7 @@ export enum MessageName {
   AUTO_NM_SUCCESS = 70,
   NM_CANT_INSTALL_EXTERNAL_SOFT_LINK = 71,
   NM_PRESERVE_SYMLINKS_REQUIRED = 72,
+  UPDATE_LOCKFILE_ONLY_SKIP_LINK = 73,
 }
 
 export function stringifyMessageName(name: MessageName | number): string {

packages/yarnpkg-core/sources/Project.ts

@@ -55,6 +55,19 @@ const FETCHER_CONCURRENCY = 32;
 const gzip = promisify(zlib.gzip);
 const gunzip = promisify(zlib.gunzip);
 
+export enum InstallMode {
+  /**
+   * Doesn't run the link step, and only fetches what's necessary to compute
+   * an updated lockfile.
+   */
+  LockfileUpdate = `lockfile-update`,
+
+  /**
+   * Don't run the build scripts.
+   */
+  SkippedBuilds = `skipped-builds`,
+}
+
 export type InstallOptions = {
   /**
    * Instance of the cache that the project will use when packages have to be
@@ -98,18 +111,22 @@ export type InstallOptions = {
    */
   lockfileOnly?: boolean,
 
+  /**
+   * Changes which artifacts are generated during the install. Check the
+   * enumeration documentation for details.
+   */
+  mode?: InstallMode,
+
   /**
    * If true (the default), Yarn will update the workspace manifests once the
    * install has completed.
    */
   persistProject?: boolean,
 
   /**
-   * If true, Yarn will skip the build step during the install. Contrary to
-   * setting the `enableScripts` setting to false, setting this won't cause
-   * the generated artifacts to change.
+   * @deprecated Use `mode=skipped-builds`
    */
-  skipBuild?: boolean,
+  skipBuild?: never,
 };
 
 const INSTALL_STATE_FIELDS = {
@@ -625,7 +642,7 @@ export class Project {
     return null;
   }
 
-  async resolveEverything(opts: {report: Report, lockfileOnly: true, resolver?: Resolver} | {report: Report, lockfileOnly?: boolean, cache: Cache, resolver?: Resolver}) {
+  async resolveEverything(opts: Pick<InstallOptions, `report` | `resolver` | `mode`> & ({report: Report, lockfileOnly: true} | {lockfileOnly?: boolean, cache: Cache})) {
     if (!this.workspacesByCwd || !this.workspacesByIdent)
       throw new Error(`Workspaces must have been setup before calling this function`);
 
@@ -830,11 +847,11 @@ export class Project {
     this.refreshWorkspaceDependencies();
   }
 
-  async fetchEverything({cache, report, fetcher: userFetcher}: InstallOptions) {
+  async fetchEverything({cache, report, fetcher: userFetcher, mode}: InstallOptions) {
     const fetcher = userFetcher || this.configuration.makeFetcher();
     const fetcherOptions = {checksums: this.storedChecksums, project: this, cache, fetcher, report};
 
-    const locatorHashes = Array.from(
+    let locatorHashes = Array.from(
       new Set(
         miscUtils.sortMap(this.storedResolutions.values(), [
           (locatorHash: LocatorHash) => {
@@ -848,6 +865,12 @@ export class Project {
       )
     );
 
+    // In "dependency update" mode, we won't trigger the link step. As a
+    // result, we only need to fetch the packages that are missing their
+    // hashes (to add them to the lockfile).
+    if (mode === InstallMode.LockfileUpdate)
+      locatorHashes = locatorHashes.filter(locatorHash => !this.storedChecksums.has(locatorHash));
+
     let firstError = false;
 
     const progress = Report.progressViaCounter(locatorHashes.length);
@@ -892,7 +915,7 @@ export class Project {
     }
   }
 
-  async linkEverything({cache, report, fetcher: optFetcher, skipBuild}: InstallOptions) {
+  async linkEverything({cache, report, fetcher: optFetcher, mode}: InstallOptions) {
     const fetcher = optFetcher || this.configuration.makeFetcher();
     const fetcherOptions = {checksums: this.storedChecksums, project: this, cache, fetcher, report, skipIntegrityCheck: true};
 
@@ -1107,7 +1130,7 @@ export class Project {
 
     // Step 4: Build the packages in multiple steps
 
-    if (skipBuild)
+    if (mode === InstallMode.SkippedBuilds)
       return;
 
     const readyPackages = new Set(this.storedPackages.keys());
@@ -1439,7 +1462,7 @@ export class Project {
     await opts.report.startTimerPromise(`Fetch step`, async () => {
       await this.fetchEverything(opts);
 
-      if (typeof opts.persistProject === `undefined` || opts.persistProject) {
+      if ((typeof opts.persistProject === `undefined` || opts.persistProject) && opts.mode !== InstallMode.LockfileUpdate) {
         await this.cacheCleanup(opts);
       }
     });
@@ -1456,6 +1479,11 @@ export class Project {
       await this.persist();
 
     await opts.report.startTimerPromise(`Link step`, async () => {
+      if (opts.mode === InstallMode.LockfileUpdate) {
+        opts.report.reportWarning(MessageName.UPDATE_LOCKFILE_ONLY_SKIP_LINK, `Skipped due to ${formatUtils.pretty(this.configuration, `mode=lockfile-update`, formatUtils.Type.CODE)}`);
+        return;
+      }
+
       await this.linkEverything(opts);
 
       const after = await Promise.all(immutablePatterns.map(async pattern => {

packages/yarnpkg-core/sources/index.ts

@@ -25,7 +25,7 @@ export type {AllDependencies, HardDependencies, DependencyMeta, PeerDependencyMe
 export {MessageName}                                                                                      from './MessageName';
 export type {CommandContext, Hooks, Plugin}                                                               from './Plugin';
 export type {PeerRequirement}                                                                             from './Project';
-export {Project}                                                                                          from './Project';
+export {Project, InstallMode}                                                                             from './Project';
 export {TAG_REGEXP}                                                                                       from './ProtocolResolver';
 export {ReportError, Report}                                                                              from './Report';
 export type {Resolver, ResolveOptions, MinimalResolveOptions}                                             from './Resolver';