From 3557ebd2db7176b717465b764251e803491e4f3f Mon Sep 17 00:00:00 2001 From: Fuxing Loh Date: Wed, 31 May 2023 11:33:04 +0800 Subject: [PATCH 1/2] feat(sticky-turbo): support extends on turbo.json feat(sticky-jest-turbo): detect turbo is the current process to prevent re-invocation chore(*): fix wrong lint script This PR contains breaking changes when `sticky-turbo` is used programmatically. --- packages/sticky-jest/package.json | 2 +- packages/sticky-turbo-jest/jest-turbo.js | 51 ++---- packages/sticky-turbo/package.json | 9 +- packages/sticky-turbo/src/Turbo.ts | 214 +++++++++++++++++++++++ packages/sticky-turbo/src/Turbo.unit.ts | 41 +++++ packages/sticky-turbo/src/TurboJson.ts | 64 ------- packages/sticky-turbo/src/index.ts | 2 +- pnpm-lock.yaml | 4 + 8 files changed, 286 insertions(+), 101 deletions(-) create mode 100644 packages/sticky-turbo/src/Turbo.ts create mode 100644 packages/sticky-turbo/src/Turbo.unit.ts delete mode 100644 packages/sticky-turbo/src/TurboJson.ts diff --git a/packages/sticky-jest/package.json b/packages/sticky-jest/package.json index 084bb27..e2ec7c2 100644 --- a/packages/sticky-jest/package.json +++ b/packages/sticky-jest/package.json @@ -11,7 +11,7 @@ "scripts": { "build": "tsc -b ./tsconfig.build.json", "clean": "rm -rf dist", - "lint": "eslint src", + "lint": "eslint .", "test": "jest" }, "eslintConfig": { diff --git a/packages/sticky-turbo-jest/jest-turbo.js b/packages/sticky-turbo-jest/jest-turbo.js index 28061cf..3376281 100644 --- a/packages/sticky-turbo-jest/jest-turbo.js +++ b/packages/sticky-turbo-jest/jest-turbo.js @@ -1,41 +1,24 @@ -const { spawnSync } = require('node:child_process'); -const { existsSync } = require('node:fs'); -const { join } = require('node:path'); -const { TurboJson, PackageJson } = require('@birthdayresearch/sticky-turbo'); +const { Turbo } = require('@birthdayresearch/sticky-turbo'); module.exports = async function (_, project) { - const turboJson = new TurboJson(project.rootDir); - const packageJson = new PackageJson(project.rootDir); - const displayName = project.displayName.name; - - for (const script of turboJson.getPipeline(displayName)?.dependsOn ?? []) { - if (script.startsWith('^')) { - await run(turboJson.getRootDir(), script.substring(1), `${packageJson.getName()}^...`); - } else if (packageJson.hasScript(script)) { - await run(turboJson.getRootDir(), script, packageJson.getName()); - } + if (isTurbo()) { + console.log('jest-turbo: turbo is already running. skipping...'); + return; } -}; -async function run(rootDir, script, filter) { - const bin = './node_modules/.bin/turbo'; - const args = ['run', script, `--filter=${filter}`, `--output-logs=new-only`]; - - if (!existsSync(join(rootDir, bin))) { - throw new Error(`Cannot find ${bin}`); - } - - const spawn = spawnSync(bin, args, { - stdio: ['inherit', 'inherit', 'pipe'], - cwd: rootDir, + const turbo = new Turbo(project.rootDir); + // project.displayName represent the script + const script = project.displayName.name; + turbo.runBefore(script, { + 'output-logs': 'new-only', }); +}; - // Throw error if non-zero exit code encountered - if (spawn.status !== 0) { - const failureMessage = - spawn.stderr && spawn.stderr.length > 0 - ? spawn.stderr.toString('utf-8') - : `Encountered non-zero exit code while running script: ${script}`; - throw new Error(failureMessage); - } +/** + * Detects if the current process is a turbo invocation. + * If so, we don't need to run turbo again. + * @return {boolean} + */ +function isTurbo() { + return process.env.TURBO_INVOCATION_DIR !== undefined; } diff --git a/packages/sticky-turbo/package.json b/packages/sticky-turbo/package.json index aedfd7d..93eb99f 100644 --- a/packages/sticky-turbo/package.json +++ b/packages/sticky-turbo/package.json @@ -10,7 +10,8 @@ "scripts": { "build": "tsc -b ./tsconfig.build.json", "clean": "rm -rf dist", - "lint": "eslint src" + "lint": "eslint .", + "test": "jest" }, "eslintConfig": { "parserOptions": { @@ -20,7 +21,13 @@ "@birthdayresearch" ] }, + "jest": { + "preset": "@birthdayresearch/sticky-jest" + }, "dependencies": { "turbo": "1.9.8" + }, + "devDependencies": { + "@birthdayresearch/sticky-jest": "workspace:*" } } diff --git a/packages/sticky-turbo/src/Turbo.ts b/packages/sticky-turbo/src/Turbo.ts new file mode 100644 index 0000000..d60844a --- /dev/null +++ b/packages/sticky-turbo/src/Turbo.ts @@ -0,0 +1,214 @@ +import { spawnSync, SpawnSyncReturns, StdioOptions } from 'node:child_process'; +import { existsSync, readFileSync } from 'node:fs'; +import { dirname, join } from 'node:path'; + +import { PackageJson } from './PackageJson'; + +export class Turbo { + private readonly rootDir: string; + + constructor(protected readonly cwd: string, depth = 4) { + const path = findRootTurboJsonPath(cwd, depth); + if (path === undefined) { + throw new Error('turbo.json not found'); + } + this.rootDir = dirname(path); + } + + getRootDir(): string { + return this.rootDir; + } + + /** + * @param {string} task to plan + * @param {Record} opts for turbo CLI + * @return {string[]} planned list of packages to run + */ + planPackages(task: string, opts?: Record): string[] { + const { tasks } = this.plan(task, opts); + return tasks + .filter((t: any) => t.task === task) + .filter((t: any) => t.command !== '') + .map((t: any) => t.package as string); + } + + /** + * @param {string} task to run + * @param {Record} opts for turbo CLI + * @deprecated use runTask + */ + run(task: string, opts?: Record): void { + this.exec(['run', task], opts); + } + + /** + * @param {string} task to run + * @param {Record} opts for turbo CLI + */ + runTask(task: string, opts?: Record): void { + this.exec(['run', task, ...optionsAsArgs(opts)]); + } + + runTasks( + pipelines: { + task: string; + opts?: Record; + }[], + opts?: Record, + ): void { + /** + * Optimize the order of the pipelines to run where the most concurrent pipelines are run first. + */ + const priority: Record = { + 'build:docker': 1, + default: Number.MAX_VALUE, + }; + pipelines.sort((a, b): number => (priority[a.task] || priority.default) - (priority[b.task] || priority.default)); + + for (const pipeline of pipelines) { + this.exec([ + 'run', + pipeline.task, + ...optionsAsArgs({ + ...opts, + ...pipeline.opts, + }), + ]); + } + } + + /** + * By taking advantage of content-aware hashing from turborepo. `dependsOn` only runs if the pipeline `inputs` has + * changed. + * + * @param {string} task `dependsOn` without running the script + * @param {Record} opts for turbo CLI + */ + runBefore(task: string, opts?: Record): void { + const packageJson = new PackageJson(this.cwd); + const plan = this.plan(task, { + ...opts, + only: undefined, + filter: packageJson.getName(), + }); + + const pipelines = plan.tasks[0].resolvedTaskDefinition.dependsOn + .map((dependOnScript: string) => { + if (dependOnScript.startsWith('^')) { + return { + task: dependOnScript.substring(1), + opts: { + filter: `${packageJson.getName()}^...`, + }, + }; + } + if (packageJson.hasScript(dependOnScript)) { + return { + task: dependOnScript, + opts: { + filter: packageJson.getName(), + }, + }; + } + + return undefined; + }) + .filter((p: any) => p !== undefined); + + this.runTasks(pipelines, opts); + } + + /** + * @param {string} task to plan + * @param {Record} opts for turbo CLI + * @return {any} json object + */ + private plan(task: string, opts?: Record): any { + const spawn = this.exec( + ['run', task], + { + ...opts, + dry: 'json', + }, + 'pipe', + ); + + return JSON.parse(spawn.stdout); + } + + /** + * @throws Error + */ + private exec( + args: string[], + opts?: Record, + stdio: StdioOptions = ['inherit', 'inherit', 'pipe'], + ): SpawnSyncReturns { + const bin = './node_modules/.bin/turbo'; + + if (!existsSync(join(this.getRootDir(), bin))) { + throw new Error(`Cannot find ${bin}`); + } + + const spawn = spawnSync(bin, [...args, ...optionsAsArgs(opts)], { + stdio, + maxBuffer: 20_000_000, + cwd: this.getRootDir(), + encoding: 'utf-8', + }); + + // throw an error if non-zero exit code encountered + if (spawn.status !== 0) { + if (spawn.stderr?.length === 0) { + throw new Error(`Encountered non-zero exit code while running: ${args.join(' ')}`); + } + if (spawn.stderr?.includes('error preparing engine: Could not find the following tasks in project: ')) { + // Allow skipping of scripts that are not found in the project. + // Porting back a desired behavior that was present in <1.8.0 + // TODO: we need to hide this misleading error "Could not find the following tasks in project" in console stdout + // See https://github.com/vercel/turbo/pull/3828 + return spawn; + } + + throw new Error(spawn.stderr); + } + + return spawn; + } +} + +/** + * Find the path of root turbo.json, locates where the monorepo root is. + * + * @param cwd {string} of the current working directory + * @param depth {number} on how far up to search + */ +export function findRootTurboJsonPath(cwd: string, depth: number = 4): string | undefined { + const paths = cwd.split('/'); + + for (let i = 0; i < depth; i += 1) { + const path = `${paths.join('/')}/turbo.json`; + if (existsSync(path)) { + const object = JSON.parse(readFileSync(path, { encoding: 'utf-8' })); + if (!object.extends?.includes('//')) { + return path; + } + } + paths.pop(); + } + + return undefined; +} + +function optionsAsArgs(options?: Record): string[] { + if (!options) { + return []; + } + + return Object.entries(options).map(([key, value]) => { + if (value === undefined) { + return `--${key}`; + } + return `--${key}=${value}`; + }); +} diff --git a/packages/sticky-turbo/src/Turbo.unit.ts b/packages/sticky-turbo/src/Turbo.unit.ts new file mode 100644 index 0000000..1bc28b7 --- /dev/null +++ b/packages/sticky-turbo/src/Turbo.unit.ts @@ -0,0 +1,41 @@ +import * as process from 'process'; + +import { Turbo } from './Turbo'; + +it(`turbo.runTask('invalid-script') should be ignored; porting a desired behavior that was present in <1.8.0`, async () => { + const turbo = new Turbo(process.cwd()); + turbo.runTask('invalid-script', { + filter: `@levain-dev/sticky-turbo^...`, + }); +}); + +it(`turbo.runTasks('invalid-script') should be ignored; porting a desired behavior that was present in <1.8.0`, async () => { + const turbo = new Turbo(process.cwd()); + turbo.runTasks([ + { + task: 'invalid-script', + opts: { + filter: `@levain-dev/sticky-turbo^...`, + }, + }, + ]); +}); + +it(`should turbo.runBefore('build') without failure`, async () => { + const turbo = new Turbo(process.cwd()); + turbo.runBefore('build', { + dry: 'json', + }); +}); + +it(`turbo.planPackages('test') should generate a list of packages to run`, async () => { + const turbo = new Turbo(process.cwd()); + const packages = turbo.planPackages('test'); + expect(packages).toStrictEqual( + expect.arrayContaining([ + '@birthdayresearch/sticky-jest', + '@birthdayresearch/sticky-testcontainers', + '@birthdayresearch/sticky-turbo', + ]), + ); +}); diff --git a/packages/sticky-turbo/src/TurboJson.ts b/packages/sticky-turbo/src/TurboJson.ts deleted file mode 100644 index e3823e1..0000000 --- a/packages/sticky-turbo/src/TurboJson.ts +++ /dev/null @@ -1,64 +0,0 @@ -import { existsSync, readFileSync } from 'node:fs'; -import { dirname } from 'node:path'; - -interface TurboPipeline { - inputs?: string[]; - outputs?: string[]; - dependsOn?: string[]; -} - -/** - * `turbo.json` represented as constructable Class. - * This class will automatically locate where is the turbo.json and subsequently finding the project root. - */ -export class TurboJson { - private readonly json: { - pipeline: Record; - }; - - private readonly rootDir: string; - - constructor(cwd: string, depth = 4) { - const path = getTurboJsonPath(cwd, depth); - if (path === undefined) { - throw new Error('turbo.json not found'); - } - this.rootDir = dirname(path); - this.json = JSON.parse(readFileSync(path, { encoding: 'utf-8' })); - } - - /** - * @return {string} root of the project dir - */ - getRootDir() { - return this.rootDir; - } - - /** - * @param {string} name of a pipeline - * @return {TurboPipeline} with cache information about the pipeline - */ - getPipeline(name: string): TurboPipeline | undefined { - return this.json.pipeline[name]; - } -} - -/** - * Find turbo.json file, for locating where the monorepo root is. - * - * @param cwd {string} of the current working directory - * @param depth {number} on how far up to search - */ -export function getTurboJsonPath(cwd: string, depth: number = 4): string | undefined { - const paths = cwd.split('/'); - - for (let i = 0; i < depth; i += 1) { - const path = `${paths.join('/')}/turbo.json`; - if (existsSync(path)) { - return path; - } - paths.pop(); - } - - return undefined; -} diff --git a/packages/sticky-turbo/src/index.ts b/packages/sticky-turbo/src/index.ts index 0617c15..6eaacdb 100644 --- a/packages/sticky-turbo/src/index.ts +++ b/packages/sticky-turbo/src/index.ts @@ -1,2 +1,2 @@ export * from './PackageJson'; -export * from './TurboJson'; +export * from './Turbo'; diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 94a3266..b538d3a 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -119,6 +119,10 @@ importers: turbo: specifier: 1.9.8 version: 1.9.8 + devDependencies: + '@birthdayresearch/sticky-jest': + specifier: workspace:* + version: link:../sticky-jest packages/sticky-turbo-jest: dependencies: From 64f445eb762a7ea5c6eadc174a0494a030ed20fd Mon Sep 17 00:00:00 2001 From: Fuxing Loh Date: Wed, 31 May 2023 11:38:50 +0800 Subject: [PATCH 2/2] fix tsconfig --- packages/sticky-jest/tsconfig.build.json | 5 +++++ packages/sticky-jest/tsconfig.json | 8 +++----- packages/sticky-testcontainers/tsconfig.json | 8 +++----- packages/sticky-turbo/tsconfig.build.json | 5 +++++ packages/sticky-turbo/tsconfig.json | 8 +++----- 5 files changed, 19 insertions(+), 15 deletions(-) diff --git a/packages/sticky-jest/tsconfig.build.json b/packages/sticky-jest/tsconfig.build.json index 0d3d95f..95f8624 100644 --- a/packages/sticky-jest/tsconfig.build.json +++ b/packages/sticky-jest/tsconfig.build.json @@ -1,4 +1,9 @@ { "extends": "./tsconfig.json", + "compilerOptions": { + "rootDir": "./src", + "outDir": "./dist" + }, + "include": ["./src"], "exclude": ["**/*.unit.ts"] } diff --git a/packages/sticky-jest/tsconfig.json b/packages/sticky-jest/tsconfig.json index 237546e..92e49bc 100644 --- a/packages/sticky-jest/tsconfig.json +++ b/packages/sticky-jest/tsconfig.json @@ -1,8 +1,6 @@ { - "extends": "@birthdayresearch/sticky-typescript/tsconfig.json", + "extends": "@birthdayresearch/sticky-typescript", "compilerOptions": { - "rootDir": "./src", - "outDir": "./dist" - }, - "include": ["src"] + "rootDir": "./" + } } diff --git a/packages/sticky-testcontainers/tsconfig.json b/packages/sticky-testcontainers/tsconfig.json index f4e41d5..92e49bc 100644 --- a/packages/sticky-testcontainers/tsconfig.json +++ b/packages/sticky-testcontainers/tsconfig.json @@ -1,8 +1,6 @@ { - "extends": "@birthdayresearch/sticky-typescript/tsconfig.json", + "extends": "@birthdayresearch/sticky-typescript", "compilerOptions": { - "rootDir": "./", - "outDir": "./dist" - }, - "include": ["src"] + "rootDir": "./" + } } diff --git a/packages/sticky-turbo/tsconfig.build.json b/packages/sticky-turbo/tsconfig.build.json index 0d3d95f..95f8624 100644 --- a/packages/sticky-turbo/tsconfig.build.json +++ b/packages/sticky-turbo/tsconfig.build.json @@ -1,4 +1,9 @@ { "extends": "./tsconfig.json", + "compilerOptions": { + "rootDir": "./src", + "outDir": "./dist" + }, + "include": ["./src"], "exclude": ["**/*.unit.ts"] } diff --git a/packages/sticky-turbo/tsconfig.json b/packages/sticky-turbo/tsconfig.json index 237546e..92e49bc 100644 --- a/packages/sticky-turbo/tsconfig.json +++ b/packages/sticky-turbo/tsconfig.json @@ -1,8 +1,6 @@ { - "extends": "@birthdayresearch/sticky-typescript/tsconfig.json", + "extends": "@birthdayresearch/sticky-typescript", "compilerOptions": { - "rootDir": "./src", - "outDir": "./dist" - }, - "include": ["src"] + "rootDir": "./" + } }