Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

refactor!: move ZipFS and ZipOpenFS into @yarnpkg/libzip #4853

Merged
merged 5 commits into from
Sep 19, 2022
Merged

refactor!: move ZipFS and ZipOpenFS into @yarnpkg/libzip #4853

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
fd7c5d9
feat: Moves ZipFS and ZipOpenFS into @yarnpkg/libzip
arcanis Sep 16, 2022
78bd1f9
Version
arcanis Sep 16, 2022
78a2333
Fixes unit tests
arcanis Sep 19, 2022
e30c95f
Fixes constraint
arcanis Sep 19, 2022
7241242
Fixes the lockfile
arcanis Sep 19, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
148 changes: 84 additions & 64 deletions .pnp.cjs
View file
Open in desktop

Large diffs are not rendered by default.

39 changes: 39 additions & 0 deletions .yarn/versions/5be37cc3.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
releases:
"@yarnpkg/cli": major
"@yarnpkg/core": major
"@yarnpkg/fslib": major
"@yarnpkg/libzip": major

declined:
- "@yarnpkg/plugin-compat"
- "@yarnpkg/plugin-constraints"
- "@yarnpkg/plugin-dlx"
- "@yarnpkg/plugin-essentials"
- "@yarnpkg/plugin-exec"
- "@yarnpkg/plugin-file"
- "@yarnpkg/plugin-git"
- "@yarnpkg/plugin-github"
- "@yarnpkg/plugin-http"
- "@yarnpkg/plugin-init"
- "@yarnpkg/plugin-interactive-tools"
- "@yarnpkg/plugin-link"
- "@yarnpkg/plugin-nm"
- "@yarnpkg/plugin-npm"
- "@yarnpkg/plugin-npm-cli"
- "@yarnpkg/plugin-pack"
- "@yarnpkg/plugin-patch"
- "@yarnpkg/plugin-pnp"
- "@yarnpkg/plugin-pnpm"
- "@yarnpkg/plugin-stage"
- "@yarnpkg/plugin-typescript"
- "@yarnpkg/plugin-version"
- "@yarnpkg/plugin-workspace-tools"
- vscode-zipfs
- "@yarnpkg/builder"
- "@yarnpkg/doctor"
- "@yarnpkg/extensions"
- "@yarnpkg/nm"
- "@yarnpkg/pnp"
- "@yarnpkg/pnpify"
- "@yarnpkg/sdks"
- "@yarnpkg/shell"
1 change: 1 addition & 0 deletions packages/plugin-file/package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
},
"dependencies": {
"@yarnpkg/fslib": "workspace:^",
"@yarnpkg/libzip": "workspace:^",
"tslib": "^2.4.0"
},
"peerDependencies": {
Expand Down
3 changes: 2 additions & 1 deletion packages/plugin-file/sources/fileUtils.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
import {structUtils, FetchOptions, Locator, miscUtils, tgzUtils, Ident, FetchResult} from '@yarnpkg/core';
import {ppath, PortablePath, npath, CwdFS, ZipFS} from '@yarnpkg/fslib';
import {ppath, PortablePath, npath, CwdFS} from '@yarnpkg/fslib';
import {ZipFS} from '@yarnpkg/libzip';

export function parseSpec(spec: string) {
const {params, selector} = structUtils.parseRange(spec);
Expand Down
5 changes: 2 additions & 3 deletions packages/plugin-nm/sources/NodeModulesLinker.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,8 @@ import {Linker, LinkOptions, MinimalLinkOptions, LinkType} from
import {LocatorHash, Descriptor, DependencyMeta, Configuration} from '@yarnpkg/core';
import {MessageName, Project, FetchResult, Installer} from '@yarnpkg/core';
import {PortablePath, npath, ppath, toFilename, Filename} from '@yarnpkg/fslib';
import {VirtualFS, ZipOpenFS, xfs, FakeFS, NativePath} from '@yarnpkg/fslib';
import {getLibzipPromise} from '@yarnpkg/libzip';
import {VirtualFS, xfs, FakeFS, NativePath} from '@yarnpkg/fslib';
import {ZipOpenFS} from '@yarnpkg/libzip';
import {buildNodeModulesTree} from '@yarnpkg/nm';
import {NodeModulesLocatorMap, buildLocatorMap, NodeModulesHoistingLimits} from '@yarnpkg/nm';
import {parseSyml} from '@yarnpkg/parsers';
Expand Down Expand Up @@ -219,7 +219,6 @@ class NodeModulesInstaller implements Installer {

const defaultFsLayer = new VirtualFS({
baseFs: new ZipOpenFS({
libzip: await getLibzipPromise(),
maxOpenFiles: 80,
readOnlyArchives: true,
}),
Expand Down
5 changes: 2 additions & 3 deletions packages/plugin-nm/sources/PnpLooseLinker.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
import {LinkOptions, structUtils} from '@yarnpkg/core';
import {VirtualFS, ZipOpenFS, ppath, Filename} from '@yarnpkg/fslib';
import {getLibzipPromise} from '@yarnpkg/libzip';
import {VirtualFS, ppath, Filename} from '@yarnpkg/fslib';
import {ZipOpenFS} from '@yarnpkg/libzip';
import {NodeModulesPackageNode, buildNodeModulesTree} from '@yarnpkg/nm';
import {PnpInstaller, PnpLinker} from '@yarnpkg/plugin-pnp';
import {PnpSettings, makeRuntimeApi, DependencyTarget} from '@yarnpkg/pnp';
Expand All @@ -19,7 +19,6 @@ class PnpLooseInstaller extends PnpInstaller {
async transformPnpSettings(pnpSettings: PnpSettings) {
const defaultFsLayer = new VirtualFS({
baseFs: new ZipOpenFS({
libzip: await getLibzipPromise(),
maxOpenFiles: 80,
readOnlyArchives: true,
}),
Expand Down
9 changes: 2 additions & 7 deletions packages/plugin-patch/sources/PatchFetcher.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
import {Fetcher, FetchOptions, MinimalFetchOptions, ReportError, MessageName, Report} from '@yarnpkg/core';
import {Locator} from '@yarnpkg/core';
import {miscUtils, structUtils} from '@yarnpkg/core';
import {ppath, xfs, ZipFS, Filename, CwdFS, PortablePath} from '@yarnpkg/fslib';
import {getLibzipPromise} from '@yarnpkg/libzip';
import {ppath, xfs, Filename, CwdFS, PortablePath} from '@yarnpkg/fslib';
import {ZipFS} from '@yarnpkg/libzip';

import * as patchUtils from './patchUtils';
import {UnmatchedHunkError} from './tools/UnmatchedHunkError';
Expand Down Expand Up @@ -49,11 +49,8 @@ export class PatchFetcher implements Fetcher {
const sourceFetch = await opts.fetcher.fetch(sourceLocator, opts);
const prefixPath = structUtils.getIdentVendorPath(locator);

const libzip = await getLibzipPromise();

// First we create a copy of the package that we'll be free to mutate
const initialCopy = new ZipFS(currentFile, {
libzip,
create: true,
level: opts.project.configuration.get(`compressionLevel`),
});
Expand All @@ -73,7 +70,6 @@ export class PatchFetcher implements Fetcher {
// single time) because it lets us easily rollback when hitting errors
// on optional patches (we just need to call `discardAndClose`).
const patchedPackage = new ZipFS(currentFile, {
libzip,
level: opts.project.configuration.get(`compressionLevel`),
});

Expand Down Expand Up @@ -122,7 +118,6 @@ export class PatchFetcher implements Fetcher {
}

return new ZipFS(currentFile, {
libzip,
level: opts.project.configuration.get(`compressionLevel`),
});
}
Expand Down
7 changes: 3 additions & 4 deletions packages/vscode-zipfs/sources/ZipFSProvider.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,12 +1,11 @@
import {ZipOpenFS, VirtualFS, PosixFS, npath} from '@yarnpkg/fslib';
import {getLibzipSync} from '@yarnpkg/libzip';
import * as vscode from 'vscode';
import {VirtualFS, PosixFS, npath} from '@yarnpkg/fslib';
import {ZipOpenFS} from '@yarnpkg/libzip';
import * as vscode from 'vscode';

export class ZipFSProvider implements vscode.FileSystemProvider {
private readonly fs = new PosixFS(
new VirtualFS({
baseFs: new ZipOpenFS({
libzip: getLibzipSync(),
useCache: true,
maxOpenFiles: 80,
}),
Expand Down
32 changes: 15 additions & 17 deletions packages/yarnpkg-core/sources/Cache.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,16 +1,16 @@
import {FakeFS, LazyFS, NodeFS, ZipFS, PortablePath, Filename, AliasFS} from '@yarnpkg/fslib';
import {ppath, xfs, DEFAULT_COMPRESSION_LEVEL} from '@yarnpkg/fslib';
import {getLibzipPromise} from '@yarnpkg/libzip';
import {randomBytes} from 'crypto';
import fs from 'fs';

import {Configuration} from './Configuration';
import {MessageName} from './MessageName';
import {ReportError} from './Report';
import * as hashUtils from './hashUtils';
import * as miscUtils from './miscUtils';
import * as structUtils from './structUtils';
import {LocatorHash, Locator} from './types';
import {FakeFS, LazyFS, NodeFS, PortablePath, Filename, AliasFS} from '@yarnpkg/fslib';
import {ppath, xfs} from '@yarnpkg/fslib';
import {DEFAULT_COMPRESSION_LEVEL, ZipFS} from '@yarnpkg/libzip';
import {randomBytes} from 'crypto';
import fs from 'fs';

import {Configuration} from './Configuration';
import {MessageName} from './MessageName';
import {ReportError} from './Report';
import * as hashUtils from './hashUtils';
import * as miscUtils from './miscUtils';
import * as structUtils from './structUtils';
import {LocatorHash, Locator} from './types';

const CACHE_VERSION = 9;

Expand Down Expand Up @@ -172,7 +172,7 @@ export class Cache {
// it seem like it actually exist on the disk, at the location of the
// cache the package would fill if it was normally fetched.
const makeMockPackage = () => {
const zipFs = new ZipFS(null, {libzip});
const zipFs = new ZipFS();

const rootPackageDir = ppath.join(PortablePath.root, structUtils.getIdentVendorPath(locator));
zipFs.mkdirSync(rootPackageDir, {recursive: true});
Expand Down Expand Up @@ -363,11 +363,9 @@ export class Cache {

let zipFs: ZipFS | undefined;

const libzip = await getLibzipPromise();

const zipFsBuilder = shouldMock
? () => makeMockPackage()
: () => new ZipFS(cachePath, {baseFs, libzip, readOnly: true});
: () => new ZipFS(cachePath, {baseFs, readOnly: true});

const lazyFs = new LazyFS<PortablePath>(() => miscUtils.prettifySyncErrors(() => {
return zipFs = zipFsBuilder();
Expand Down
2 changes: 1 addition & 1 deletion packages/yarnpkg-core/sources/Configuration.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
import {Filename, PortablePath, npath, ppath, xfs} from '@yarnpkg/fslib';
import {DEFAULT_COMPRESSION_LEVEL} from '@yarnpkg/fslib';
import {DEFAULT_COMPRESSION_LEVEL} from '@yarnpkg/libzip';
import {parseSyml, stringifySyml} from '@yarnpkg/parsers';
import camelcase from 'camelcase';
import {isCI, isPR, GITHUB_ACTIONS} from 'ci-info';
Expand Down
48 changes: 22 additions & 26 deletions packages/yarnpkg-core/sources/scriptUtils.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,25 +1,25 @@
import {CwdFS, Filename, NativePath, PortablePath, ZipOpenFS} from '@yarnpkg/fslib';
import {xfs, npath, ppath, toFilename} from '@yarnpkg/fslib';
import {getLibzipPromise} from '@yarnpkg/libzip';
import {execute} from '@yarnpkg/shell';
import capitalize from 'lodash/capitalize';
import pLimit from 'p-limit';
import {PassThrough, Readable, Writable} from 'stream';

import {Configuration} from './Configuration';
import {Manifest} from './Manifest';
import {MessageName} from './MessageName';
import {Project} from './Project';
import {ReportError, Report} from './Report';
import {StreamReport} from './StreamReport';
import {Workspace} from './Workspace';
import {YarnVersion} from './YarnVersion';
import * as execUtils from './execUtils';
import * as formatUtils from './formatUtils';
import * as miscUtils from './miscUtils';
import * as semverUtils from './semverUtils';
import * as structUtils from './structUtils';
import {LocatorHash, Locator} from './types';
import {CwdFS, Filename, NativePath, PortablePath} from '@yarnpkg/fslib';
import {xfs, npath, ppath, toFilename} from '@yarnpkg/fslib';
import {ZipOpenFS} from '@yarnpkg/libzip';
import {execute} from '@yarnpkg/shell';
import capitalize from 'lodash/capitalize';
import pLimit from 'p-limit';
import {PassThrough, Readable, Writable} from 'stream';

import {Configuration} from './Configuration';
import {Manifest} from './Manifest';
import {MessageName} from './MessageName';
import {Project} from './Project';
import {ReportError, Report} from './Report';
import {StreamReport} from './StreamReport';
import {Workspace} from './Workspace';
import {YarnVersion} from './YarnVersion';
import * as execUtils from './execUtils';
import * as formatUtils from './formatUtils';
import * as miscUtils from './miscUtils';
import * as semverUtils from './semverUtils';
import * as structUtils from './structUtils';
import {LocatorHash, Locator} from './types';

/**
* @internal
Expand Down Expand Up @@ -432,8 +432,6 @@ export async function hasPackageScript(locator: Locator, scriptName: string, {pr
const manifest = await Manifest.find(PortablePath.dot, {baseFs: packageFs});

return manifest.scripts.has(scriptName);
}, {
libzip: await getLibzipPromise(),
});
}

Expand Down Expand Up @@ -545,8 +543,6 @@ async function initializePackageEnvironment(locator: Locator, {project, binFolde
cwd = packageLocation;

return {manifest, binFolder, env, cwd};
}, {
libzip: await getLibzipPromise(),
});
}

Expand Down
22 changes: 10 additions & 12 deletions packages/yarnpkg-core/sources/tgzUtils.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,11 +1,11 @@
import {Filename, FakeFS, PortablePath, ZipCompression, ZipFS, NodeFS, ppath, xfs, npath, constants} from '@yarnpkg/fslib';
import {getLibzipPromise} from '@yarnpkg/libzip';
import {PassThrough, Readable} from 'stream';
import tar from 'tar';
import {Filename, FakeFS, PortablePath, NodeFS, ppath, xfs, npath, constants} from '@yarnpkg/fslib';
import {ZipCompression, ZipFS} from '@yarnpkg/libzip';
import {PassThrough, Readable} from 'stream';
import tar from 'tar';

import {WorkerPool} from './WorkerPool';
import * as miscUtils from './miscUtils';
import {getContent as getZipWorkerSource, ConvertToZipPayload} from './worker-zip';
import {WorkerPool} from './WorkerPool';
import * as miscUtils from './miscUtils';
import {getContent as getZipWorkerSource, ConvertToZipPayload} from './worker-zip';

interface MakeArchiveFromDirectoryOptions {
baseFs?: FakeFS<PortablePath>;
Expand All @@ -15,16 +15,14 @@ interface MakeArchiveFromDirectoryOptions {
}

export async function makeArchiveFromDirectory(source: PortablePath, {baseFs = new NodeFS(), prefixPath = PortablePath.root, compressionLevel, inMemory = false}: MakeArchiveFromDirectoryOptions = {}): Promise<ZipFS> {
const libzip = await getLibzipPromise();

let zipFs;
if (inMemory) {
zipFs = new ZipFS(null, {libzip, level: compressionLevel});
zipFs = new ZipFS(null, {level: compressionLevel});
} else {
const tmpFolder = await xfs.mktempPromise();
const tmpFile = ppath.join(tmpFolder, `archive.zip` as Filename);

zipFs = new ZipFS(tmpFile, {create: true, libzip, level: compressionLevel});
zipFs = new ZipFS(tmpFile, {create: true, level: compressionLevel});
}

const target = ppath.resolve(PortablePath.root, prefixPath!);
Expand All @@ -49,7 +47,7 @@ export async function convertToZip(tgz: Buffer, opts: ExtractBufferOptions) {

await workerPool.run({tmpFile, tgz, opts});

return new ZipFS(tmpFile, {libzip: await getLibzipPromise(), level: opts.compressionLevel});
return new ZipFS(tmpFile, {level: opts.compressionLevel});
}

async function * parseTar(tgz: Buffer) {
Expand Down
6 changes: 3 additions & 3 deletions packages/yarnpkg-core/sources/worker-zip/Worker.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
import {PortablePath, statUtils, ZipFS} from '@yarnpkg/fslib';
import {getLibzipPromise} from '@yarnpkg/libzip';
import {PortablePath, statUtils} from '@yarnpkg/fslib';
import {ZipFS} from '@yarnpkg/libzip';
import {parentPort} from 'worker_threads';

import {extractArchiveTo, ExtractBufferOptions} from '../tgzUtils';
Expand All @@ -13,7 +13,7 @@ parentPort.on(`message`, async (data: ConvertToZipPayload) => {
const {opts, tgz, tmpFile} = data;
const {compressionLevel, ...bufferOpts} = opts;

const zipFs = new ZipFS(tmpFile, {create: true, libzip: await getLibzipPromise(), level: compressionLevel, stats: statUtils.makeDefaultStats()});
const zipFs = new ZipFS(tmpFile, {create: true, level: compressionLevel, stats: statUtils.makeDefaultStats()});

// Buffers sent through Node are turned into regular Uint8Arrays
const tgzBuffer = Buffer.from(tgz.buffer, tgz.byteOffset, tgz.byteLength);
Expand Down
6 changes: 3 additions & 3 deletions packages/yarnpkg-core/tests/TestPlugin.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
import {Descriptor, Fetcher, FetchOptions, LinkType, Locator, MinimalResolveOptions, Package, Plugin, ResolveOptions, Resolver, structUtils} from '@yarnpkg/core';
import {PortablePath, xfs, ZipFS} from '@yarnpkg/fslib';
import {getLibzipPromise} from '@yarnpkg/libzip';
import {PortablePath, xfs} from '@yarnpkg/fslib';
import {ZipFS} from '@yarnpkg/libzip';

export class UnboundDescriptorResolver implements Resolver {
supportsDescriptor(descriptor: Descriptor, opts: MinimalResolveOptions) {
Expand Down Expand Up @@ -148,7 +148,7 @@ class NoopFetcher implements Fetcher {

async fetch(locator: Locator, opts: FetchOptions) {
const tempDir = await xfs.mktempPromise();
return {packageFs: new ZipFS(`${tempDir}/archive.zip` as PortablePath, {libzip: await getLibzipPromise(), create: true}), prefixPath: PortablePath.dot};
return {packageFs: new ZipFS(`${tempDir}/archive.zip` as PortablePath, {create: true}), prefixPath: PortablePath.dot};
}
}

Expand Down
Loading